etlplus 0.14.3__py3-none-any.whl → 0.16.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- etlplus/README.md +4 -4
- etlplus/api/README.md +33 -2
- etlplus/api/auth.py +1 -1
- etlplus/api/config.py +5 -10
- etlplus/api/endpoint_client.py +4 -4
- etlplus/api/pagination/config.py +1 -1
- etlplus/api/pagination/paginator.py +6 -7
- etlplus/api/rate_limiting/config.py +4 -4
- etlplus/api/rate_limiting/rate_limiter.py +1 -1
- etlplus/api/retry_manager.py +2 -2
- etlplus/api/transport.py +1 -1
- etlplus/api/types.py +99 -0
- etlplus/api/utils.py +6 -2
- etlplus/cli/README.md +2 -2
- etlplus/cli/commands.py +75 -42
- etlplus/cli/constants.py +1 -1
- etlplus/cli/handlers.py +33 -15
- etlplus/cli/io.py +2 -2
- etlplus/cli/main.py +2 -2
- etlplus/cli/state.py +4 -7
- etlplus/connector/__init__.py +43 -0
- etlplus/connector/api.py +161 -0
- etlplus/connector/connector.py +26 -0
- etlplus/connector/core.py +132 -0
- etlplus/connector/database.py +122 -0
- etlplus/connector/enums.py +52 -0
- etlplus/connector/file.py +120 -0
- etlplus/connector/types.py +40 -0
- etlplus/connector/utils.py +122 -0
- etlplus/database/README.md +2 -2
- etlplus/database/ddl.py +2 -2
- etlplus/database/engine.py +19 -3
- etlplus/database/orm.py +2 -0
- etlplus/enums.py +1 -33
- etlplus/file/README.md +2 -2
- etlplus/file/_imports.py +1 -0
- etlplus/file/_io.py +52 -4
- etlplus/file/accdb.py +3 -2
- etlplus/file/arrow.py +3 -2
- etlplus/file/avro.py +3 -2
- etlplus/file/bson.py +3 -2
- etlplus/file/cbor.py +3 -2
- etlplus/file/cfg.py +3 -2
- etlplus/file/conf.py +3 -2
- etlplus/file/core.py +11 -8
- etlplus/file/csv.py +3 -2
- etlplus/file/dat.py +3 -2
- etlplus/file/dta.py +3 -2
- etlplus/file/duckdb.py +3 -2
- etlplus/file/enums.py +1 -1
- etlplus/file/feather.py +3 -2
- etlplus/file/fwf.py +3 -2
- etlplus/file/gz.py +3 -2
- etlplus/file/hbs.py +3 -2
- etlplus/file/hdf5.py +3 -2
- etlplus/file/ini.py +3 -2
- etlplus/file/ion.py +3 -2
- etlplus/file/jinja2.py +3 -2
- etlplus/file/json.py +5 -16
- etlplus/file/log.py +3 -2
- etlplus/file/mat.py +3 -2
- etlplus/file/mdb.py +3 -2
- etlplus/file/msgpack.py +3 -2
- etlplus/file/mustache.py +3 -2
- etlplus/file/nc.py +3 -2
- etlplus/file/ndjson.py +3 -2
- etlplus/file/numbers.py +3 -2
- etlplus/file/ods.py +3 -2
- etlplus/file/orc.py +3 -2
- etlplus/file/parquet.py +3 -2
- etlplus/file/pb.py +3 -2
- etlplus/file/pbf.py +3 -2
- etlplus/file/properties.py +3 -2
- etlplus/file/proto.py +3 -2
- etlplus/file/psv.py +3 -2
- etlplus/file/rda.py +3 -2
- etlplus/file/rds.py +3 -2
- etlplus/file/sas7bdat.py +3 -2
- etlplus/file/sav.py +3 -2
- etlplus/file/sqlite.py +3 -2
- etlplus/file/stub.py +1 -0
- etlplus/file/sylk.py +3 -2
- etlplus/file/tab.py +3 -2
- etlplus/file/toml.py +3 -2
- etlplus/file/tsv.py +3 -2
- etlplus/file/txt.py +4 -3
- etlplus/file/vm.py +3 -2
- etlplus/file/wks.py +3 -2
- etlplus/file/xls.py +3 -2
- etlplus/file/xlsm.py +3 -2
- etlplus/file/xlsx.py +3 -2
- etlplus/file/xml.py +9 -3
- etlplus/file/xpt.py +3 -2
- etlplus/file/yaml.py +5 -16
- etlplus/file/zip.py +3 -2
- etlplus/file/zsav.py +3 -2
- etlplus/ops/extract.py +13 -1
- etlplus/ops/load.py +15 -2
- etlplus/ops/run.py +18 -13
- etlplus/ops/transform.py +2 -2
- etlplus/ops/utils.py +6 -35
- etlplus/ops/validate.py +3 -3
- etlplus/templates/README.md +2 -2
- etlplus/types.py +3 -2
- etlplus/utils.py +163 -29
- etlplus/{config → workflow}/README.md +6 -6
- etlplus/workflow/__init__.py +32 -0
- etlplus/{dag.py → workflow/dag.py} +6 -4
- etlplus/{config → workflow}/jobs.py +101 -38
- etlplus/{config → workflow}/pipeline.py +59 -51
- etlplus/{config → workflow}/profile.py +8 -5
- {etlplus-0.14.3.dist-info → etlplus-0.16.0.dist-info}/METADATA +4 -4
- etlplus-0.16.0.dist-info/RECORD +141 -0
- {etlplus-0.14.3.dist-info → etlplus-0.16.0.dist-info}/WHEEL +1 -1
- etlplus/config/__init__.py +0 -56
- etlplus/config/connector.py +0 -372
- etlplus/config/types.py +0 -204
- etlplus/config/utils.py +0 -120
- etlplus-0.14.3.dist-info/RECORD +0 -135
- {etlplus-0.14.3.dist-info → etlplus-0.16.0.dist-info}/entry_points.txt +0 -0
- {etlplus-0.14.3.dist-info → etlplus-0.16.0.dist-info}/licenses/LICENSE +0 -0
- {etlplus-0.14.3.dist-info → etlplus-0.16.0.dist-info}/top_level.txt +0 -0
etlplus/file/conf.py
CHANGED
|
@@ -30,6 +30,7 @@ from . import stub
|
|
|
30
30
|
|
|
31
31
|
|
|
32
32
|
__all__ = [
|
|
33
|
+
# Functions
|
|
33
34
|
'read',
|
|
34
35
|
'write',
|
|
35
36
|
]
|
|
@@ -42,7 +43,7 @@ def read(
|
|
|
42
43
|
path: Path,
|
|
43
44
|
) -> JSONList:
|
|
44
45
|
"""
|
|
45
|
-
Read CONF content from
|
|
46
|
+
Read CONF content from *path*.
|
|
46
47
|
|
|
47
48
|
Parameters
|
|
48
49
|
----------
|
|
@@ -62,7 +63,7 @@ def write(
|
|
|
62
63
|
data: JSONData,
|
|
63
64
|
) -> int:
|
|
64
65
|
"""
|
|
65
|
-
Write
|
|
66
|
+
Write *data* to CONF at *path* and return record count.
|
|
66
67
|
|
|
67
68
|
Parameters
|
|
68
69
|
----------
|
etlplus/file/core.py
CHANGED
|
@@ -22,7 +22,10 @@ from .enums import infer_file_format_and_compression
|
|
|
22
22
|
# SECTION: EXPORTS ========================================================== #
|
|
23
23
|
|
|
24
24
|
|
|
25
|
-
__all__ = [
|
|
25
|
+
__all__ = [
|
|
26
|
+
# Classes
|
|
27
|
+
'File',
|
|
28
|
+
]
|
|
26
29
|
|
|
27
30
|
|
|
28
31
|
# SECTION: INTERNAL FUNCTIONS =============================================== #
|
|
@@ -30,7 +33,7 @@ __all__ = ['File']
|
|
|
30
33
|
|
|
31
34
|
def _accepts_root_tag(handler: object) -> bool:
|
|
32
35
|
"""
|
|
33
|
-
Return True when
|
|
36
|
+
Return True when *handler* supports a ``root_tag`` argument.
|
|
34
37
|
|
|
35
38
|
Parameters
|
|
36
39
|
----------
|
|
@@ -57,7 +60,7 @@ def _accepts_root_tag(handler: object) -> bool:
|
|
|
57
60
|
@cache
|
|
58
61
|
def _module_for_format(file_format: FileFormat) -> ModuleType:
|
|
59
62
|
"""
|
|
60
|
-
Import and return the module for
|
|
63
|
+
Import and return the module for *file_format*.
|
|
61
64
|
|
|
62
65
|
Parameters
|
|
63
66
|
----------
|
|
@@ -112,8 +115,8 @@ class File:
|
|
|
112
115
|
"""
|
|
113
116
|
Auto-detect and set the file format on initialization.
|
|
114
117
|
|
|
115
|
-
If no explicit
|
|
116
|
-
the file path's extension and update :attr:`file_format`. If the
|
|
118
|
+
If no explicit :attr:`file_format` is provided, attempt to infer it
|
|
119
|
+
from the file path's extension and update :attr:`file_format`. If the
|
|
117
120
|
extension is unknown, the attribute is left as ``None`` and will be
|
|
118
121
|
validated later by :meth:`_ensure_format`.
|
|
119
122
|
"""
|
|
@@ -262,7 +265,7 @@ class File:
|
|
|
262
265
|
|
|
263
266
|
def read(self) -> JSONData:
|
|
264
267
|
"""
|
|
265
|
-
Read structured data from :attr
|
|
268
|
+
Read structured data from :attr:path` using :attr:`file_format`.
|
|
266
269
|
|
|
267
270
|
Returns
|
|
268
271
|
-------
|
|
@@ -291,7 +294,7 @@ class File:
|
|
|
291
294
|
root_tag: str = xml.DEFAULT_XML_ROOT,
|
|
292
295
|
) -> int:
|
|
293
296
|
"""
|
|
294
|
-
Write
|
|
297
|
+
Write *data* to *path* using :attr:`file_format`.
|
|
295
298
|
|
|
296
299
|
Parameters
|
|
297
300
|
----------
|
|
@@ -299,7 +302,7 @@ class File:
|
|
|
299
302
|
Data to write to the file.
|
|
300
303
|
root_tag : str, optional
|
|
301
304
|
Root tag name to use when writing XML files. Defaults to
|
|
302
|
-
``
|
|
305
|
+
``xml.DEFAULT_XML_ROOT``.
|
|
303
306
|
|
|
304
307
|
Returns
|
|
305
308
|
-------
|
etlplus/file/csv.py
CHANGED
|
@@ -29,6 +29,7 @@ from ._io import write_delimited
|
|
|
29
29
|
|
|
30
30
|
|
|
31
31
|
__all__ = [
|
|
32
|
+
# Functions
|
|
32
33
|
'read',
|
|
33
34
|
'write',
|
|
34
35
|
]
|
|
@@ -41,7 +42,7 @@ def read(
|
|
|
41
42
|
path: Path,
|
|
42
43
|
) -> JSONList:
|
|
43
44
|
"""
|
|
44
|
-
Read CSV content from
|
|
45
|
+
Read CSV content from *path*.
|
|
45
46
|
|
|
46
47
|
Parameters
|
|
47
48
|
----------
|
|
@@ -61,7 +62,7 @@ def write(
|
|
|
61
62
|
data: JSONData,
|
|
62
63
|
) -> int:
|
|
63
64
|
"""
|
|
64
|
-
Write
|
|
65
|
+
Write *data* to CSV at *path* and return record count.
|
|
65
66
|
|
|
66
67
|
Parameters
|
|
67
68
|
----------
|
etlplus/file/dat.py
CHANGED
|
@@ -28,6 +28,7 @@ from . import stub
|
|
|
28
28
|
|
|
29
29
|
|
|
30
30
|
__all__ = [
|
|
31
|
+
# Functions
|
|
31
32
|
'read',
|
|
32
33
|
'write',
|
|
33
34
|
]
|
|
@@ -40,7 +41,7 @@ def read(
|
|
|
40
41
|
path: Path,
|
|
41
42
|
) -> JSONList:
|
|
42
43
|
"""
|
|
43
|
-
Read DAT content from
|
|
44
|
+
Read DAT content from *path*.
|
|
44
45
|
|
|
45
46
|
Parameters
|
|
46
47
|
----------
|
|
@@ -60,7 +61,7 @@ def write(
|
|
|
60
61
|
data: JSONData,
|
|
61
62
|
) -> int:
|
|
62
63
|
"""
|
|
63
|
-
Write
|
|
64
|
+
Write *data* to DAT file at *path* and return record count.
|
|
64
65
|
|
|
65
66
|
Parameters
|
|
66
67
|
----------
|
etlplus/file/dta.py
CHANGED
|
@@ -27,6 +27,7 @@ from . import stub
|
|
|
27
27
|
|
|
28
28
|
|
|
29
29
|
__all__ = [
|
|
30
|
+
# Functions
|
|
30
31
|
'read',
|
|
31
32
|
'write',
|
|
32
33
|
]
|
|
@@ -39,7 +40,7 @@ def read(
|
|
|
39
40
|
path: Path,
|
|
40
41
|
) -> JSONList:
|
|
41
42
|
"""
|
|
42
|
-
Read DTA content from
|
|
43
|
+
Read DTA content from *path*.
|
|
43
44
|
|
|
44
45
|
Parameters
|
|
45
46
|
----------
|
|
@@ -59,7 +60,7 @@ def write(
|
|
|
59
60
|
data: JSONData,
|
|
60
61
|
) -> int:
|
|
61
62
|
"""
|
|
62
|
-
Write
|
|
63
|
+
Write *data* to DTA file at *path* and return record count.
|
|
63
64
|
|
|
64
65
|
Parameters
|
|
65
66
|
----------
|
etlplus/file/duckdb.py
CHANGED
|
@@ -28,6 +28,7 @@ from . import stub
|
|
|
28
28
|
|
|
29
29
|
|
|
30
30
|
__all__ = [
|
|
31
|
+
# Functions
|
|
31
32
|
'read',
|
|
32
33
|
'write',
|
|
33
34
|
]
|
|
@@ -40,7 +41,7 @@ def read(
|
|
|
40
41
|
path: Path,
|
|
41
42
|
) -> JSONList:
|
|
42
43
|
"""
|
|
43
|
-
Read DUCKDB content from
|
|
44
|
+
Read DUCKDB content from *path*.
|
|
44
45
|
|
|
45
46
|
Parameters
|
|
46
47
|
----------
|
|
@@ -60,7 +61,7 @@ def write(
|
|
|
60
61
|
data: JSONData,
|
|
61
62
|
) -> int:
|
|
62
63
|
"""
|
|
63
|
-
Write
|
|
64
|
+
Write *data* to DUCKDB at *path* and return record count.
|
|
64
65
|
|
|
65
66
|
Parameters
|
|
66
67
|
----------
|
etlplus/file/enums.py
CHANGED
|
@@ -281,7 +281,7 @@ def infer_file_format_and_compression(
|
|
|
281
281
|
A filename, extension, MIME type, or existing enum member.
|
|
282
282
|
filename : object | None, optional
|
|
283
283
|
A filename to consult for extension-based inference (e.g. when
|
|
284
|
-
|
|
284
|
+
*value* is ``application/octet-stream``).
|
|
285
285
|
|
|
286
286
|
Returns
|
|
287
287
|
-------
|
etlplus/file/feather.py
CHANGED
|
@@ -30,6 +30,7 @@ from ._io import normalize_records
|
|
|
30
30
|
|
|
31
31
|
|
|
32
32
|
__all__ = [
|
|
33
|
+
# Functions
|
|
33
34
|
'read',
|
|
34
35
|
'write',
|
|
35
36
|
]
|
|
@@ -42,7 +43,7 @@ def read(
|
|
|
42
43
|
path: Path,
|
|
43
44
|
) -> JSONList:
|
|
44
45
|
"""
|
|
45
|
-
Read Feather content from
|
|
46
|
+
Read Feather content from *path*.
|
|
46
47
|
|
|
47
48
|
Parameters
|
|
48
49
|
----------
|
|
@@ -75,7 +76,7 @@ def write(
|
|
|
75
76
|
data: JSONData,
|
|
76
77
|
) -> int:
|
|
77
78
|
"""
|
|
78
|
-
Write
|
|
79
|
+
Write *data* to Feather at *path* and return record count.
|
|
79
80
|
|
|
80
81
|
Parameters
|
|
81
82
|
----------
|
etlplus/file/fwf.py
CHANGED
|
@@ -27,6 +27,7 @@ from . import stub
|
|
|
27
27
|
|
|
28
28
|
|
|
29
29
|
__all__ = [
|
|
30
|
+
# Functions
|
|
30
31
|
'read',
|
|
31
32
|
'write',
|
|
32
33
|
]
|
|
@@ -39,7 +40,7 @@ def read(
|
|
|
39
40
|
path: Path,
|
|
40
41
|
) -> JSONList:
|
|
41
42
|
"""
|
|
42
|
-
Read FWF content from
|
|
43
|
+
Read FWF content from *path*.
|
|
43
44
|
|
|
44
45
|
Parameters
|
|
45
46
|
----------
|
|
@@ -59,7 +60,7 @@ def write(
|
|
|
59
60
|
data: JSONData,
|
|
60
61
|
) -> int:
|
|
61
62
|
"""
|
|
62
|
-
Write
|
|
63
|
+
Write *data* to FWF file at *path* and return record count.
|
|
63
64
|
|
|
64
65
|
Parameters
|
|
65
66
|
----------
|
etlplus/file/gz.py
CHANGED
|
@@ -19,6 +19,7 @@ from .enums import infer_file_format_and_compression
|
|
|
19
19
|
|
|
20
20
|
|
|
21
21
|
__all__ = [
|
|
22
|
+
# Functions
|
|
22
23
|
'read',
|
|
23
24
|
'write',
|
|
24
25
|
]
|
|
@@ -65,7 +66,7 @@ def read(
|
|
|
65
66
|
path: Path,
|
|
66
67
|
) -> JSONData:
|
|
67
68
|
"""
|
|
68
|
-
Read GZ content from
|
|
69
|
+
Read GZ content from *path* and parse the inner payload.
|
|
69
70
|
|
|
70
71
|
Parameters
|
|
71
72
|
----------
|
|
@@ -94,7 +95,7 @@ def write(
|
|
|
94
95
|
data: JSONData,
|
|
95
96
|
) -> int:
|
|
96
97
|
"""
|
|
97
|
-
Write
|
|
98
|
+
Write *data* to GZ at *path* and return record count.
|
|
98
99
|
|
|
99
100
|
Parameters
|
|
100
101
|
----------
|
etlplus/file/hbs.py
CHANGED
|
@@ -28,6 +28,7 @@ from . import stub
|
|
|
28
28
|
|
|
29
29
|
|
|
30
30
|
__all__ = [
|
|
31
|
+
# Functions
|
|
31
32
|
'read',
|
|
32
33
|
'write',
|
|
33
34
|
]
|
|
@@ -40,7 +41,7 @@ def read(
|
|
|
40
41
|
path: Path,
|
|
41
42
|
) -> JSONList:
|
|
42
43
|
"""
|
|
43
|
-
Read ZSAV content from
|
|
44
|
+
Read ZSAV content from *path*.
|
|
44
45
|
|
|
45
46
|
Parameters
|
|
46
47
|
----------
|
|
@@ -60,7 +61,7 @@ def write(
|
|
|
60
61
|
data: JSONData,
|
|
61
62
|
) -> int:
|
|
62
63
|
"""
|
|
63
|
-
Write
|
|
64
|
+
Write *data* to HBS file at *path* and return record count.
|
|
64
65
|
|
|
65
66
|
Parameters
|
|
66
67
|
----------
|
etlplus/file/hdf5.py
CHANGED
|
@@ -28,6 +28,7 @@ from . import stub
|
|
|
28
28
|
|
|
29
29
|
|
|
30
30
|
__all__ = [
|
|
31
|
+
# Functions
|
|
31
32
|
'read',
|
|
32
33
|
'write',
|
|
33
34
|
]
|
|
@@ -40,7 +41,7 @@ def read(
|
|
|
40
41
|
path: Path,
|
|
41
42
|
) -> JSONList:
|
|
42
43
|
"""
|
|
43
|
-
Read HDF5 content from
|
|
44
|
+
Read HDF5 content from *path*.
|
|
44
45
|
|
|
45
46
|
Parameters
|
|
46
47
|
----------
|
|
@@ -60,7 +61,7 @@ def write(
|
|
|
60
61
|
data: JSONData,
|
|
61
62
|
) -> int:
|
|
62
63
|
"""
|
|
63
|
-
Write
|
|
64
|
+
Write *data* to HDF5 file at *path* and return record count.
|
|
64
65
|
|
|
65
66
|
Parameters
|
|
66
67
|
----------
|
etlplus/file/ini.py
CHANGED
|
@@ -29,6 +29,7 @@ from . import stub
|
|
|
29
29
|
|
|
30
30
|
|
|
31
31
|
__all__ = [
|
|
32
|
+
# Functions
|
|
32
33
|
'read',
|
|
33
34
|
'write',
|
|
34
35
|
]
|
|
@@ -41,7 +42,7 @@ def read(
|
|
|
41
42
|
path: Path,
|
|
42
43
|
) -> JSONList:
|
|
43
44
|
"""
|
|
44
|
-
Read INI content from
|
|
45
|
+
Read INI content from *path*.
|
|
45
46
|
|
|
46
47
|
Parameters
|
|
47
48
|
----------
|
|
@@ -61,7 +62,7 @@ def write(
|
|
|
61
62
|
data: JSONData,
|
|
62
63
|
) -> int:
|
|
63
64
|
"""
|
|
64
|
-
Write
|
|
65
|
+
Write *data* to INI at *path* and return record count.
|
|
65
66
|
|
|
66
67
|
Parameters
|
|
67
68
|
----------
|
etlplus/file/ion.py
CHANGED
|
@@ -28,6 +28,7 @@ from . import stub
|
|
|
28
28
|
|
|
29
29
|
|
|
30
30
|
__all__ = [
|
|
31
|
+
# Functions
|
|
31
32
|
'read',
|
|
32
33
|
'write',
|
|
33
34
|
]
|
|
@@ -40,7 +41,7 @@ def read(
|
|
|
40
41
|
path: Path,
|
|
41
42
|
) -> JSONList:
|
|
42
43
|
"""
|
|
43
|
-
Read ION content from
|
|
44
|
+
Read ION content from *path*.
|
|
44
45
|
|
|
45
46
|
Parameters
|
|
46
47
|
----------
|
|
@@ -60,7 +61,7 @@ def write(
|
|
|
60
61
|
data: JSONData,
|
|
61
62
|
) -> int:
|
|
62
63
|
"""
|
|
63
|
-
Write
|
|
64
|
+
Write *data* to ION at *path* and return record count.
|
|
64
65
|
|
|
65
66
|
Parameters
|
|
66
67
|
----------
|
etlplus/file/jinja2.py
CHANGED
|
@@ -28,6 +28,7 @@ from . import stub
|
|
|
28
28
|
|
|
29
29
|
|
|
30
30
|
__all__ = [
|
|
31
|
+
# Functions
|
|
31
32
|
'read',
|
|
32
33
|
'write',
|
|
33
34
|
]
|
|
@@ -40,7 +41,7 @@ def read(
|
|
|
40
41
|
path: Path,
|
|
41
42
|
) -> JSONList:
|
|
42
43
|
"""
|
|
43
|
-
Read JINJA2 content from
|
|
44
|
+
Read JINJA2 content from *path*.
|
|
44
45
|
|
|
45
46
|
Parameters
|
|
46
47
|
----------
|
|
@@ -60,7 +61,7 @@ def write(
|
|
|
60
61
|
data: JSONData,
|
|
61
62
|
) -> int:
|
|
62
63
|
"""
|
|
63
|
-
Write
|
|
64
|
+
Write *data* to JINJA2 file at *path* and return record count.
|
|
64
65
|
|
|
65
66
|
Parameters
|
|
66
67
|
----------
|
etlplus/file/json.py
CHANGED
|
@@ -20,17 +20,16 @@ from __future__ import annotations
|
|
|
20
20
|
|
|
21
21
|
import json
|
|
22
22
|
from pathlib import Path
|
|
23
|
-
from typing import cast
|
|
24
23
|
|
|
25
24
|
from ..types import JSONData
|
|
26
|
-
from ..types import JSONDict
|
|
27
|
-
from ..types import JSONList
|
|
28
25
|
from ..utils import count_records
|
|
26
|
+
from ._io import coerce_record_payload
|
|
29
27
|
|
|
30
28
|
# SECTION: EXPORTS ========================================================== #
|
|
31
29
|
|
|
32
30
|
|
|
33
31
|
__all__ = [
|
|
32
|
+
# Functions
|
|
34
33
|
'read',
|
|
35
34
|
'write',
|
|
36
35
|
]
|
|
@@ -43,7 +42,7 @@ def read(
|
|
|
43
42
|
path: Path,
|
|
44
43
|
) -> JSONData:
|
|
45
44
|
"""
|
|
46
|
-
Read JSON content from
|
|
45
|
+
Read JSON content from *path*.
|
|
47
46
|
|
|
48
47
|
Validates that the JSON root is a dict or a list of dicts.
|
|
49
48
|
|
|
@@ -65,17 +64,7 @@ def read(
|
|
|
65
64
|
with path.open('r', encoding='utf-8') as handle:
|
|
66
65
|
loaded = json.load(handle)
|
|
67
66
|
|
|
68
|
-
|
|
69
|
-
return cast(JSONDict, loaded)
|
|
70
|
-
if isinstance(loaded, list):
|
|
71
|
-
if all(isinstance(item, dict) for item in loaded):
|
|
72
|
-
return cast(JSONList, loaded)
|
|
73
|
-
raise TypeError(
|
|
74
|
-
'JSON array must contain only objects (dicts) when loading file',
|
|
75
|
-
)
|
|
76
|
-
raise TypeError(
|
|
77
|
-
'JSON root must be an object or an array of objects when loading file',
|
|
78
|
-
)
|
|
67
|
+
return coerce_record_payload(loaded, format_name='JSON')
|
|
79
68
|
|
|
80
69
|
|
|
81
70
|
def write(
|
|
@@ -83,7 +72,7 @@ def write(
|
|
|
83
72
|
data: JSONData,
|
|
84
73
|
) -> int:
|
|
85
74
|
"""
|
|
86
|
-
Write
|
|
75
|
+
Write *data* as formatted JSON to *path*.
|
|
87
76
|
|
|
88
77
|
Parameters
|
|
89
78
|
----------
|
etlplus/file/log.py
CHANGED
|
@@ -28,6 +28,7 @@ from . import stub
|
|
|
28
28
|
|
|
29
29
|
|
|
30
30
|
__all__ = [
|
|
31
|
+
# Functions
|
|
31
32
|
'read',
|
|
32
33
|
'write',
|
|
33
34
|
]
|
|
@@ -40,7 +41,7 @@ def read(
|
|
|
40
41
|
path: Path,
|
|
41
42
|
) -> JSONList:
|
|
42
43
|
"""
|
|
43
|
-
Read LOG content from
|
|
44
|
+
Read LOG content from *path*.
|
|
44
45
|
|
|
45
46
|
Parameters
|
|
46
47
|
----------
|
|
@@ -60,7 +61,7 @@ def write(
|
|
|
60
61
|
data: JSONData,
|
|
61
62
|
) -> int:
|
|
62
63
|
"""
|
|
63
|
-
Write
|
|
64
|
+
Write *data* to LOG at *path* and return record count.
|
|
64
65
|
|
|
65
66
|
Parameters
|
|
66
67
|
----------
|
etlplus/file/mat.py
CHANGED
|
@@ -28,6 +28,7 @@ from . import stub
|
|
|
28
28
|
|
|
29
29
|
|
|
30
30
|
__all__ = [
|
|
31
|
+
# Functions
|
|
31
32
|
'read',
|
|
32
33
|
'write',
|
|
33
34
|
]
|
|
@@ -40,7 +41,7 @@ def read(
|
|
|
40
41
|
path: Path,
|
|
41
42
|
) -> JSONList:
|
|
42
43
|
"""
|
|
43
|
-
Read MAT content from
|
|
44
|
+
Read MAT content from *path*.
|
|
44
45
|
|
|
45
46
|
Parameters
|
|
46
47
|
----------
|
|
@@ -60,7 +61,7 @@ def write(
|
|
|
60
61
|
data: JSONData,
|
|
61
62
|
) -> int:
|
|
62
63
|
"""
|
|
63
|
-
Write
|
|
64
|
+
Write *data* to MAT file at *path* and return record count.
|
|
64
65
|
|
|
65
66
|
Parameters
|
|
66
67
|
----------
|
etlplus/file/mdb.py
CHANGED
|
@@ -28,6 +28,7 @@ from . import stub
|
|
|
28
28
|
|
|
29
29
|
|
|
30
30
|
__all__ = [
|
|
31
|
+
# Functions
|
|
31
32
|
'read',
|
|
32
33
|
'write',
|
|
33
34
|
]
|
|
@@ -40,7 +41,7 @@ def read(
|
|
|
40
41
|
path: Path,
|
|
41
42
|
) -> JSONList:
|
|
42
43
|
"""
|
|
43
|
-
Read CSV content from
|
|
44
|
+
Read CSV content from *path*.
|
|
44
45
|
|
|
45
46
|
Parameters
|
|
46
47
|
----------
|
|
@@ -60,7 +61,7 @@ def write(
|
|
|
60
61
|
data: JSONData,
|
|
61
62
|
) -> int:
|
|
62
63
|
"""
|
|
63
|
-
Write
|
|
64
|
+
Write *data* to CSV at *path* and return record count.
|
|
64
65
|
|
|
65
66
|
Parameters
|
|
66
67
|
----------
|
etlplus/file/msgpack.py
CHANGED
|
@@ -28,6 +28,7 @@ from . import stub
|
|
|
28
28
|
|
|
29
29
|
|
|
30
30
|
__all__ = [
|
|
31
|
+
# Functions
|
|
31
32
|
'read',
|
|
32
33
|
'write',
|
|
33
34
|
]
|
|
@@ -40,7 +41,7 @@ def read(
|
|
|
40
41
|
path: Path,
|
|
41
42
|
) -> JSONList:
|
|
42
43
|
"""
|
|
43
|
-
Read MsgPack content from
|
|
44
|
+
Read MsgPack content from *path*.
|
|
44
45
|
|
|
45
46
|
Parameters
|
|
46
47
|
----------
|
|
@@ -60,7 +61,7 @@ def write(
|
|
|
60
61
|
data: JSONData,
|
|
61
62
|
) -> int:
|
|
62
63
|
"""
|
|
63
|
-
Write
|
|
64
|
+
Write *data* to MsgPack at *path* and return record count.
|
|
64
65
|
|
|
65
66
|
Parameters
|
|
66
67
|
----------
|
etlplus/file/mustache.py
CHANGED
|
@@ -28,6 +28,7 @@ from . import stub
|
|
|
28
28
|
|
|
29
29
|
|
|
30
30
|
__all__ = [
|
|
31
|
+
# Functions
|
|
31
32
|
'read',
|
|
32
33
|
'write',
|
|
33
34
|
]
|
|
@@ -40,7 +41,7 @@ def read(
|
|
|
40
41
|
path: Path,
|
|
41
42
|
) -> JSONList:
|
|
42
43
|
"""
|
|
43
|
-
Read MUSTACHE content from
|
|
44
|
+
Read MUSTACHE content from *path*.
|
|
44
45
|
|
|
45
46
|
Parameters
|
|
46
47
|
----------
|
|
@@ -60,7 +61,7 @@ def write(
|
|
|
60
61
|
data: JSONData,
|
|
61
62
|
) -> int:
|
|
62
63
|
"""
|
|
63
|
-
Write
|
|
64
|
+
Write *data* to MUSTACHE file at *path* and return record count.
|
|
64
65
|
|
|
65
66
|
Parameters
|
|
66
67
|
----------
|
etlplus/file/nc.py
CHANGED
|
@@ -28,6 +28,7 @@ from . import stub
|
|
|
28
28
|
|
|
29
29
|
|
|
30
30
|
__all__ = [
|
|
31
|
+
# Functions
|
|
31
32
|
'read',
|
|
32
33
|
'write',
|
|
33
34
|
]
|
|
@@ -40,7 +41,7 @@ def read(
|
|
|
40
41
|
path: Path,
|
|
41
42
|
) -> JSONList:
|
|
42
43
|
"""
|
|
43
|
-
Read NC content from
|
|
44
|
+
Read NC content from *path*.
|
|
44
45
|
|
|
45
46
|
Parameters
|
|
46
47
|
----------
|
|
@@ -60,7 +61,7 @@ def write(
|
|
|
60
61
|
data: JSONData,
|
|
61
62
|
) -> int:
|
|
62
63
|
"""
|
|
63
|
-
Write
|
|
64
|
+
Write *data* to NC file at *path* and return record count.
|
|
64
65
|
|
|
65
66
|
Parameters
|
|
66
67
|
----------
|
etlplus/file/ndjson.py
CHANGED
|
@@ -31,6 +31,7 @@ from ._io import normalize_records
|
|
|
31
31
|
|
|
32
32
|
|
|
33
33
|
__all__ = [
|
|
34
|
+
# Functions
|
|
34
35
|
'read',
|
|
35
36
|
'write',
|
|
36
37
|
]
|
|
@@ -43,7 +44,7 @@ def read(
|
|
|
43
44
|
path: Path,
|
|
44
45
|
) -> JSONList:
|
|
45
46
|
"""
|
|
46
|
-
Read NDJSON content from
|
|
47
|
+
Read NDJSON content from *path*.
|
|
47
48
|
|
|
48
49
|
Parameters
|
|
49
50
|
----------
|
|
@@ -80,7 +81,7 @@ def write(
|
|
|
80
81
|
data: JSONData,
|
|
81
82
|
) -> int:
|
|
82
83
|
"""
|
|
83
|
-
Write
|
|
84
|
+
Write *data* to NDJSON at *path*.
|
|
84
85
|
|
|
85
86
|
Parameters
|
|
86
87
|
----------
|
etlplus/file/numbers.py
CHANGED
|
@@ -25,6 +25,7 @@ from . import stub
|
|
|
25
25
|
|
|
26
26
|
|
|
27
27
|
__all__ = [
|
|
28
|
+
# Functions
|
|
28
29
|
'read',
|
|
29
30
|
'write',
|
|
30
31
|
]
|
|
@@ -37,7 +38,7 @@ def read(
|
|
|
37
38
|
path: Path,
|
|
38
39
|
) -> JSONList:
|
|
39
40
|
"""
|
|
40
|
-
Read NUMBERS content from
|
|
41
|
+
Read NUMBERS content from *path*.
|
|
41
42
|
|
|
42
43
|
Parameters
|
|
43
44
|
----------
|
|
@@ -57,7 +58,7 @@ def write(
|
|
|
57
58
|
data: JSONData,
|
|
58
59
|
) -> int:
|
|
59
60
|
"""
|
|
60
|
-
Write
|
|
61
|
+
Write *data* to NUMBERS file at *path* and return record count.
|
|
61
62
|
|
|
62
63
|
Parameters
|
|
63
64
|
----------
|