etlplus 0.9.2__py3-none-any.whl → 0.10.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- etlplus/__init__.py +26 -1
- etlplus/api/README.md +3 -51
- etlplus/api/__init__.py +0 -10
- etlplus/api/config.py +28 -39
- etlplus/api/endpoint_client.py +3 -3
- etlplus/api/pagination/client.py +1 -1
- etlplus/api/rate_limiting/config.py +1 -13
- etlplus/api/rate_limiting/rate_limiter.py +11 -8
- etlplus/api/request_manager.py +6 -11
- etlplus/api/transport.py +2 -14
- etlplus/api/types.py +6 -96
- etlplus/cli/commands.py +43 -76
- etlplus/cli/constants.py +1 -1
- etlplus/cli/handlers.py +12 -40
- etlplus/cli/io.py +2 -2
- etlplus/cli/main.py +1 -1
- etlplus/cli/state.py +7 -4
- etlplus/{workflow → config}/__init__.py +23 -10
- etlplus/{workflow → config}/connector.py +44 -58
- etlplus/{workflow → config}/jobs.py +32 -105
- etlplus/{workflow → config}/pipeline.py +51 -59
- etlplus/{workflow → config}/profile.py +5 -8
- etlplus/config/types.py +204 -0
- etlplus/config/utils.py +120 -0
- etlplus/database/ddl.py +1 -1
- etlplus/database/engine.py +3 -19
- etlplus/database/orm.py +0 -2
- etlplus/database/schema.py +1 -1
- etlplus/enums.py +288 -0
- etlplus/{ops/extract.py → extract.py} +99 -81
- etlplus/file.py +652 -0
- etlplus/{ops/load.py → load.py} +101 -78
- etlplus/{ops/run.py → run.py} +127 -159
- etlplus/{api/utils.py → run_helpers.py} +153 -209
- etlplus/{ops/transform.py → transform.py} +68 -75
- etlplus/types.py +4 -5
- etlplus/utils.py +2 -136
- etlplus/{ops/validate.py → validate.py} +12 -22
- etlplus/validation/__init__.py +44 -0
- etlplus/{ops → validation}/utils.py +17 -53
- {etlplus-0.9.2.dist-info → etlplus-0.10.2.dist-info}/METADATA +17 -210
- etlplus-0.10.2.dist-info/RECORD +65 -0
- {etlplus-0.9.2.dist-info → etlplus-0.10.2.dist-info}/WHEEL +1 -1
- etlplus/README.md +0 -37
- etlplus/api/enums.py +0 -51
- etlplus/cli/README.md +0 -40
- etlplus/database/README.md +0 -48
- etlplus/file/README.md +0 -105
- etlplus/file/__init__.py +0 -25
- etlplus/file/_imports.py +0 -141
- etlplus/file/_io.py +0 -160
- etlplus/file/accdb.py +0 -78
- etlplus/file/arrow.py +0 -78
- etlplus/file/avro.py +0 -176
- etlplus/file/bson.py +0 -77
- etlplus/file/cbor.py +0 -78
- etlplus/file/cfg.py +0 -79
- etlplus/file/conf.py +0 -80
- etlplus/file/core.py +0 -322
- etlplus/file/csv.py +0 -79
- etlplus/file/dat.py +0 -78
- etlplus/file/dta.py +0 -77
- etlplus/file/duckdb.py +0 -78
- etlplus/file/enums.py +0 -343
- etlplus/file/feather.py +0 -111
- etlplus/file/fwf.py +0 -77
- etlplus/file/gz.py +0 -123
- etlplus/file/hbs.py +0 -78
- etlplus/file/hdf5.py +0 -78
- etlplus/file/ini.py +0 -79
- etlplus/file/ion.py +0 -78
- etlplus/file/jinja2.py +0 -78
- etlplus/file/json.py +0 -98
- etlplus/file/log.py +0 -78
- etlplus/file/mat.py +0 -78
- etlplus/file/mdb.py +0 -78
- etlplus/file/msgpack.py +0 -78
- etlplus/file/mustache.py +0 -78
- etlplus/file/nc.py +0 -78
- etlplus/file/ndjson.py +0 -108
- etlplus/file/numbers.py +0 -75
- etlplus/file/ods.py +0 -79
- etlplus/file/orc.py +0 -111
- etlplus/file/parquet.py +0 -113
- etlplus/file/pb.py +0 -78
- etlplus/file/pbf.py +0 -77
- etlplus/file/properties.py +0 -78
- etlplus/file/proto.py +0 -77
- etlplus/file/psv.py +0 -79
- etlplus/file/rda.py +0 -78
- etlplus/file/rds.py +0 -78
- etlplus/file/sas7bdat.py +0 -78
- etlplus/file/sav.py +0 -77
- etlplus/file/sqlite.py +0 -78
- etlplus/file/stub.py +0 -84
- etlplus/file/sylk.py +0 -77
- etlplus/file/tab.py +0 -81
- etlplus/file/toml.py +0 -78
- etlplus/file/tsv.py +0 -80
- etlplus/file/txt.py +0 -102
- etlplus/file/vm.py +0 -78
- etlplus/file/wks.py +0 -77
- etlplus/file/xls.py +0 -88
- etlplus/file/xlsm.py +0 -79
- etlplus/file/xlsx.py +0 -99
- etlplus/file/xml.py +0 -185
- etlplus/file/xpt.py +0 -78
- etlplus/file/yaml.py +0 -95
- etlplus/file/zip.py +0 -175
- etlplus/file/zsav.py +0 -77
- etlplus/ops/README.md +0 -50
- etlplus/ops/__init__.py +0 -61
- etlplus/templates/README.md +0 -46
- etlplus/workflow/README.md +0 -52
- etlplus/workflow/dag.py +0 -105
- etlplus/workflow/types.py +0 -115
- etlplus-0.9.2.dist-info/RECORD +0 -134
- {etlplus-0.9.2.dist-info → etlplus-0.10.2.dist-info}/entry_points.txt +0 -0
- {etlplus-0.9.2.dist-info → etlplus-0.10.2.dist-info}/licenses/LICENSE +0 -0
- {etlplus-0.9.2.dist-info → etlplus-0.10.2.dist-info}/top_level.txt +0 -0
etlplus/file/log.py
DELETED
|
@@ -1,78 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
:mod:`etlplus.file.log` module.
|
|
3
|
-
|
|
4
|
-
Helpers for reading/writing generic log (LOG) files.
|
|
5
|
-
|
|
6
|
-
Notes
|
|
7
|
-
-----
|
|
8
|
-
- A LOG file is a plain text file that contains log messages generated by
|
|
9
|
-
applications or systems.
|
|
10
|
-
- Common cases:
|
|
11
|
-
- Each line in the file represents a single log entry.
|
|
12
|
-
- Log entries may include timestamps, log levels, and messages.
|
|
13
|
-
- Formats may vary widely depending on the application generating the logs.
|
|
14
|
-
- Rule of thumb:
|
|
15
|
-
- If the file is a generic log file, use this module for reading and
|
|
16
|
-
writing.
|
|
17
|
-
"""
|
|
18
|
-
|
|
19
|
-
from __future__ import annotations
|
|
20
|
-
|
|
21
|
-
from pathlib import Path
|
|
22
|
-
|
|
23
|
-
from ..types import JSONData
|
|
24
|
-
from ..types import JSONList
|
|
25
|
-
from . import stub
|
|
26
|
-
|
|
27
|
-
# SECTION: EXPORTS ========================================================== #
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
__all__ = [
|
|
31
|
-
'read',
|
|
32
|
-
'write',
|
|
33
|
-
]
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
# SECTION: FUNCTIONS ======================================================== #
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
def read(
|
|
40
|
-
path: Path,
|
|
41
|
-
) -> JSONList:
|
|
42
|
-
"""
|
|
43
|
-
Read LOG content from ``path``.
|
|
44
|
-
|
|
45
|
-
Parameters
|
|
46
|
-
----------
|
|
47
|
-
path : Path
|
|
48
|
-
Path to the LOG file on disk.
|
|
49
|
-
|
|
50
|
-
Returns
|
|
51
|
-
-------
|
|
52
|
-
JSONList
|
|
53
|
-
The list of dictionaries read from the LOG file.
|
|
54
|
-
"""
|
|
55
|
-
return stub.read(path, format_name='LOG')
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
def write(
|
|
59
|
-
path: Path,
|
|
60
|
-
data: JSONData,
|
|
61
|
-
) -> int:
|
|
62
|
-
"""
|
|
63
|
-
Write ``data`` to LOG at ``path`` and return record count.
|
|
64
|
-
|
|
65
|
-
Parameters
|
|
66
|
-
----------
|
|
67
|
-
path : Path
|
|
68
|
-
Path to the LOG file on disk.
|
|
69
|
-
data : JSONData
|
|
70
|
-
Data to write as LOG. Should be a list of dictionaries or a
|
|
71
|
-
single dictionary.
|
|
72
|
-
|
|
73
|
-
Returns
|
|
74
|
-
-------
|
|
75
|
-
int
|
|
76
|
-
The number of rows written to the LOG file.
|
|
77
|
-
"""
|
|
78
|
-
return stub.write(path, data, format_name='LOG')
|
etlplus/file/mat.py
DELETED
|
@@ -1,78 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
:mod:`etlplus.file.mat` module.
|
|
3
|
-
|
|
4
|
-
Helpers for reading/writing MATLAB (MAT) data files.
|
|
5
|
-
|
|
6
|
-
Notes
|
|
7
|
-
-----
|
|
8
|
-
- A MAT file is a binary file format used by MATLAB to store variables,
|
|
9
|
-
arrays, and other data structures.
|
|
10
|
-
- Common cases:
|
|
11
|
-
- Storing numerical arrays and matrices.
|
|
12
|
-
- Saving workspace variables.
|
|
13
|
-
- Sharing data between MATLAB and other programming environments.
|
|
14
|
-
- Rule of thumb:
|
|
15
|
-
- If the file follows the MAT-file specification, use this module for
|
|
16
|
-
reading and writing.
|
|
17
|
-
"""
|
|
18
|
-
|
|
19
|
-
from __future__ import annotations
|
|
20
|
-
|
|
21
|
-
from pathlib import Path
|
|
22
|
-
|
|
23
|
-
from ..types import JSONData
|
|
24
|
-
from ..types import JSONList
|
|
25
|
-
from . import stub
|
|
26
|
-
|
|
27
|
-
# SECTION: EXPORTS ========================================================== #
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
__all__ = [
|
|
31
|
-
'read',
|
|
32
|
-
'write',
|
|
33
|
-
]
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
# SECTION: FUNCTIONS ======================================================== #
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
def read(
|
|
40
|
-
path: Path,
|
|
41
|
-
) -> JSONList:
|
|
42
|
-
"""
|
|
43
|
-
Read MAT content from ``path``.
|
|
44
|
-
|
|
45
|
-
Parameters
|
|
46
|
-
----------
|
|
47
|
-
path : Path
|
|
48
|
-
Path to the MAT file on disk.
|
|
49
|
-
|
|
50
|
-
Returns
|
|
51
|
-
-------
|
|
52
|
-
JSONList
|
|
53
|
-
The list of dictionaries read from the MAT file.
|
|
54
|
-
"""
|
|
55
|
-
return stub.read(path, format_name='MAT')
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
def write(
|
|
59
|
-
path: Path,
|
|
60
|
-
data: JSONData,
|
|
61
|
-
) -> int:
|
|
62
|
-
"""
|
|
63
|
-
Write ``data`` to MAT file at ``path`` and return record count.
|
|
64
|
-
|
|
65
|
-
Parameters
|
|
66
|
-
----------
|
|
67
|
-
path : Path
|
|
68
|
-
Path to the MAT file on disk.
|
|
69
|
-
data : JSONData
|
|
70
|
-
Data to write as MAT file. Should be a list of dictionaries or a
|
|
71
|
-
single dictionary.
|
|
72
|
-
|
|
73
|
-
Returns
|
|
74
|
-
-------
|
|
75
|
-
int
|
|
76
|
-
The number of rows written to the MAT file.
|
|
77
|
-
"""
|
|
78
|
-
return stub.write(path, data, format_name='MAT')
|
etlplus/file/mdb.py
DELETED
|
@@ -1,78 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
:mod:`etlplus.file.mdb` module.
|
|
3
|
-
|
|
4
|
-
Helpers for reading/writing newer Microsoft Access database (MDB) files.
|
|
5
|
-
|
|
6
|
-
Notes
|
|
7
|
-
-----
|
|
8
|
-
- An MDB file is a proprietary database file format used by Microsoft Access
|
|
9
|
-
2003 and earlier.
|
|
10
|
-
- Common cases:
|
|
11
|
-
- Storing relational data for small to medium-sized applications.
|
|
12
|
-
- Desktop database applications.
|
|
13
|
-
- Data management for non-enterprise solutions.
|
|
14
|
-
- Rule of thumb:
|
|
15
|
-
- If the file follows the MDB specification, use this module for reading
|
|
16
|
-
and writing.
|
|
17
|
-
"""
|
|
18
|
-
|
|
19
|
-
from __future__ import annotations
|
|
20
|
-
|
|
21
|
-
from pathlib import Path
|
|
22
|
-
|
|
23
|
-
from ..types import JSONData
|
|
24
|
-
from ..types import JSONList
|
|
25
|
-
from . import stub
|
|
26
|
-
|
|
27
|
-
# SECTION: EXPORTS ========================================================== #
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
__all__ = [
|
|
31
|
-
'read',
|
|
32
|
-
'write',
|
|
33
|
-
]
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
# SECTION: FUNCTIONS ======================================================== #
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
def read(
|
|
40
|
-
path: Path,
|
|
41
|
-
) -> JSONList:
|
|
42
|
-
"""
|
|
43
|
-
Read CSV content from ``path``.
|
|
44
|
-
|
|
45
|
-
Parameters
|
|
46
|
-
----------
|
|
47
|
-
path : Path
|
|
48
|
-
Path to the CSV file on disk.
|
|
49
|
-
|
|
50
|
-
Returns
|
|
51
|
-
-------
|
|
52
|
-
JSONList
|
|
53
|
-
The list of dictionaries read from the CSV file.
|
|
54
|
-
"""
|
|
55
|
-
return stub.read(path, format_name='DAT')
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
def write(
|
|
59
|
-
path: Path,
|
|
60
|
-
data: JSONData,
|
|
61
|
-
) -> int:
|
|
62
|
-
"""
|
|
63
|
-
Write ``data`` to CSV at ``path`` and return record count.
|
|
64
|
-
|
|
65
|
-
Parameters
|
|
66
|
-
----------
|
|
67
|
-
path : Path
|
|
68
|
-
Path to the CSV file on disk.
|
|
69
|
-
data : JSONData
|
|
70
|
-
Data to write as CSV. Should be a list of dictionaries or a
|
|
71
|
-
single dictionary.
|
|
72
|
-
|
|
73
|
-
Returns
|
|
74
|
-
-------
|
|
75
|
-
int
|
|
76
|
-
The number of rows written to the CSV file.
|
|
77
|
-
"""
|
|
78
|
-
return stub.write(path, data, format_name='DAT')
|
etlplus/file/msgpack.py
DELETED
|
@@ -1,78 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
:mod:`etlplus.file.msgpack` module.
|
|
3
|
-
|
|
4
|
-
Helpers for reading/writing MessagePack (MSGPACK) files.
|
|
5
|
-
|
|
6
|
-
Notes
|
|
7
|
-
-----
|
|
8
|
-
- A MsgPack file is a binary serialization format that is more compact than
|
|
9
|
-
JSON.
|
|
10
|
-
- Common cases:
|
|
11
|
-
- Efficient data storage and transmission.
|
|
12
|
-
- Inter-process communication.
|
|
13
|
-
- Data serialization in performance-critical applications.
|
|
14
|
-
- Rule of thumb:
|
|
15
|
-
- If the file follows the MsgPack specification, use this module for
|
|
16
|
-
reading and writing.
|
|
17
|
-
"""
|
|
18
|
-
|
|
19
|
-
from __future__ import annotations
|
|
20
|
-
|
|
21
|
-
from pathlib import Path
|
|
22
|
-
|
|
23
|
-
from ..types import JSONData
|
|
24
|
-
from ..types import JSONList
|
|
25
|
-
from . import stub
|
|
26
|
-
|
|
27
|
-
# SECTION: EXPORTS ========================================================== #
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
__all__ = [
|
|
31
|
-
'read',
|
|
32
|
-
'write',
|
|
33
|
-
]
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
# SECTION: FUNCTIONS ======================================================== #
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
def read(
|
|
40
|
-
path: Path,
|
|
41
|
-
) -> JSONList:
|
|
42
|
-
"""
|
|
43
|
-
Read MsgPack content from ``path``.
|
|
44
|
-
|
|
45
|
-
Parameters
|
|
46
|
-
----------
|
|
47
|
-
path : Path
|
|
48
|
-
Path to the MsgPack file on disk.
|
|
49
|
-
|
|
50
|
-
Returns
|
|
51
|
-
-------
|
|
52
|
-
JSONList
|
|
53
|
-
The list of dictionaries read from the MsgPack file.
|
|
54
|
-
"""
|
|
55
|
-
return stub.read(path, format_name='MSGPACK')
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
def write(
|
|
59
|
-
path: Path,
|
|
60
|
-
data: JSONData,
|
|
61
|
-
) -> int:
|
|
62
|
-
"""
|
|
63
|
-
Write ``data`` to MsgPack at ``path`` and return record count.
|
|
64
|
-
|
|
65
|
-
Parameters
|
|
66
|
-
----------
|
|
67
|
-
path : Path
|
|
68
|
-
Path to the MsgPack file on disk.
|
|
69
|
-
data : JSONData
|
|
70
|
-
Data to write as MsgPack. Should be a list of dictionaries or a
|
|
71
|
-
single dictionary.
|
|
72
|
-
|
|
73
|
-
Returns
|
|
74
|
-
-------
|
|
75
|
-
int
|
|
76
|
-
The number of rows written to the MsgPack file.
|
|
77
|
-
"""
|
|
78
|
-
return stub.write(path, data, format_name='MSGPACK')
|
etlplus/file/mustache.py
DELETED
|
@@ -1,78 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
:mod:`etlplus.file.mustache` module.
|
|
3
|
-
|
|
4
|
-
Helpers for reading/writing Mustache (MUSTACHE) template files.
|
|
5
|
-
|
|
6
|
-
Notes
|
|
7
|
-
-----
|
|
8
|
-
- A MUSTACHE file is a text file used for generating HTML or other text formats
|
|
9
|
-
by combining templates with data.
|
|
10
|
-
- Common cases:
|
|
11
|
-
- HTML templates.
|
|
12
|
-
- Email templates.
|
|
13
|
-
- Configuration files.
|
|
14
|
-
- Rule of thumb:
|
|
15
|
-
- If you need to work with Mustache template files, use this module for
|
|
16
|
-
reading and writing.
|
|
17
|
-
"""
|
|
18
|
-
|
|
19
|
-
from __future__ import annotations
|
|
20
|
-
|
|
21
|
-
from pathlib import Path
|
|
22
|
-
|
|
23
|
-
from ..types import JSONData
|
|
24
|
-
from ..types import JSONList
|
|
25
|
-
from . import stub
|
|
26
|
-
|
|
27
|
-
# SECTION: EXPORTS ========================================================== #
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
__all__ = [
|
|
31
|
-
'read',
|
|
32
|
-
'write',
|
|
33
|
-
]
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
# SECTION: FUNCTIONS ======================================================== #
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
def read(
|
|
40
|
-
path: Path,
|
|
41
|
-
) -> JSONList:
|
|
42
|
-
"""
|
|
43
|
-
Read MUSTACHE content from ``path``.
|
|
44
|
-
|
|
45
|
-
Parameters
|
|
46
|
-
----------
|
|
47
|
-
path : Path
|
|
48
|
-
Path to the MUSTACHE file on disk.
|
|
49
|
-
|
|
50
|
-
Returns
|
|
51
|
-
-------
|
|
52
|
-
JSONList
|
|
53
|
-
The list of dictionaries read from the MUSTACHE file.
|
|
54
|
-
"""
|
|
55
|
-
return stub.read(path, format_name='MUSTACHE')
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
def write(
|
|
59
|
-
path: Path,
|
|
60
|
-
data: JSONData,
|
|
61
|
-
) -> int:
|
|
62
|
-
"""
|
|
63
|
-
Write ``data`` to MUSTACHE file at ``path`` and return record count.
|
|
64
|
-
|
|
65
|
-
Parameters
|
|
66
|
-
----------
|
|
67
|
-
path : Path
|
|
68
|
-
Path to the MUSTACHE file on disk.
|
|
69
|
-
data : JSONData
|
|
70
|
-
Data to write as MUSTACHE file. Should be a list of dictionaries or a
|
|
71
|
-
single dictionary.
|
|
72
|
-
|
|
73
|
-
Returns
|
|
74
|
-
-------
|
|
75
|
-
int
|
|
76
|
-
The number of rows written to the MUSTACHE file.
|
|
77
|
-
"""
|
|
78
|
-
return stub.write(path, data, format_name='MUSTACHE')
|
etlplus/file/nc.py
DELETED
|
@@ -1,78 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
:mod:`etlplus.file.nc` module.
|
|
3
|
-
|
|
4
|
-
Helpers for reading/writing NetCDF (NC) data files.
|
|
5
|
-
|
|
6
|
-
Notes
|
|
7
|
-
-----
|
|
8
|
-
- A NC file is a binary file format used for array-oriented scientific data,
|
|
9
|
-
particularly in meteorology, oceanography, and climate science.
|
|
10
|
-
- Common cases:
|
|
11
|
-
- Storing multi-dimensional scientific data.
|
|
12
|
-
- Sharing large datasets in research communities.
|
|
13
|
-
- Efficient data access and manipulation.
|
|
14
|
-
- Rule of thumb:
|
|
15
|
-
- If the file follows the NetCDF standard, use this module for
|
|
16
|
-
reading and writing.
|
|
17
|
-
"""
|
|
18
|
-
|
|
19
|
-
from __future__ import annotations
|
|
20
|
-
|
|
21
|
-
from pathlib import Path
|
|
22
|
-
|
|
23
|
-
from ..types import JSONData
|
|
24
|
-
from ..types import JSONList
|
|
25
|
-
from . import stub
|
|
26
|
-
|
|
27
|
-
# SECTION: EXPORTS ========================================================== #
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
__all__ = [
|
|
31
|
-
'read',
|
|
32
|
-
'write',
|
|
33
|
-
]
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
# SECTION: FUNCTIONS ======================================================== #
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
def read(
|
|
40
|
-
path: Path,
|
|
41
|
-
) -> JSONList:
|
|
42
|
-
"""
|
|
43
|
-
Read NC content from ``path``.
|
|
44
|
-
|
|
45
|
-
Parameters
|
|
46
|
-
----------
|
|
47
|
-
path : Path
|
|
48
|
-
Path to the NC file on disk.
|
|
49
|
-
|
|
50
|
-
Returns
|
|
51
|
-
-------
|
|
52
|
-
JSONList
|
|
53
|
-
The list of dictionaries read from the NC file.
|
|
54
|
-
"""
|
|
55
|
-
return stub.read(path, format_name='NC')
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
def write(
|
|
59
|
-
path: Path,
|
|
60
|
-
data: JSONData,
|
|
61
|
-
) -> int:
|
|
62
|
-
"""
|
|
63
|
-
Write ``data`` to NC file at ``path`` and return record count.
|
|
64
|
-
|
|
65
|
-
Parameters
|
|
66
|
-
----------
|
|
67
|
-
path : Path
|
|
68
|
-
Path to the NC file on disk.
|
|
69
|
-
data : JSONData
|
|
70
|
-
Data to write as NC file. Should be a list of dictionaries or a
|
|
71
|
-
single dictionary.
|
|
72
|
-
|
|
73
|
-
Returns
|
|
74
|
-
-------
|
|
75
|
-
int
|
|
76
|
-
The number of rows written to the NC file.
|
|
77
|
-
"""
|
|
78
|
-
return stub.write(path, data, format_name='NC')
|
etlplus/file/ndjson.py
DELETED
|
@@ -1,108 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
:mod:`etlplus.file.ndjson` module.
|
|
3
|
-
|
|
4
|
-
Helpers for reading/writing Newline Delimited JSON (NDJSON) files.
|
|
5
|
-
|
|
6
|
-
Notes
|
|
7
|
-
-----
|
|
8
|
-
- An NDJSON file is a format where each line is a separate JSON object.
|
|
9
|
-
- Common cases:
|
|
10
|
-
- Streaming JSON data.
|
|
11
|
-
- Log files with JSON entries.
|
|
12
|
-
- Large datasets that are processed line-by-line.
|
|
13
|
-
- Rule of thumb:
|
|
14
|
-
- If the file follows the NDJSON specification, use this module for
|
|
15
|
-
reading and writing.
|
|
16
|
-
"""
|
|
17
|
-
|
|
18
|
-
from __future__ import annotations
|
|
19
|
-
|
|
20
|
-
import json
|
|
21
|
-
from pathlib import Path
|
|
22
|
-
from typing import cast
|
|
23
|
-
|
|
24
|
-
from ..types import JSONData
|
|
25
|
-
from ..types import JSONDict
|
|
26
|
-
from ..types import JSONList
|
|
27
|
-
from ..utils import count_records
|
|
28
|
-
from ._io import normalize_records
|
|
29
|
-
|
|
30
|
-
# SECTION: EXPORTS ========================================================== #
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
__all__ = [
|
|
34
|
-
'read',
|
|
35
|
-
'write',
|
|
36
|
-
]
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
# SECTION: FUNCTIONS ======================================================== #
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
def read(
|
|
43
|
-
path: Path,
|
|
44
|
-
) -> JSONList:
|
|
45
|
-
"""
|
|
46
|
-
Read NDJSON content from ``path``.
|
|
47
|
-
|
|
48
|
-
Parameters
|
|
49
|
-
----------
|
|
50
|
-
path : Path
|
|
51
|
-
Path to the NDJSON file on disk.
|
|
52
|
-
|
|
53
|
-
Returns
|
|
54
|
-
-------
|
|
55
|
-
JSONList
|
|
56
|
-
The list of dictionaries read from the NDJSON file.
|
|
57
|
-
|
|
58
|
-
Raises
|
|
59
|
-
------
|
|
60
|
-
TypeError
|
|
61
|
-
If any line in the NDJSON file is not a JSON object (dict).
|
|
62
|
-
"""
|
|
63
|
-
rows: JSONList = []
|
|
64
|
-
with path.open('r', encoding='utf-8') as handle:
|
|
65
|
-
for idx, line in enumerate(handle, start=1):
|
|
66
|
-
text = line.strip()
|
|
67
|
-
if not text:
|
|
68
|
-
continue
|
|
69
|
-
payload = json.loads(text)
|
|
70
|
-
if not isinstance(payload, dict):
|
|
71
|
-
raise TypeError(
|
|
72
|
-
f'NDJSON lines must be objects (dicts) (line {idx})',
|
|
73
|
-
)
|
|
74
|
-
rows.append(cast(JSONDict, payload))
|
|
75
|
-
return rows
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
def write(
|
|
79
|
-
path: Path,
|
|
80
|
-
data: JSONData,
|
|
81
|
-
) -> int:
|
|
82
|
-
"""
|
|
83
|
-
Write ``data`` to NDJSON at ``path``.
|
|
84
|
-
|
|
85
|
-
Parameters
|
|
86
|
-
----------
|
|
87
|
-
path : Path
|
|
88
|
-
Path to the NDJSON file on disk.
|
|
89
|
-
data : JSONData
|
|
90
|
-
Data to write.
|
|
91
|
-
|
|
92
|
-
Returns
|
|
93
|
-
-------
|
|
94
|
-
int
|
|
95
|
-
Number of records written.
|
|
96
|
-
"""
|
|
97
|
-
rows = normalize_records(data, 'NDJSON')
|
|
98
|
-
|
|
99
|
-
if not rows:
|
|
100
|
-
return 0
|
|
101
|
-
|
|
102
|
-
path.parent.mkdir(parents=True, exist_ok=True)
|
|
103
|
-
with path.open('w', encoding='utf-8') as handle:
|
|
104
|
-
for row in rows:
|
|
105
|
-
handle.write(json.dumps(row, ensure_ascii=False))
|
|
106
|
-
handle.write('\n')
|
|
107
|
-
|
|
108
|
-
return count_records(rows)
|
etlplus/file/numbers.py
DELETED
|
@@ -1,75 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
:mod:`etlplus.file.numbers` module.
|
|
3
|
-
|
|
4
|
-
Helpers for reading/writing Apple Numbers (NUMBERS) spreadsheet files.
|
|
5
|
-
|
|
6
|
-
Notes
|
|
7
|
-
-----
|
|
8
|
-
- A NUMBERS file is a spreadsheet file created by Apple Numbers.
|
|
9
|
-
- Common cases:
|
|
10
|
-
- Spreadsheet files created by Apple Numbers.
|
|
11
|
-
- Rule of thumb:
|
|
12
|
-
- If you need to read/write NUMBERS files, consider converting them to
|
|
13
|
-
more common formats like CSV or XLSX for better compatibility.
|
|
14
|
-
"""
|
|
15
|
-
|
|
16
|
-
from __future__ import annotations
|
|
17
|
-
|
|
18
|
-
from pathlib import Path
|
|
19
|
-
|
|
20
|
-
from ..types import JSONData
|
|
21
|
-
from ..types import JSONList
|
|
22
|
-
from . import stub
|
|
23
|
-
|
|
24
|
-
# SECTION: EXPORTS ========================================================== #
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
__all__ = [
|
|
28
|
-
'read',
|
|
29
|
-
'write',
|
|
30
|
-
]
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
# SECTION: FUNCTIONS ======================================================== #
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
def read(
|
|
37
|
-
path: Path,
|
|
38
|
-
) -> JSONList:
|
|
39
|
-
"""
|
|
40
|
-
Read NUMBERS content from ``path``.
|
|
41
|
-
|
|
42
|
-
Parameters
|
|
43
|
-
----------
|
|
44
|
-
path : Path
|
|
45
|
-
Path to the NUMBERS file on disk.
|
|
46
|
-
|
|
47
|
-
Returns
|
|
48
|
-
-------
|
|
49
|
-
JSONList
|
|
50
|
-
The list of dictionaries read from the NUMBERS file.
|
|
51
|
-
"""
|
|
52
|
-
return stub.read(path, format_name='NUMBERS')
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
def write(
|
|
56
|
-
path: Path,
|
|
57
|
-
data: JSONData,
|
|
58
|
-
) -> int:
|
|
59
|
-
"""
|
|
60
|
-
Write ``data`` to NUMBERS file at ``path`` and return record count.
|
|
61
|
-
|
|
62
|
-
Parameters
|
|
63
|
-
----------
|
|
64
|
-
path : Path
|
|
65
|
-
Path to the NUMBERS file on disk.
|
|
66
|
-
data : JSONData
|
|
67
|
-
Data to write as NUMBERS file. Should be a list of dictionaries or a
|
|
68
|
-
single dictionary.
|
|
69
|
-
|
|
70
|
-
Returns
|
|
71
|
-
-------
|
|
72
|
-
int
|
|
73
|
-
The number of rows written to the NUMBERS file.
|
|
74
|
-
"""
|
|
75
|
-
return stub.write(path, data, format_name='NUMBERS')
|