etlplus 0.9.0__py3-none-any.whl → 0.9.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- etlplus/README.md +37 -0
- etlplus/__init__.py +1 -26
- etlplus/api/README.md +51 -3
- etlplus/api/__init__.py +10 -0
- etlplus/api/config.py +39 -28
- etlplus/api/endpoint_client.py +3 -3
- etlplus/api/enums.py +51 -0
- etlplus/api/pagination/client.py +1 -1
- etlplus/api/rate_limiting/config.py +13 -1
- etlplus/api/rate_limiting/rate_limiter.py +8 -11
- etlplus/api/request_manager.py +11 -6
- etlplus/api/transport.py +14 -2
- etlplus/api/types.py +96 -6
- etlplus/{run_helpers.py → api/utils.py} +209 -153
- etlplus/cli/README.md +40 -0
- etlplus/cli/commands.py +94 -61
- etlplus/cli/constants.py +1 -1
- etlplus/cli/handlers.py +40 -12
- etlplus/cli/io.py +2 -2
- etlplus/cli/main.py +1 -1
- etlplus/cli/state.py +4 -7
- etlplus/database/README.md +48 -0
- etlplus/database/ddl.py +1 -1
- etlplus/database/engine.py +19 -3
- etlplus/database/orm.py +2 -0
- etlplus/database/schema.py +1 -1
- etlplus/enums.py +1 -107
- etlplus/file/README.md +105 -0
- etlplus/file/__init__.py +25 -0
- etlplus/file/_imports.py +141 -0
- etlplus/file/_io.py +160 -0
- etlplus/file/accdb.py +78 -0
- etlplus/file/arrow.py +78 -0
- etlplus/file/avro.py +176 -0
- etlplus/file/bson.py +77 -0
- etlplus/file/cbor.py +78 -0
- etlplus/file/cfg.py +79 -0
- etlplus/file/conf.py +80 -0
- etlplus/file/core.py +322 -0
- etlplus/file/csv.py +79 -0
- etlplus/file/dat.py +78 -0
- etlplus/file/dta.py +77 -0
- etlplus/file/duckdb.py +78 -0
- etlplus/file/enums.py +343 -0
- etlplus/file/feather.py +111 -0
- etlplus/file/fwf.py +77 -0
- etlplus/file/gz.py +123 -0
- etlplus/file/hbs.py +78 -0
- etlplus/file/hdf5.py +78 -0
- etlplus/file/ini.py +79 -0
- etlplus/file/ion.py +78 -0
- etlplus/file/jinja2.py +78 -0
- etlplus/file/json.py +98 -0
- etlplus/file/log.py +78 -0
- etlplus/file/mat.py +78 -0
- etlplus/file/mdb.py +78 -0
- etlplus/file/msgpack.py +78 -0
- etlplus/file/mustache.py +78 -0
- etlplus/file/nc.py +78 -0
- etlplus/file/ndjson.py +108 -0
- etlplus/file/numbers.py +75 -0
- etlplus/file/ods.py +79 -0
- etlplus/file/orc.py +111 -0
- etlplus/file/parquet.py +113 -0
- etlplus/file/pb.py +78 -0
- etlplus/file/pbf.py +77 -0
- etlplus/file/properties.py +78 -0
- etlplus/file/proto.py +77 -0
- etlplus/file/psv.py +79 -0
- etlplus/file/rda.py +78 -0
- etlplus/file/rds.py +78 -0
- etlplus/file/sas7bdat.py +78 -0
- etlplus/file/sav.py +77 -0
- etlplus/file/sqlite.py +78 -0
- etlplus/file/stub.py +84 -0
- etlplus/file/sylk.py +77 -0
- etlplus/file/tab.py +81 -0
- etlplus/file/toml.py +78 -0
- etlplus/file/tsv.py +80 -0
- etlplus/file/txt.py +102 -0
- etlplus/file/vm.py +78 -0
- etlplus/file/wks.py +77 -0
- etlplus/file/xls.py +88 -0
- etlplus/file/xlsm.py +79 -0
- etlplus/file/xlsx.py +99 -0
- etlplus/file/xml.py +185 -0
- etlplus/file/xpt.py +78 -0
- etlplus/file/yaml.py +95 -0
- etlplus/file/zip.py +175 -0
- etlplus/file/zsav.py +77 -0
- etlplus/ops/README.md +50 -0
- etlplus/ops/__init__.py +61 -0
- etlplus/{extract.py → ops/extract.py} +81 -99
- etlplus/{load.py → ops/load.py} +78 -101
- etlplus/{run.py → ops/run.py} +159 -127
- etlplus/{transform.py → ops/transform.py} +75 -68
- etlplus/{validation → ops}/utils.py +53 -17
- etlplus/{validate.py → ops/validate.py} +22 -12
- etlplus/templates/README.md +46 -0
- etlplus/types.py +5 -4
- etlplus/utils.py +136 -2
- etlplus/workflow/README.md +52 -0
- etlplus/{config → workflow}/__init__.py +10 -23
- etlplus/{config → workflow}/connector.py +58 -44
- etlplus/workflow/dag.py +105 -0
- etlplus/{config → workflow}/jobs.py +105 -32
- etlplus/{config → workflow}/pipeline.py +59 -51
- etlplus/{config → workflow}/profile.py +8 -5
- etlplus/workflow/types.py +115 -0
- {etlplus-0.9.0.dist-info → etlplus-0.9.2.dist-info}/METADATA +210 -17
- etlplus-0.9.2.dist-info/RECORD +134 -0
- {etlplus-0.9.0.dist-info → etlplus-0.9.2.dist-info}/WHEEL +1 -1
- etlplus/config/types.py +0 -204
- etlplus/config/utils.py +0 -120
- etlplus/file.py +0 -657
- etlplus/validation/__init__.py +0 -44
- etlplus-0.9.0.dist-info/RECORD +0 -65
- {etlplus-0.9.0.dist-info → etlplus-0.9.2.dist-info}/entry_points.txt +0 -0
- {etlplus-0.9.0.dist-info → etlplus-0.9.2.dist-info}/licenses/LICENSE +0 -0
- {etlplus-0.9.0.dist-info → etlplus-0.9.2.dist-info}/top_level.txt +0 -0
etlplus/file/sas7bdat.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""
|
|
2
|
+
:mod:`etlplus.file.sas7bdat` module.
|
|
3
|
+
|
|
4
|
+
Helpers for reading/writing SAS (SAS7BDAT) data files.
|
|
5
|
+
|
|
6
|
+
Notes
|
|
7
|
+
-----
|
|
8
|
+
- A SAS7BDAT file is a binary file format used by SAS to store datasets,
|
|
9
|
+
including variables, labels, and data types.
|
|
10
|
+
- Common cases:
|
|
11
|
+
- Delimited text files (e.g., CSV, TSV).
|
|
12
|
+
- Fixed-width formatted files.
|
|
13
|
+
- Custom formats specific to certain applications.
|
|
14
|
+
- Rule of thumb:
|
|
15
|
+
- If the file does not follow a specific standard format, use this module
|
|
16
|
+
for reading and writing.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
from ..types import JSONData
|
|
24
|
+
from ..types import JSONList
|
|
25
|
+
from . import stub
|
|
26
|
+
|
|
27
|
+
# SECTION: EXPORTS ========================================================== #
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
__all__ = [
|
|
31
|
+
'read',
|
|
32
|
+
'write',
|
|
33
|
+
]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
# SECTION: FUNCTIONS ======================================================== #
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def read(
|
|
40
|
+
path: Path,
|
|
41
|
+
) -> JSONList:
|
|
42
|
+
"""
|
|
43
|
+
Read DAT content from ``path``.
|
|
44
|
+
|
|
45
|
+
Parameters
|
|
46
|
+
----------
|
|
47
|
+
path : Path
|
|
48
|
+
Path to the SAS7BDAT file on disk.
|
|
49
|
+
|
|
50
|
+
Returns
|
|
51
|
+
-------
|
|
52
|
+
JSONList
|
|
53
|
+
The list of dictionaries read from the SAS7BDAT file.
|
|
54
|
+
"""
|
|
55
|
+
return stub.read(path, format_name='SAS7BDAT')
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def write(
|
|
59
|
+
path: Path,
|
|
60
|
+
data: JSONData,
|
|
61
|
+
) -> int:
|
|
62
|
+
"""
|
|
63
|
+
Write ``data`` to SAS7BDAT file at ``path`` and return record count.
|
|
64
|
+
|
|
65
|
+
Parameters
|
|
66
|
+
----------
|
|
67
|
+
path : Path
|
|
68
|
+
Path to the SAS7BDAT file on disk.
|
|
69
|
+
data : JSONData
|
|
70
|
+
Data to write as SAS7BDAT file. Should be a list of dictionaries or a
|
|
71
|
+
single dictionary.
|
|
72
|
+
|
|
73
|
+
Returns
|
|
74
|
+
-------
|
|
75
|
+
int
|
|
76
|
+
The number of rows written to the SAS7BDAT file.
|
|
77
|
+
"""
|
|
78
|
+
return stub.write(path, data, format_name='SAS7BDAT')
|
etlplus/file/sav.py
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""
|
|
2
|
+
:mod:`etlplus.file.sav` module.
|
|
3
|
+
|
|
4
|
+
Helpers for reading/writing SPSS (SAV) data files.
|
|
5
|
+
|
|
6
|
+
Notes
|
|
7
|
+
-----
|
|
8
|
+
- A SAV file is a binary file format used by SPSS to store datasets, including
|
|
9
|
+
variables, labels, and data types.
|
|
10
|
+
- Common cases:
|
|
11
|
+
- Reading data for analysis in Python.
|
|
12
|
+
- Writing processed data back to SPSS format.
|
|
13
|
+
- Rule of thumb:
|
|
14
|
+
- If you need to work with SPSS data files, use this module for reading
|
|
15
|
+
and writing.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
from ..types import JSONData
|
|
23
|
+
from ..types import JSONList
|
|
24
|
+
from . import stub
|
|
25
|
+
|
|
26
|
+
# SECTION: EXPORTS ========================================================== #
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
__all__ = [
|
|
30
|
+
'read',
|
|
31
|
+
'write',
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
# SECTION: FUNCTIONS ======================================================== #
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def read(
|
|
39
|
+
path: Path,
|
|
40
|
+
) -> JSONList:
|
|
41
|
+
"""
|
|
42
|
+
Read SAV content from ``path``.
|
|
43
|
+
|
|
44
|
+
Parameters
|
|
45
|
+
----------
|
|
46
|
+
path : Path
|
|
47
|
+
Path to the SAV file on disk.
|
|
48
|
+
|
|
49
|
+
Returns
|
|
50
|
+
-------
|
|
51
|
+
JSONList
|
|
52
|
+
The list of dictionaries read from the SAV file.
|
|
53
|
+
"""
|
|
54
|
+
return stub.read(path, format_name='SAV')
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def write(
|
|
58
|
+
path: Path,
|
|
59
|
+
data: JSONData,
|
|
60
|
+
) -> int:
|
|
61
|
+
"""
|
|
62
|
+
Write ``data`` to SAV file at ``path`` and return record count.
|
|
63
|
+
|
|
64
|
+
Parameters
|
|
65
|
+
----------
|
|
66
|
+
path : Path
|
|
67
|
+
Path to the SAV file on disk.
|
|
68
|
+
data : JSONData
|
|
69
|
+
Data to write as SAV file. Should be a list of dictionaries or a
|
|
70
|
+
single dictionary.
|
|
71
|
+
|
|
72
|
+
Returns
|
|
73
|
+
-------
|
|
74
|
+
int
|
|
75
|
+
The number of rows written to the SAV file.
|
|
76
|
+
"""
|
|
77
|
+
return stub.write(path, data, format_name='SAV')
|
etlplus/file/sqlite.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""
|
|
2
|
+
:mod:`etlplus.file.sqlite` module.
|
|
3
|
+
|
|
4
|
+
Helpers for reading/writing SQLite database (SQLITE) files.
|
|
5
|
+
|
|
6
|
+
Notes
|
|
7
|
+
-----
|
|
8
|
+
- A SQLITE file is a self-contained, serverless database file format used by
|
|
9
|
+
SQLite.
|
|
10
|
+
- Common cases:
|
|
11
|
+
- Lightweight database applications.
|
|
12
|
+
- Embedded database solutions.
|
|
13
|
+
- Mobile and desktop applications requiring local data storage.
|
|
14
|
+
- Rule of thumb:
|
|
15
|
+
- If the file follows the SQLITE specification, use this module for reading
|
|
16
|
+
and writing.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
from ..types import JSONData
|
|
24
|
+
from ..types import JSONList
|
|
25
|
+
from . import stub
|
|
26
|
+
|
|
27
|
+
# SECTION: EXPORTS ========================================================== #
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
__all__ = [
|
|
31
|
+
'read',
|
|
32
|
+
'write',
|
|
33
|
+
]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
# SECTION: FUNCTIONS ======================================================== #
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def read(
|
|
40
|
+
path: Path,
|
|
41
|
+
) -> JSONList:
|
|
42
|
+
"""
|
|
43
|
+
Read SQLITE content from ``path``.
|
|
44
|
+
|
|
45
|
+
Parameters
|
|
46
|
+
----------
|
|
47
|
+
path : Path
|
|
48
|
+
Path to the SQLITE file on disk.
|
|
49
|
+
|
|
50
|
+
Returns
|
|
51
|
+
-------
|
|
52
|
+
JSONList
|
|
53
|
+
The list of dictionaries read from the SQLITE file.
|
|
54
|
+
"""
|
|
55
|
+
return stub.read(path, format_name='SQLITE')
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def write(
|
|
59
|
+
path: Path,
|
|
60
|
+
data: JSONData,
|
|
61
|
+
) -> int:
|
|
62
|
+
"""
|
|
63
|
+
Write ``data`` to SQLITE at ``path`` and return record count.
|
|
64
|
+
|
|
65
|
+
Parameters
|
|
66
|
+
----------
|
|
67
|
+
path : Path
|
|
68
|
+
Path to the SQLITE file on disk.
|
|
69
|
+
data : JSONData
|
|
70
|
+
Data to write as SQLITE. Should be a list of dictionaries or a
|
|
71
|
+
single dictionary.
|
|
72
|
+
|
|
73
|
+
Returns
|
|
74
|
+
-------
|
|
75
|
+
int
|
|
76
|
+
The number of rows written to the SQLITE file.
|
|
77
|
+
"""
|
|
78
|
+
return stub.write(path, data, format_name='SQLITE')
|
etlplus/file/stub.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""
|
|
2
|
+
:mod:`etlplus.file.stub` module.
|
|
3
|
+
|
|
4
|
+
Helpers for reading/writing stubbed files.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from ..types import JSONData
|
|
12
|
+
from ..types import JSONList
|
|
13
|
+
|
|
14
|
+
# SECTION: EXPORTS ========================================================== #
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
__all__ = [
|
|
18
|
+
'read',
|
|
19
|
+
'write',
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
# SECTION: FUNCTIONS ======================================================== #
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def read(
|
|
27
|
+
path: Path,
|
|
28
|
+
format_name: str = 'Stubbed',
|
|
29
|
+
) -> JSONList:
|
|
30
|
+
"""
|
|
31
|
+
Raises a :class:`NotImplementedError` for stubbed reads.
|
|
32
|
+
|
|
33
|
+
Parameters
|
|
34
|
+
----------
|
|
35
|
+
path : Path
|
|
36
|
+
Path to the stubbed file on disk.
|
|
37
|
+
format_name : str
|
|
38
|
+
Human-readable format name.
|
|
39
|
+
|
|
40
|
+
Returns
|
|
41
|
+
-------
|
|
42
|
+
JSONList
|
|
43
|
+
The list of dictionaries read from the stubbed file.
|
|
44
|
+
|
|
45
|
+
Raises
|
|
46
|
+
------
|
|
47
|
+
NotImplementedError
|
|
48
|
+
Always, since this is a stub implementation.
|
|
49
|
+
"""
|
|
50
|
+
_ = path
|
|
51
|
+
raise NotImplementedError(f'{format_name} read is not implemented yet')
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def write(
|
|
55
|
+
path: Path,
|
|
56
|
+
data: JSONData,
|
|
57
|
+
format_name: str = 'Stubbed',
|
|
58
|
+
) -> int:
|
|
59
|
+
"""
|
|
60
|
+
Raises a :class:`NotImplementedError` for stubbed writes.
|
|
61
|
+
|
|
62
|
+
Parameters
|
|
63
|
+
----------
|
|
64
|
+
path : Path
|
|
65
|
+
Path to the stubbed file on disk.
|
|
66
|
+
data : JSONData
|
|
67
|
+
Data to write as stubbed file. Should be a list of dictionaries or a
|
|
68
|
+
single dictionary.
|
|
69
|
+
format_name : str
|
|
70
|
+
Human-readable format name.
|
|
71
|
+
|
|
72
|
+
Returns
|
|
73
|
+
-------
|
|
74
|
+
int
|
|
75
|
+
The number of rows written to the stubbed file.
|
|
76
|
+
|
|
77
|
+
Raises
|
|
78
|
+
------
|
|
79
|
+
NotImplementedError
|
|
80
|
+
Always, since this is a stub implementation.
|
|
81
|
+
"""
|
|
82
|
+
_ = path
|
|
83
|
+
_ = data
|
|
84
|
+
raise NotImplementedError(f'{format_name} write is not implemented yet')
|
etlplus/file/sylk.py
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""
|
|
2
|
+
:mod:`etlplus.file.sylk` module.
|
|
3
|
+
|
|
4
|
+
Helpers for reading/writing Symbolic Link (SYLK) data files.
|
|
5
|
+
|
|
6
|
+
Notes
|
|
7
|
+
-----
|
|
8
|
+
- A SYLK file is a text-based file format used to represent spreadsheet
|
|
9
|
+
data, including cell values, formulas, and formatting.
|
|
10
|
+
- Common cases:
|
|
11
|
+
- Storing spreadsheet data in a human-readable format.
|
|
12
|
+
- Exchanging data between different spreadsheet applications.
|
|
13
|
+
- Rule of thumb:
|
|
14
|
+
- If you need to work with SYLK files, use this module for reading
|
|
15
|
+
and writing.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
from ..types import JSONData
|
|
23
|
+
from ..types import JSONList
|
|
24
|
+
from . import stub
|
|
25
|
+
|
|
26
|
+
# SECTION: EXPORTS ========================================================== #
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
__all__ = [
|
|
30
|
+
'read',
|
|
31
|
+
'write',
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
# SECTION: FUNCTIONS ======================================================== #
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def read(
|
|
39
|
+
path: Path,
|
|
40
|
+
) -> JSONList:
|
|
41
|
+
"""
|
|
42
|
+
Read SYLK content from ``path``.
|
|
43
|
+
|
|
44
|
+
Parameters
|
|
45
|
+
----------
|
|
46
|
+
path : Path
|
|
47
|
+
Path to the SYLK file on disk.
|
|
48
|
+
|
|
49
|
+
Returns
|
|
50
|
+
-------
|
|
51
|
+
JSONList
|
|
52
|
+
The list of dictionaries read from the SYLK file.
|
|
53
|
+
"""
|
|
54
|
+
return stub.read(path, format_name='SYLK')
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def write(
|
|
58
|
+
path: Path,
|
|
59
|
+
data: JSONData,
|
|
60
|
+
) -> int:
|
|
61
|
+
"""
|
|
62
|
+
Write ``data`` to SYLK file at ``path`` and return record count.
|
|
63
|
+
|
|
64
|
+
Parameters
|
|
65
|
+
----------
|
|
66
|
+
path : Path
|
|
67
|
+
Path to the SYLK file on disk.
|
|
68
|
+
data : JSONData
|
|
69
|
+
Data to write as SYLK file. Should be a list of dictionaries or a
|
|
70
|
+
single dictionary.
|
|
71
|
+
|
|
72
|
+
Returns
|
|
73
|
+
-------
|
|
74
|
+
int
|
|
75
|
+
The number of rows written to the SYLK file.
|
|
76
|
+
"""
|
|
77
|
+
return stub.write(path, data, format_name='SYLK')
|
etlplus/file/tab.py
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""
|
|
2
|
+
:mod:`etlplus.file.tab` module.
|
|
3
|
+
|
|
4
|
+
Helpers for reading/writing "tab"-formatted (TAB) files.
|
|
5
|
+
|
|
6
|
+
Notes
|
|
7
|
+
-----
|
|
8
|
+
- A TAB file is not necessarily a TSV file when tabs aren’t actually the
|
|
9
|
+
delimiter that defines the fields, even if the text looks column-aligned.
|
|
10
|
+
- Common cases:
|
|
11
|
+
- Fixed-width text (FWF) that uses tabs for alignment.
|
|
12
|
+
- Mixed whitespace (tabs + spaces) as “pretty printing”.
|
|
13
|
+
- Tabs embedded inside quoted fields (or unescaped tabs in free text).
|
|
14
|
+
- Header/metadata lines or multi-line records that break TSV assumptions.
|
|
15
|
+
- Not actually tab-delimited despite the name.
|
|
16
|
+
- Rule of thumb:
|
|
17
|
+
- If the file is truly tab-delimited, use :mod:`etlplus.file.tsv`.
|
|
18
|
+
- If the file has fixed-width fields, use :mod:`etlplus.file.fwf`.
|
|
19
|
+
- Otherwise, use :mod:`etlplus.file.tab` (i.e., this module).
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
|
|
26
|
+
from ..types import JSONData
|
|
27
|
+
from ..types import JSONList
|
|
28
|
+
from . import stub
|
|
29
|
+
|
|
30
|
+
# SECTION: EXPORTS ========================================================== #
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
__all__ = [
|
|
34
|
+
'read',
|
|
35
|
+
'write',
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
# SECTION: FUNCTIONS ======================================================== #
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def read(
|
|
43
|
+
path: Path,
|
|
44
|
+
) -> JSONList:
|
|
45
|
+
"""
|
|
46
|
+
Read TAB content from ``path``.
|
|
47
|
+
|
|
48
|
+
Parameters
|
|
49
|
+
----------
|
|
50
|
+
path : Path
|
|
51
|
+
Path to the TAB file on disk.
|
|
52
|
+
|
|
53
|
+
Returns
|
|
54
|
+
-------
|
|
55
|
+
JSONList
|
|
56
|
+
The list of dictionaries read from the TAB file.
|
|
57
|
+
"""
|
|
58
|
+
return stub.read(path, format_name='TAB')
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def write(
|
|
62
|
+
path: Path,
|
|
63
|
+
data: JSONData,
|
|
64
|
+
) -> int:
|
|
65
|
+
"""
|
|
66
|
+
Write ``data`` to TAB file at ``path`` and return record count.
|
|
67
|
+
|
|
68
|
+
Parameters
|
|
69
|
+
----------
|
|
70
|
+
path : Path
|
|
71
|
+
Path to the TAB file on disk.
|
|
72
|
+
data : JSONData
|
|
73
|
+
Data to write as TAB file. Should be a list of dictionaries or a
|
|
74
|
+
single dictionary.
|
|
75
|
+
|
|
76
|
+
Returns
|
|
77
|
+
-------
|
|
78
|
+
int
|
|
79
|
+
The number of rows written to the TAB file.
|
|
80
|
+
"""
|
|
81
|
+
return stub.write(path, data, format_name='TAB')
|
etlplus/file/toml.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""
|
|
2
|
+
:mod:`etlplus.file.toml` module.
|
|
3
|
+
|
|
4
|
+
Helpers for reading/writing Tom's Obvious Minimal Language (TOML) files.
|
|
5
|
+
|
|
6
|
+
Notes
|
|
7
|
+
-----
|
|
8
|
+
- A TOML file is a configuration file that uses the TOML syntax.
|
|
9
|
+
- Common cases:
|
|
10
|
+
- Simple key-value pairs.
|
|
11
|
+
- Nested tables and arrays.
|
|
12
|
+
- Data types such as strings, integers, floats, booleans, dates, and
|
|
13
|
+
arrays.
|
|
14
|
+
- Rule of thumb:
|
|
15
|
+
- If the file follows the TOML specification, use this module for
|
|
16
|
+
reading and writing.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
from ..types import JSONData
|
|
24
|
+
from ..types import JSONList
|
|
25
|
+
from . import stub
|
|
26
|
+
|
|
27
|
+
# SECTION: EXPORTS ========================================================== #
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
__all__ = [
|
|
31
|
+
'read',
|
|
32
|
+
'write',
|
|
33
|
+
]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
# SECTION: FUNCTIONS ======================================================== #
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def read(
|
|
40
|
+
path: Path,
|
|
41
|
+
) -> JSONList:
|
|
42
|
+
"""
|
|
43
|
+
Read TOML content from ``path``.
|
|
44
|
+
|
|
45
|
+
Parameters
|
|
46
|
+
----------
|
|
47
|
+
path : Path
|
|
48
|
+
Path to the TOML file on disk.
|
|
49
|
+
|
|
50
|
+
Returns
|
|
51
|
+
-------
|
|
52
|
+
JSONList
|
|
53
|
+
The list of dictionaries read from the TOML file.
|
|
54
|
+
"""
|
|
55
|
+
return stub.read(path, format_name='TOML')
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def write(
|
|
59
|
+
path: Path,
|
|
60
|
+
data: JSONData,
|
|
61
|
+
) -> int:
|
|
62
|
+
"""
|
|
63
|
+
Write ``data`` to TOML at ``path`` and return record count.
|
|
64
|
+
|
|
65
|
+
Parameters
|
|
66
|
+
----------
|
|
67
|
+
path : Path
|
|
68
|
+
Path to the TOML file on disk.
|
|
69
|
+
data : JSONData
|
|
70
|
+
Data to write as TOML. Should be a list of dictionaries or a
|
|
71
|
+
single dictionary.
|
|
72
|
+
|
|
73
|
+
Returns
|
|
74
|
+
-------
|
|
75
|
+
int
|
|
76
|
+
The number of rows written to the TOML file.
|
|
77
|
+
"""
|
|
78
|
+
return stub.write(path, data, format_name='TOML')
|
etlplus/file/tsv.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""
|
|
2
|
+
:mod:`etlplus.file.tsv` module.
|
|
3
|
+
|
|
4
|
+
Helpers for reading/writing Tab-Separated Values (TSV) files.
|
|
5
|
+
|
|
6
|
+
Notes
|
|
7
|
+
-----
|
|
8
|
+
- A TSV file is a plain text file that uses the tab character (``\t``) to
|
|
9
|
+
separate values.
|
|
10
|
+
- Common cases:
|
|
11
|
+
- Each line in the file represents a single record.
|
|
12
|
+
- The first line often contains headers that define the column names.
|
|
13
|
+
- Values may be enclosed in quotes, especially if they contain tabs
|
|
14
|
+
or special characters.
|
|
15
|
+
- Rule of thumb:
|
|
16
|
+
- If the file follows the TSV specification, use this module for
|
|
17
|
+
reading and writing.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
|
|
24
|
+
from ..types import JSONData
|
|
25
|
+
from ..types import JSONList
|
|
26
|
+
from ._io import read_delimited
|
|
27
|
+
from ._io import write_delimited
|
|
28
|
+
|
|
29
|
+
# SECTION: EXPORTS ========================================================== #
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
__all__ = [
|
|
33
|
+
'read',
|
|
34
|
+
'write',
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
# SECTION: FUNCTIONS ======================================================== #
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def read(
|
|
42
|
+
path: Path,
|
|
43
|
+
) -> JSONList:
|
|
44
|
+
"""
|
|
45
|
+
Read TSV content from ``path``.
|
|
46
|
+
|
|
47
|
+
Parameters
|
|
48
|
+
----------
|
|
49
|
+
path : Path
|
|
50
|
+
Path to the TSV file on disk.
|
|
51
|
+
|
|
52
|
+
Returns
|
|
53
|
+
-------
|
|
54
|
+
JSONList
|
|
55
|
+
The list of dictionaries read from the TSV file.
|
|
56
|
+
"""
|
|
57
|
+
return read_delimited(path, delimiter='\t')
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def write(
|
|
61
|
+
path: Path,
|
|
62
|
+
data: JSONData,
|
|
63
|
+
) -> int:
|
|
64
|
+
"""
|
|
65
|
+
Write ``data`` to TSV at ``path`` and return record count.
|
|
66
|
+
|
|
67
|
+
Parameters
|
|
68
|
+
----------
|
|
69
|
+
path : Path
|
|
70
|
+
Path to the TSV file on disk.
|
|
71
|
+
data : JSONData
|
|
72
|
+
Data to write as TSV. Should be a list of dictionaries or a
|
|
73
|
+
single dictionary.
|
|
74
|
+
|
|
75
|
+
Returns
|
|
76
|
+
-------
|
|
77
|
+
int
|
|
78
|
+
The number of rows written to the TSV file.
|
|
79
|
+
"""
|
|
80
|
+
return write_delimited(path, data, delimiter='\t')
|