etlplus 0.9.2__py3-none-any.whl → 0.10.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- etlplus/__init__.py +26 -1
- etlplus/api/README.md +3 -51
- etlplus/api/__init__.py +0 -10
- etlplus/api/config.py +28 -39
- etlplus/api/endpoint_client.py +3 -3
- etlplus/api/pagination/client.py +1 -1
- etlplus/api/rate_limiting/config.py +1 -13
- etlplus/api/rate_limiting/rate_limiter.py +11 -8
- etlplus/api/request_manager.py +6 -11
- etlplus/api/transport.py +2 -14
- etlplus/api/types.py +6 -96
- etlplus/cli/commands.py +43 -76
- etlplus/cli/constants.py +1 -1
- etlplus/cli/handlers.py +12 -40
- etlplus/cli/io.py +2 -2
- etlplus/cli/main.py +1 -1
- etlplus/cli/state.py +7 -4
- etlplus/{workflow → config}/__init__.py +23 -10
- etlplus/{workflow → config}/connector.py +44 -58
- etlplus/{workflow → config}/jobs.py +32 -105
- etlplus/{workflow → config}/pipeline.py +51 -59
- etlplus/{workflow → config}/profile.py +5 -8
- etlplus/config/types.py +204 -0
- etlplus/config/utils.py +120 -0
- etlplus/database/ddl.py +1 -1
- etlplus/database/engine.py +3 -19
- etlplus/database/orm.py +0 -2
- etlplus/database/schema.py +1 -1
- etlplus/enums.py +266 -0
- etlplus/{ops/extract.py → extract.py} +99 -81
- etlplus/file.py +652 -0
- etlplus/{ops/load.py → load.py} +101 -78
- etlplus/{ops/run.py → run.py} +127 -159
- etlplus/{api/utils.py → run_helpers.py} +153 -209
- etlplus/{ops/transform.py → transform.py} +68 -75
- etlplus/types.py +4 -5
- etlplus/utils.py +2 -136
- etlplus/{ops/validate.py → validate.py} +12 -22
- etlplus/validation/__init__.py +44 -0
- etlplus/{ops → validation}/utils.py +17 -53
- {etlplus-0.9.2.dist-info → etlplus-0.10.1.dist-info}/METADATA +17 -210
- etlplus-0.10.1.dist-info/RECORD +65 -0
- {etlplus-0.9.2.dist-info → etlplus-0.10.1.dist-info}/WHEEL +1 -1
- etlplus/README.md +0 -37
- etlplus/api/enums.py +0 -51
- etlplus/cli/README.md +0 -40
- etlplus/database/README.md +0 -48
- etlplus/file/README.md +0 -105
- etlplus/file/__init__.py +0 -25
- etlplus/file/_imports.py +0 -141
- etlplus/file/_io.py +0 -160
- etlplus/file/accdb.py +0 -78
- etlplus/file/arrow.py +0 -78
- etlplus/file/avro.py +0 -176
- etlplus/file/bson.py +0 -77
- etlplus/file/cbor.py +0 -78
- etlplus/file/cfg.py +0 -79
- etlplus/file/conf.py +0 -80
- etlplus/file/core.py +0 -322
- etlplus/file/csv.py +0 -79
- etlplus/file/dat.py +0 -78
- etlplus/file/dta.py +0 -77
- etlplus/file/duckdb.py +0 -78
- etlplus/file/enums.py +0 -343
- etlplus/file/feather.py +0 -111
- etlplus/file/fwf.py +0 -77
- etlplus/file/gz.py +0 -123
- etlplus/file/hbs.py +0 -78
- etlplus/file/hdf5.py +0 -78
- etlplus/file/ini.py +0 -79
- etlplus/file/ion.py +0 -78
- etlplus/file/jinja2.py +0 -78
- etlplus/file/json.py +0 -98
- etlplus/file/log.py +0 -78
- etlplus/file/mat.py +0 -78
- etlplus/file/mdb.py +0 -78
- etlplus/file/msgpack.py +0 -78
- etlplus/file/mustache.py +0 -78
- etlplus/file/nc.py +0 -78
- etlplus/file/ndjson.py +0 -108
- etlplus/file/numbers.py +0 -75
- etlplus/file/ods.py +0 -79
- etlplus/file/orc.py +0 -111
- etlplus/file/parquet.py +0 -113
- etlplus/file/pb.py +0 -78
- etlplus/file/pbf.py +0 -77
- etlplus/file/properties.py +0 -78
- etlplus/file/proto.py +0 -77
- etlplus/file/psv.py +0 -79
- etlplus/file/rda.py +0 -78
- etlplus/file/rds.py +0 -78
- etlplus/file/sas7bdat.py +0 -78
- etlplus/file/sav.py +0 -77
- etlplus/file/sqlite.py +0 -78
- etlplus/file/stub.py +0 -84
- etlplus/file/sylk.py +0 -77
- etlplus/file/tab.py +0 -81
- etlplus/file/toml.py +0 -78
- etlplus/file/tsv.py +0 -80
- etlplus/file/txt.py +0 -102
- etlplus/file/vm.py +0 -78
- etlplus/file/wks.py +0 -77
- etlplus/file/xls.py +0 -88
- etlplus/file/xlsm.py +0 -79
- etlplus/file/xlsx.py +0 -99
- etlplus/file/xml.py +0 -185
- etlplus/file/xpt.py +0 -78
- etlplus/file/yaml.py +0 -95
- etlplus/file/zip.py +0 -175
- etlplus/file/zsav.py +0 -77
- etlplus/ops/README.md +0 -50
- etlplus/ops/__init__.py +0 -61
- etlplus/templates/README.md +0 -46
- etlplus/workflow/README.md +0 -52
- etlplus/workflow/dag.py +0 -105
- etlplus/workflow/types.py +0 -115
- etlplus-0.9.2.dist-info/RECORD +0 -134
- {etlplus-0.9.2.dist-info → etlplus-0.10.1.dist-info}/entry_points.txt +0 -0
- {etlplus-0.9.2.dist-info → etlplus-0.10.1.dist-info}/licenses/LICENSE +0 -0
- {etlplus-0.9.2.dist-info → etlplus-0.10.1.dist-info}/top_level.txt +0 -0
etlplus/file/bson.py
DELETED
|
@@ -1,77 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
:mod:`etlplus.file.bson` module.
|
|
3
|
-
|
|
4
|
-
Helpers for reading/writing Binary JSON (BSON) files.
|
|
5
|
-
|
|
6
|
-
Notes
|
|
7
|
-
-----
|
|
8
|
-
- A BSON file is a binary-encoded serialization of JSON-like documents.
|
|
9
|
-
- Common cases:
|
|
10
|
-
- Data storage in MongoDB.
|
|
11
|
-
- Efficient data interchange between systems.
|
|
12
|
-
- Handling of complex data types not supported in standard JSON.
|
|
13
|
-
- Rule of thumb:
|
|
14
|
-
- If the file follows the BSON specification, use this module for reading
|
|
15
|
-
and writing.
|
|
16
|
-
"""
|
|
17
|
-
|
|
18
|
-
from __future__ import annotations
|
|
19
|
-
|
|
20
|
-
from pathlib import Path
|
|
21
|
-
|
|
22
|
-
from ..types import JSONData
|
|
23
|
-
from ..types import JSONList
|
|
24
|
-
from . import stub
|
|
25
|
-
|
|
26
|
-
# SECTION: EXPORTS ========================================================== #
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
__all__ = [
|
|
30
|
-
'read',
|
|
31
|
-
'write',
|
|
32
|
-
]
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
# SECTION: FUNCTIONS ======================================================== #
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
def read(
|
|
39
|
-
path: Path,
|
|
40
|
-
) -> JSONList:
|
|
41
|
-
"""
|
|
42
|
-
Read BSON content from ``path``.
|
|
43
|
-
|
|
44
|
-
Parameters
|
|
45
|
-
----------
|
|
46
|
-
path : Path
|
|
47
|
-
Path to the BSON file on disk.
|
|
48
|
-
|
|
49
|
-
Returns
|
|
50
|
-
-------
|
|
51
|
-
JSONList
|
|
52
|
-
The list of dictionaries read from the BSON file.
|
|
53
|
-
"""
|
|
54
|
-
return stub.read(path, format_name='BSON')
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
def write(
|
|
58
|
-
path: Path,
|
|
59
|
-
data: JSONData,
|
|
60
|
-
) -> int:
|
|
61
|
-
"""
|
|
62
|
-
Write ``data`` to BSON at ``path`` and return record count.
|
|
63
|
-
|
|
64
|
-
Parameters
|
|
65
|
-
----------
|
|
66
|
-
path : Path
|
|
67
|
-
Path to the BSON file on disk.
|
|
68
|
-
data : JSONData
|
|
69
|
-
Data to write as BSON. Should be a list of dictionaries or a
|
|
70
|
-
single dictionary.
|
|
71
|
-
|
|
72
|
-
Returns
|
|
73
|
-
-------
|
|
74
|
-
int
|
|
75
|
-
The number of rows written to the BSON file.
|
|
76
|
-
"""
|
|
77
|
-
return stub.write(path, data, format_name='BSON')
|
etlplus/file/cbor.py
DELETED
|
@@ -1,78 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
:mod:`etlplus.file.cbor` module.
|
|
3
|
-
|
|
4
|
-
Helpers for reading/writing Concise Binary Object Representation (CBOR) files.
|
|
5
|
-
|
|
6
|
-
Notes
|
|
7
|
-
-----
|
|
8
|
-
- A CBOR file is a binary data format designed for small code size and message
|
|
9
|
-
size, suitable for constrained environments.
|
|
10
|
-
- Common cases:
|
|
11
|
-
- IoT data interchange.
|
|
12
|
-
- Efficient data serialization.
|
|
13
|
-
- Storage of structured data in a compact binary form.
|
|
14
|
-
- Rule of thumb:
|
|
15
|
-
- If the file follows the CBOR specification, use this module for reading
|
|
16
|
-
and writing.
|
|
17
|
-
"""
|
|
18
|
-
|
|
19
|
-
from __future__ import annotations
|
|
20
|
-
|
|
21
|
-
from pathlib import Path
|
|
22
|
-
|
|
23
|
-
from ..types import JSONData
|
|
24
|
-
from ..types import JSONList
|
|
25
|
-
from . import stub
|
|
26
|
-
|
|
27
|
-
# SECTION: EXPORTS ========================================================== #
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
__all__ = [
|
|
31
|
-
'read',
|
|
32
|
-
'write',
|
|
33
|
-
]
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
# SECTION: FUNCTIONS ======================================================== #
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
def read(
|
|
40
|
-
path: Path,
|
|
41
|
-
) -> JSONList:
|
|
42
|
-
"""
|
|
43
|
-
Read CBOR content from ``path``.
|
|
44
|
-
|
|
45
|
-
Parameters
|
|
46
|
-
----------
|
|
47
|
-
path : Path
|
|
48
|
-
Path to the CBOR file on disk.
|
|
49
|
-
|
|
50
|
-
Returns
|
|
51
|
-
-------
|
|
52
|
-
JSONList
|
|
53
|
-
The list of dictionaries read from the CBOR file.
|
|
54
|
-
"""
|
|
55
|
-
return stub.read(path, format_name='CBOR')
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
def write(
|
|
59
|
-
path: Path,
|
|
60
|
-
data: JSONData,
|
|
61
|
-
) -> int:
|
|
62
|
-
"""
|
|
63
|
-
Write ``data`` to CBOR at ``path`` and return record count.
|
|
64
|
-
|
|
65
|
-
Parameters
|
|
66
|
-
----------
|
|
67
|
-
path : Path
|
|
68
|
-
Path to the CBOR file on disk.
|
|
69
|
-
data : JSONData
|
|
70
|
-
Data to write as CBOR. Should be a list of dictionaries or a
|
|
71
|
-
single dictionary.
|
|
72
|
-
|
|
73
|
-
Returns
|
|
74
|
-
-------
|
|
75
|
-
int
|
|
76
|
-
The number of rows written to the CBOR file.
|
|
77
|
-
"""
|
|
78
|
-
return stub.write(path, data, format_name='CBOR')
|
etlplus/file/cfg.py
DELETED
|
@@ -1,79 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
:mod:`etlplus.file.cfg` module.
|
|
3
|
-
|
|
4
|
-
Helpers for reading/writing config (CFG) files.
|
|
5
|
-
|
|
6
|
-
Notes
|
|
7
|
-
-----
|
|
8
|
-
- A CFG file is a configuration file that may use various syntaxes, such as
|
|
9
|
-
INI, YAML, or custom formats.
|
|
10
|
-
- Common cases:
|
|
11
|
-
- INI-style key-value pairs with sections (such as in Python ecosystems,
|
|
12
|
-
using ``configparser``).
|
|
13
|
-
- YAML-like structures with indentation.
|
|
14
|
-
- Custom formats specific to certain applications.
|
|
15
|
-
- Rule of thumb:
|
|
16
|
-
- If the file follows a standard format like INI or YAML, consider using
|
|
17
|
-
dedicated parsers.
|
|
18
|
-
"""
|
|
19
|
-
|
|
20
|
-
from __future__ import annotations
|
|
21
|
-
|
|
22
|
-
from pathlib import Path
|
|
23
|
-
|
|
24
|
-
from ..types import JSONData
|
|
25
|
-
from ..types import JSONList
|
|
26
|
-
from . import stub
|
|
27
|
-
|
|
28
|
-
# SECTION: EXPORTS ========================================================== #
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
__all__ = [
|
|
32
|
-
'read',
|
|
33
|
-
'write',
|
|
34
|
-
]
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
# SECTION: FUNCTIONS ======================================================== #
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
def read(
|
|
41
|
-
path: Path,
|
|
42
|
-
) -> JSONList:
|
|
43
|
-
"""
|
|
44
|
-
Read CFG content from ``path``.
|
|
45
|
-
|
|
46
|
-
Parameters
|
|
47
|
-
----------
|
|
48
|
-
path : Path
|
|
49
|
-
Path to the CFG file on disk.
|
|
50
|
-
|
|
51
|
-
Returns
|
|
52
|
-
-------
|
|
53
|
-
JSONList
|
|
54
|
-
The list of dictionaries read from the CFG file.
|
|
55
|
-
"""
|
|
56
|
-
return stub.read(path, format_name='CFG')
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
def write(
|
|
60
|
-
path: Path,
|
|
61
|
-
data: JSONData,
|
|
62
|
-
) -> int:
|
|
63
|
-
"""
|
|
64
|
-
Write ``data`` to CFG file at ``path`` and return record count.
|
|
65
|
-
|
|
66
|
-
Parameters
|
|
67
|
-
----------
|
|
68
|
-
path : Path
|
|
69
|
-
Path to the CFG file on disk.
|
|
70
|
-
data : JSONData
|
|
71
|
-
Data to write as CFG file. Should be a list of dictionaries or a
|
|
72
|
-
single dictionary.
|
|
73
|
-
|
|
74
|
-
Returns
|
|
75
|
-
-------
|
|
76
|
-
int
|
|
77
|
-
The number of rows written to the CFG file.
|
|
78
|
-
"""
|
|
79
|
-
return stub.write(path, data, format_name='CFG')
|
etlplus/file/conf.py
DELETED
|
@@ -1,80 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
:mod:`etlplus.file.conf` module.
|
|
3
|
-
|
|
4
|
-
Helpers for reading/writing config (CONF) files.
|
|
5
|
-
|
|
6
|
-
Notes
|
|
7
|
-
-----
|
|
8
|
-
- A CONF file is a configuration file that may use various syntaxes, such as
|
|
9
|
-
INI, YAML, or custom formats.
|
|
10
|
-
- Common cases:
|
|
11
|
-
- INI-style key-value pairs with sections.
|
|
12
|
-
- YAML-like structures with indentation.
|
|
13
|
-
- Custom formats specific to certain applications (such as Unix-like
|
|
14
|
-
systems, where ``.conf`` is a strong convention for "This is a
|
|
15
|
-
configuration file").
|
|
16
|
-
- Rule of thumb:
|
|
17
|
-
- If the file follows a standard format like INI or YAML, consider using
|
|
18
|
-
dedicated parsers.
|
|
19
|
-
"""
|
|
20
|
-
|
|
21
|
-
from __future__ import annotations
|
|
22
|
-
|
|
23
|
-
from pathlib import Path
|
|
24
|
-
|
|
25
|
-
from ..types import JSONData
|
|
26
|
-
from ..types import JSONList
|
|
27
|
-
from . import stub
|
|
28
|
-
|
|
29
|
-
# SECTION: EXPORTS ========================================================== #
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
__all__ = [
|
|
33
|
-
'read',
|
|
34
|
-
'write',
|
|
35
|
-
]
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
# SECTION: FUNCTIONS ======================================================== #
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
def read(
|
|
42
|
-
path: Path,
|
|
43
|
-
) -> JSONList:
|
|
44
|
-
"""
|
|
45
|
-
Read CONF content from ``path``.
|
|
46
|
-
|
|
47
|
-
Parameters
|
|
48
|
-
----------
|
|
49
|
-
path : Path
|
|
50
|
-
Path to the CONF file on disk.
|
|
51
|
-
|
|
52
|
-
Returns
|
|
53
|
-
-------
|
|
54
|
-
JSONList
|
|
55
|
-
The list of dictionaries read from the CONF file.
|
|
56
|
-
"""
|
|
57
|
-
return stub.read(path, format_name='CONF')
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
def write(
|
|
61
|
-
path: Path,
|
|
62
|
-
data: JSONData,
|
|
63
|
-
) -> int:
|
|
64
|
-
"""
|
|
65
|
-
Write ``data`` to CONF at ``path`` and return record count.
|
|
66
|
-
|
|
67
|
-
Parameters
|
|
68
|
-
----------
|
|
69
|
-
path : Path
|
|
70
|
-
Path to the CONF file on disk.
|
|
71
|
-
data : JSONData
|
|
72
|
-
Data to write as CONF. Should be a list of dictionaries or a
|
|
73
|
-
single dictionary.
|
|
74
|
-
|
|
75
|
-
Returns
|
|
76
|
-
-------
|
|
77
|
-
int
|
|
78
|
-
The number of rows written to the CONF file.
|
|
79
|
-
"""
|
|
80
|
-
return stub.write(path, data, format_name='CONF')
|
etlplus/file/core.py
DELETED
|
@@ -1,322 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
:mod:`etlplus.file.core` module.
|
|
3
|
-
|
|
4
|
-
Shared helpers for reading and writing structured and semi-structured data
|
|
5
|
-
files.
|
|
6
|
-
"""
|
|
7
|
-
|
|
8
|
-
from __future__ import annotations
|
|
9
|
-
|
|
10
|
-
import importlib
|
|
11
|
-
import inspect
|
|
12
|
-
from dataclasses import dataclass
|
|
13
|
-
from functools import cache
|
|
14
|
-
from pathlib import Path
|
|
15
|
-
from types import ModuleType
|
|
16
|
-
|
|
17
|
-
from ..types import JSONData
|
|
18
|
-
from . import xml
|
|
19
|
-
from .enums import FileFormat
|
|
20
|
-
from .enums import infer_file_format_and_compression
|
|
21
|
-
|
|
22
|
-
# SECTION: EXPORTS ========================================================== #
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
__all__ = ['File']
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
# SECTION: INTERNAL FUNCTIONS =============================================== #
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
def _accepts_root_tag(handler: object) -> bool:
|
|
32
|
-
"""
|
|
33
|
-
Return True when ``handler`` supports a ``root_tag`` argument.
|
|
34
|
-
|
|
35
|
-
Parameters
|
|
36
|
-
----------
|
|
37
|
-
handler : object
|
|
38
|
-
Callable to inspect.
|
|
39
|
-
|
|
40
|
-
Returns
|
|
41
|
-
-------
|
|
42
|
-
bool
|
|
43
|
-
True if ``root_tag`` is accepted by the handler.
|
|
44
|
-
"""
|
|
45
|
-
if not callable(handler):
|
|
46
|
-
return False
|
|
47
|
-
try:
|
|
48
|
-
signature = inspect.signature(handler)
|
|
49
|
-
except (TypeError, ValueError):
|
|
50
|
-
return False
|
|
51
|
-
for param in signature.parameters.values():
|
|
52
|
-
if param.kind is param.VAR_KEYWORD:
|
|
53
|
-
return True
|
|
54
|
-
return 'root_tag' in signature.parameters
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
@cache
|
|
58
|
-
def _module_for_format(file_format: FileFormat) -> ModuleType:
|
|
59
|
-
"""
|
|
60
|
-
Import and return the module for ``file_format``.
|
|
61
|
-
|
|
62
|
-
Parameters
|
|
63
|
-
----------
|
|
64
|
-
file_format : FileFormat
|
|
65
|
-
File format enum value.
|
|
66
|
-
|
|
67
|
-
Returns
|
|
68
|
-
-------
|
|
69
|
-
ModuleType
|
|
70
|
-
The module implementing IO for the format.
|
|
71
|
-
"""
|
|
72
|
-
return importlib.import_module(f'{__package__}.{file_format.value}')
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
# SECTION: CLASSES ========================================================== #
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
@dataclass(slots=True)
|
|
79
|
-
class File:
|
|
80
|
-
"""
|
|
81
|
-
Convenience wrapper around structured file IO.
|
|
82
|
-
|
|
83
|
-
This class encapsulates the one-off helpers in this module as convenient
|
|
84
|
-
instance methods while retaining the original function API for
|
|
85
|
-
backward compatibility (those functions delegate to this class).
|
|
86
|
-
|
|
87
|
-
Attributes
|
|
88
|
-
----------
|
|
89
|
-
path : Path
|
|
90
|
-
Path to the file on disk.
|
|
91
|
-
file_format : FileFormat | None, optional
|
|
92
|
-
Explicit format. If omitted, the format is inferred from the file
|
|
93
|
-
extension (``.csv``, ``.json``, etc.).
|
|
94
|
-
|
|
95
|
-
Parameters
|
|
96
|
-
----------
|
|
97
|
-
path : StrPath
|
|
98
|
-
Path to the file on disk.
|
|
99
|
-
file_format : FileFormat | str | None, optional
|
|
100
|
-
Explicit format. If omitted, the format is inferred from the file
|
|
101
|
-
extension (``.csv``, ``.json``, etc.).
|
|
102
|
-
"""
|
|
103
|
-
|
|
104
|
-
# -- Attributes -- #
|
|
105
|
-
|
|
106
|
-
path: Path
|
|
107
|
-
file_format: FileFormat | None = None
|
|
108
|
-
|
|
109
|
-
# -- Magic Methods (Object Lifecycle) -- #
|
|
110
|
-
|
|
111
|
-
def __post_init__(self) -> None:
|
|
112
|
-
"""
|
|
113
|
-
Auto-detect and set the file format on initialization.
|
|
114
|
-
|
|
115
|
-
If no explicit ``file_format`` is provided, attempt to infer it from
|
|
116
|
-
the file path's extension and update :attr:`file_format`. If the
|
|
117
|
-
extension is unknown, the attribute is left as ``None`` and will be
|
|
118
|
-
validated later by :meth:`_ensure_format`.
|
|
119
|
-
"""
|
|
120
|
-
self.path = Path(self.path)
|
|
121
|
-
self.file_format = self._coerce_format(self.file_format)
|
|
122
|
-
if self.file_format is None:
|
|
123
|
-
self.file_format = self._maybe_guess_format()
|
|
124
|
-
|
|
125
|
-
# -- Internal Instance Methods -- #
|
|
126
|
-
|
|
127
|
-
def _assert_exists(self) -> None:
|
|
128
|
-
"""
|
|
129
|
-
Raise FileNotFoundError if :attr:`path` does not exist.
|
|
130
|
-
|
|
131
|
-
This centralizes existence checks across multiple read methods.
|
|
132
|
-
"""
|
|
133
|
-
if not self.path.exists():
|
|
134
|
-
raise FileNotFoundError(f'File not found: {self.path}')
|
|
135
|
-
|
|
136
|
-
def _coerce_format(
|
|
137
|
-
self,
|
|
138
|
-
file_format: FileFormat | str | None,
|
|
139
|
-
) -> FileFormat | None:
|
|
140
|
-
"""
|
|
141
|
-
Normalize the file format input.
|
|
142
|
-
|
|
143
|
-
Parameters
|
|
144
|
-
----------
|
|
145
|
-
file_format : FileFormat | str | None
|
|
146
|
-
File format specifier. Strings are coerced into
|
|
147
|
-
:class:`FileFormat`.
|
|
148
|
-
|
|
149
|
-
Returns
|
|
150
|
-
-------
|
|
151
|
-
FileFormat | None
|
|
152
|
-
A normalized file format, or ``None`` when unspecified.
|
|
153
|
-
"""
|
|
154
|
-
if file_format is None or isinstance(file_format, FileFormat):
|
|
155
|
-
return file_format
|
|
156
|
-
return FileFormat.coerce(file_format)
|
|
157
|
-
|
|
158
|
-
def _ensure_format(self) -> FileFormat:
|
|
159
|
-
"""
|
|
160
|
-
Resolve the active format, guessing from extension if needed.
|
|
161
|
-
|
|
162
|
-
Returns
|
|
163
|
-
-------
|
|
164
|
-
FileFormat
|
|
165
|
-
The resolved file format.
|
|
166
|
-
"""
|
|
167
|
-
return (
|
|
168
|
-
self.file_format
|
|
169
|
-
if self.file_format is not None
|
|
170
|
-
else self._guess_format()
|
|
171
|
-
)
|
|
172
|
-
|
|
173
|
-
def _guess_format(self) -> FileFormat:
|
|
174
|
-
"""
|
|
175
|
-
Infer the file format from the filename extension.
|
|
176
|
-
|
|
177
|
-
Returns
|
|
178
|
-
-------
|
|
179
|
-
FileFormat
|
|
180
|
-
The inferred file format based on the file extension.
|
|
181
|
-
|
|
182
|
-
Raises
|
|
183
|
-
------
|
|
184
|
-
ValueError
|
|
185
|
-
If the extension is unknown or unsupported.
|
|
186
|
-
"""
|
|
187
|
-
fmt, compression = infer_file_format_and_compression(self.path)
|
|
188
|
-
if fmt is not None:
|
|
189
|
-
return fmt
|
|
190
|
-
if compression is not None:
|
|
191
|
-
raise ValueError(
|
|
192
|
-
'Cannot infer file format from compressed file '
|
|
193
|
-
f'{self.path!r} with compression {compression.value!r}',
|
|
194
|
-
)
|
|
195
|
-
raise ValueError(
|
|
196
|
-
f'Cannot infer file format from extension {self.path.suffix!r}',
|
|
197
|
-
)
|
|
198
|
-
|
|
199
|
-
def _maybe_guess_format(self) -> FileFormat | None:
|
|
200
|
-
"""
|
|
201
|
-
Try to infer the format, returning ``None`` if it cannot be inferred.
|
|
202
|
-
|
|
203
|
-
Returns
|
|
204
|
-
-------
|
|
205
|
-
FileFormat | None
|
|
206
|
-
The inferred format, or ``None`` if inference fails.
|
|
207
|
-
"""
|
|
208
|
-
try:
|
|
209
|
-
return self._guess_format()
|
|
210
|
-
except ValueError:
|
|
211
|
-
# Leave as None; _ensure_format() will raise on use if needed.
|
|
212
|
-
return None
|
|
213
|
-
|
|
214
|
-
def _resolve_handler(self, name: str) -> object:
|
|
215
|
-
"""
|
|
216
|
-
Resolve a handler from the module for the active file format.
|
|
217
|
-
|
|
218
|
-
Parameters
|
|
219
|
-
----------
|
|
220
|
-
name : str
|
|
221
|
-
Attribute name to resolve (``'read'`` or ``'write'``).
|
|
222
|
-
|
|
223
|
-
Returns
|
|
224
|
-
-------
|
|
225
|
-
object
|
|
226
|
-
Callable handler exported by the module.
|
|
227
|
-
|
|
228
|
-
Raises
|
|
229
|
-
------
|
|
230
|
-
ValueError
|
|
231
|
-
If the resolved file format is unsupported.
|
|
232
|
-
"""
|
|
233
|
-
module = self._resolve_module()
|
|
234
|
-
try:
|
|
235
|
-
return getattr(module, name)
|
|
236
|
-
except AttributeError as e:
|
|
237
|
-
raise ValueError(
|
|
238
|
-
f'Module {module.__name__} does not implement {name}()',
|
|
239
|
-
) from e
|
|
240
|
-
|
|
241
|
-
def _resolve_module(self) -> ModuleType:
|
|
242
|
-
"""
|
|
243
|
-
Resolve the IO module for the active file format.
|
|
244
|
-
|
|
245
|
-
Returns
|
|
246
|
-
-------
|
|
247
|
-
ModuleType
|
|
248
|
-
The module that implements read/write for the format.
|
|
249
|
-
|
|
250
|
-
Raises
|
|
251
|
-
------
|
|
252
|
-
ValueError
|
|
253
|
-
If the resolved file format is unsupported.
|
|
254
|
-
"""
|
|
255
|
-
fmt = self._ensure_format()
|
|
256
|
-
try:
|
|
257
|
-
return _module_for_format(fmt)
|
|
258
|
-
except ModuleNotFoundError as e:
|
|
259
|
-
raise ValueError(f'Unsupported format: {fmt}') from e
|
|
260
|
-
|
|
261
|
-
# -- Instance Methods -- #
|
|
262
|
-
|
|
263
|
-
def read(self) -> JSONData:
|
|
264
|
-
"""
|
|
265
|
-
Read structured data from :attr:`path` using :attr:`file_format`.
|
|
266
|
-
|
|
267
|
-
Returns
|
|
268
|
-
-------
|
|
269
|
-
JSONData
|
|
270
|
-
The structured data read from the file.
|
|
271
|
-
|
|
272
|
-
Raises
|
|
273
|
-
------
|
|
274
|
-
TypeError
|
|
275
|
-
If the resolved 'read' handler is not callable.
|
|
276
|
-
"""
|
|
277
|
-
self._assert_exists()
|
|
278
|
-
reader = self._resolve_handler('read')
|
|
279
|
-
if callable(reader):
|
|
280
|
-
return reader(self.path)
|
|
281
|
-
else:
|
|
282
|
-
raise TypeError(
|
|
283
|
-
f"'read' handler for format {self.file_format} "
|
|
284
|
-
'is not callable',
|
|
285
|
-
)
|
|
286
|
-
|
|
287
|
-
def write(
|
|
288
|
-
self,
|
|
289
|
-
data: JSONData,
|
|
290
|
-
*,
|
|
291
|
-
root_tag: str = xml.DEFAULT_XML_ROOT,
|
|
292
|
-
) -> int:
|
|
293
|
-
"""
|
|
294
|
-
Write ``data`` to :attr:`path` using :attr:`file_format`.
|
|
295
|
-
|
|
296
|
-
Parameters
|
|
297
|
-
----------
|
|
298
|
-
data : JSONData
|
|
299
|
-
Data to write to the file.
|
|
300
|
-
root_tag : str, optional
|
|
301
|
-
Root tag name to use when writing XML files. Defaults to
|
|
302
|
-
``'root'``.
|
|
303
|
-
|
|
304
|
-
Returns
|
|
305
|
-
-------
|
|
306
|
-
int
|
|
307
|
-
The number of records written.
|
|
308
|
-
|
|
309
|
-
Raises
|
|
310
|
-
------
|
|
311
|
-
TypeError
|
|
312
|
-
If the resolved 'write' handler is not callable.
|
|
313
|
-
"""
|
|
314
|
-
writer = self._resolve_handler('write')
|
|
315
|
-
if not callable(writer):
|
|
316
|
-
raise TypeError(
|
|
317
|
-
f"'write' handler for format {self.file_format} "
|
|
318
|
-
'is not callable',
|
|
319
|
-
)
|
|
320
|
-
if _accepts_root_tag(writer):
|
|
321
|
-
return writer(self.path, data, root_tag=root_tag)
|
|
322
|
-
return writer(self.path, data)
|
etlplus/file/csv.py
DELETED
|
@@ -1,79 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
:mod:`etlplus.file.csv` module.
|
|
3
|
-
|
|
4
|
-
Helpers for reading/writing Comma-Separated Values (CSV) files.
|
|
5
|
-
|
|
6
|
-
Notes
|
|
7
|
-
-----
|
|
8
|
-
- A CSV file is a plain text file that uses commas to separate values.
|
|
9
|
-
- Common cases:
|
|
10
|
-
- Each line in the file represents a single record.
|
|
11
|
-
- The first line often contains headers that define the column names.
|
|
12
|
-
- Values may be enclosed in quotes, especially if they contain commas
|
|
13
|
-
or special characters.
|
|
14
|
-
- Rule of thumb:
|
|
15
|
-
- If the file follows the CSV specification, use this module for
|
|
16
|
-
reading and writing.
|
|
17
|
-
"""
|
|
18
|
-
|
|
19
|
-
from __future__ import annotations
|
|
20
|
-
|
|
21
|
-
from pathlib import Path
|
|
22
|
-
|
|
23
|
-
from ..types import JSONData
|
|
24
|
-
from ..types import JSONList
|
|
25
|
-
from ._io import read_delimited
|
|
26
|
-
from ._io import write_delimited
|
|
27
|
-
|
|
28
|
-
# SECTION: EXPORTS ========================================================== #
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
__all__ = [
|
|
32
|
-
'read',
|
|
33
|
-
'write',
|
|
34
|
-
]
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
# SECTION: FUNCTIONS ======================================================== #
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
def read(
|
|
41
|
-
path: Path,
|
|
42
|
-
) -> JSONList:
|
|
43
|
-
"""
|
|
44
|
-
Read CSV content from ``path``.
|
|
45
|
-
|
|
46
|
-
Parameters
|
|
47
|
-
----------
|
|
48
|
-
path : Path
|
|
49
|
-
Path to the CSV file on disk.
|
|
50
|
-
|
|
51
|
-
Returns
|
|
52
|
-
-------
|
|
53
|
-
JSONList
|
|
54
|
-
The list of dictionaries read from the CSV file.
|
|
55
|
-
"""
|
|
56
|
-
return read_delimited(path, delimiter=',')
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
def write(
|
|
60
|
-
path: Path,
|
|
61
|
-
data: JSONData,
|
|
62
|
-
) -> int:
|
|
63
|
-
"""
|
|
64
|
-
Write ``data`` to CSV at ``path`` and return record count.
|
|
65
|
-
|
|
66
|
-
Parameters
|
|
67
|
-
----------
|
|
68
|
-
path : Path
|
|
69
|
-
Path to the CSV file on disk.
|
|
70
|
-
data : JSONData
|
|
71
|
-
Data to write as CSV. Should be a list of dictionaries or a
|
|
72
|
-
single dictionary.
|
|
73
|
-
|
|
74
|
-
Returns
|
|
75
|
-
-------
|
|
76
|
-
int
|
|
77
|
-
The number of rows written to the CSV file.
|
|
78
|
-
"""
|
|
79
|
-
return write_delimited(path, data, delimiter=',')
|