etlplus 0.9.2__py3-none-any.whl → 0.10.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. etlplus/__init__.py +26 -1
  2. etlplus/api/README.md +3 -51
  3. etlplus/api/__init__.py +0 -10
  4. etlplus/api/config.py +28 -39
  5. etlplus/api/endpoint_client.py +3 -3
  6. etlplus/api/pagination/client.py +1 -1
  7. etlplus/api/rate_limiting/config.py +1 -13
  8. etlplus/api/rate_limiting/rate_limiter.py +11 -8
  9. etlplus/api/request_manager.py +6 -11
  10. etlplus/api/transport.py +2 -14
  11. etlplus/api/types.py +6 -96
  12. etlplus/cli/commands.py +43 -76
  13. etlplus/cli/constants.py +1 -1
  14. etlplus/cli/handlers.py +12 -40
  15. etlplus/cli/io.py +2 -2
  16. etlplus/cli/main.py +1 -1
  17. etlplus/cli/state.py +7 -4
  18. etlplus/{workflow → config}/__init__.py +23 -10
  19. etlplus/{workflow → config}/connector.py +44 -58
  20. etlplus/{workflow → config}/jobs.py +32 -105
  21. etlplus/{workflow → config}/pipeline.py +51 -59
  22. etlplus/{workflow → config}/profile.py +5 -8
  23. etlplus/config/types.py +204 -0
  24. etlplus/config/utils.py +120 -0
  25. etlplus/database/ddl.py +1 -1
  26. etlplus/database/engine.py +3 -19
  27. etlplus/database/orm.py +0 -2
  28. etlplus/database/schema.py +1 -1
  29. etlplus/enums.py +288 -0
  30. etlplus/{ops/extract.py → extract.py} +99 -81
  31. etlplus/file.py +652 -0
  32. etlplus/{ops/load.py → load.py} +101 -78
  33. etlplus/{ops/run.py → run.py} +127 -159
  34. etlplus/{api/utils.py → run_helpers.py} +153 -209
  35. etlplus/{ops/transform.py → transform.py} +68 -75
  36. etlplus/types.py +4 -5
  37. etlplus/utils.py +2 -136
  38. etlplus/{ops/validate.py → validate.py} +12 -22
  39. etlplus/validation/__init__.py +44 -0
  40. etlplus/{ops → validation}/utils.py +17 -53
  41. {etlplus-0.9.2.dist-info → etlplus-0.10.2.dist-info}/METADATA +17 -210
  42. etlplus-0.10.2.dist-info/RECORD +65 -0
  43. {etlplus-0.9.2.dist-info → etlplus-0.10.2.dist-info}/WHEEL +1 -1
  44. etlplus/README.md +0 -37
  45. etlplus/api/enums.py +0 -51
  46. etlplus/cli/README.md +0 -40
  47. etlplus/database/README.md +0 -48
  48. etlplus/file/README.md +0 -105
  49. etlplus/file/__init__.py +0 -25
  50. etlplus/file/_imports.py +0 -141
  51. etlplus/file/_io.py +0 -160
  52. etlplus/file/accdb.py +0 -78
  53. etlplus/file/arrow.py +0 -78
  54. etlplus/file/avro.py +0 -176
  55. etlplus/file/bson.py +0 -77
  56. etlplus/file/cbor.py +0 -78
  57. etlplus/file/cfg.py +0 -79
  58. etlplus/file/conf.py +0 -80
  59. etlplus/file/core.py +0 -322
  60. etlplus/file/csv.py +0 -79
  61. etlplus/file/dat.py +0 -78
  62. etlplus/file/dta.py +0 -77
  63. etlplus/file/duckdb.py +0 -78
  64. etlplus/file/enums.py +0 -343
  65. etlplus/file/feather.py +0 -111
  66. etlplus/file/fwf.py +0 -77
  67. etlplus/file/gz.py +0 -123
  68. etlplus/file/hbs.py +0 -78
  69. etlplus/file/hdf5.py +0 -78
  70. etlplus/file/ini.py +0 -79
  71. etlplus/file/ion.py +0 -78
  72. etlplus/file/jinja2.py +0 -78
  73. etlplus/file/json.py +0 -98
  74. etlplus/file/log.py +0 -78
  75. etlplus/file/mat.py +0 -78
  76. etlplus/file/mdb.py +0 -78
  77. etlplus/file/msgpack.py +0 -78
  78. etlplus/file/mustache.py +0 -78
  79. etlplus/file/nc.py +0 -78
  80. etlplus/file/ndjson.py +0 -108
  81. etlplus/file/numbers.py +0 -75
  82. etlplus/file/ods.py +0 -79
  83. etlplus/file/orc.py +0 -111
  84. etlplus/file/parquet.py +0 -113
  85. etlplus/file/pb.py +0 -78
  86. etlplus/file/pbf.py +0 -77
  87. etlplus/file/properties.py +0 -78
  88. etlplus/file/proto.py +0 -77
  89. etlplus/file/psv.py +0 -79
  90. etlplus/file/rda.py +0 -78
  91. etlplus/file/rds.py +0 -78
  92. etlplus/file/sas7bdat.py +0 -78
  93. etlplus/file/sav.py +0 -77
  94. etlplus/file/sqlite.py +0 -78
  95. etlplus/file/stub.py +0 -84
  96. etlplus/file/sylk.py +0 -77
  97. etlplus/file/tab.py +0 -81
  98. etlplus/file/toml.py +0 -78
  99. etlplus/file/tsv.py +0 -80
  100. etlplus/file/txt.py +0 -102
  101. etlplus/file/vm.py +0 -78
  102. etlplus/file/wks.py +0 -77
  103. etlplus/file/xls.py +0 -88
  104. etlplus/file/xlsm.py +0 -79
  105. etlplus/file/xlsx.py +0 -99
  106. etlplus/file/xml.py +0 -185
  107. etlplus/file/xpt.py +0 -78
  108. etlplus/file/yaml.py +0 -95
  109. etlplus/file/zip.py +0 -175
  110. etlplus/file/zsav.py +0 -77
  111. etlplus/ops/README.md +0 -50
  112. etlplus/ops/__init__.py +0 -61
  113. etlplus/templates/README.md +0 -46
  114. etlplus/workflow/README.md +0 -52
  115. etlplus/workflow/dag.py +0 -105
  116. etlplus/workflow/types.py +0 -115
  117. etlplus-0.9.2.dist-info/RECORD +0 -134
  118. {etlplus-0.9.2.dist-info → etlplus-0.10.2.dist-info}/entry_points.txt +0 -0
  119. {etlplus-0.9.2.dist-info → etlplus-0.10.2.dist-info}/licenses/LICENSE +0 -0
  120. {etlplus-0.9.2.dist-info → etlplus-0.10.2.dist-info}/top_level.txt +0 -0
etlplus/file/log.py DELETED
@@ -1,78 +0,0 @@
1
- """
2
- :mod:`etlplus.file.log` module.
3
-
4
- Helpers for reading/writing generic log (LOG) files.
5
-
6
- Notes
7
- -----
8
- - A LOG file is a plain text file that contains log messages generated by
9
- applications or systems.
10
- - Common cases:
11
- - Each line in the file represents a single log entry.
12
- - Log entries may include timestamps, log levels, and messages.
13
- - Formats may vary widely depending on the application generating the logs.
14
- - Rule of thumb:
15
- - If the file is a generic log file, use this module for reading and
16
- writing.
17
- """
18
-
19
- from __future__ import annotations
20
-
21
- from pathlib import Path
22
-
23
- from ..types import JSONData
24
- from ..types import JSONList
25
- from . import stub
26
-
27
- # SECTION: EXPORTS ========================================================== #
28
-
29
-
30
- __all__ = [
31
- 'read',
32
- 'write',
33
- ]
34
-
35
-
36
- # SECTION: FUNCTIONS ======================================================== #
37
-
38
-
39
- def read(
40
- path: Path,
41
- ) -> JSONList:
42
- """
43
- Read LOG content from ``path``.
44
-
45
- Parameters
46
- ----------
47
- path : Path
48
- Path to the LOG file on disk.
49
-
50
- Returns
51
- -------
52
- JSONList
53
- The list of dictionaries read from the LOG file.
54
- """
55
- return stub.read(path, format_name='LOG')
56
-
57
-
58
- def write(
59
- path: Path,
60
- data: JSONData,
61
- ) -> int:
62
- """
63
- Write ``data`` to LOG at ``path`` and return record count.
64
-
65
- Parameters
66
- ----------
67
- path : Path
68
- Path to the LOG file on disk.
69
- data : JSONData
70
- Data to write as LOG. Should be a list of dictionaries or a
71
- single dictionary.
72
-
73
- Returns
74
- -------
75
- int
76
- The number of rows written to the LOG file.
77
- """
78
- return stub.write(path, data, format_name='LOG')
etlplus/file/mat.py DELETED
@@ -1,78 +0,0 @@
1
- """
2
- :mod:`etlplus.file.mat` module.
3
-
4
- Helpers for reading/writing MATLAB (MAT) data files.
5
-
6
- Notes
7
- -----
8
- - A MAT file is a binary file format used by MATLAB to store variables,
9
- arrays, and other data structures.
10
- - Common cases:
11
- - Storing numerical arrays and matrices.
12
- - Saving workspace variables.
13
- - Sharing data between MATLAB and other programming environments.
14
- - Rule of thumb:
15
- - If the file follows the MAT-file specification, use this module for
16
- reading and writing.
17
- """
18
-
19
- from __future__ import annotations
20
-
21
- from pathlib import Path
22
-
23
- from ..types import JSONData
24
- from ..types import JSONList
25
- from . import stub
26
-
27
- # SECTION: EXPORTS ========================================================== #
28
-
29
-
30
- __all__ = [
31
- 'read',
32
- 'write',
33
- ]
34
-
35
-
36
- # SECTION: FUNCTIONS ======================================================== #
37
-
38
-
39
- def read(
40
- path: Path,
41
- ) -> JSONList:
42
- """
43
- Read MAT content from ``path``.
44
-
45
- Parameters
46
- ----------
47
- path : Path
48
- Path to the MAT file on disk.
49
-
50
- Returns
51
- -------
52
- JSONList
53
- The list of dictionaries read from the MAT file.
54
- """
55
- return stub.read(path, format_name='MAT')
56
-
57
-
58
- def write(
59
- path: Path,
60
- data: JSONData,
61
- ) -> int:
62
- """
63
- Write ``data`` to MAT file at ``path`` and return record count.
64
-
65
- Parameters
66
- ----------
67
- path : Path
68
- Path to the MAT file on disk.
69
- data : JSONData
70
- Data to write as MAT file. Should be a list of dictionaries or a
71
- single dictionary.
72
-
73
- Returns
74
- -------
75
- int
76
- The number of rows written to the MAT file.
77
- """
78
- return stub.write(path, data, format_name='MAT')
etlplus/file/mdb.py DELETED
@@ -1,78 +0,0 @@
1
- """
2
- :mod:`etlplus.file.mdb` module.
3
-
4
- Helpers for reading/writing newer Microsoft Access database (MDB) files.
5
-
6
- Notes
7
- -----
8
- - An MDB file is a proprietary database file format used by Microsoft Access
9
- 2003 and earlier.
10
- - Common cases:
11
- - Storing relational data for small to medium-sized applications.
12
- - Desktop database applications.
13
- - Data management for non-enterprise solutions.
14
- - Rule of thumb:
15
- - If the file follows the MDB specification, use this module for reading
16
- and writing.
17
- """
18
-
19
- from __future__ import annotations
20
-
21
- from pathlib import Path
22
-
23
- from ..types import JSONData
24
- from ..types import JSONList
25
- from . import stub
26
-
27
- # SECTION: EXPORTS ========================================================== #
28
-
29
-
30
- __all__ = [
31
- 'read',
32
- 'write',
33
- ]
34
-
35
-
36
- # SECTION: FUNCTIONS ======================================================== #
37
-
38
-
39
- def read(
40
- path: Path,
41
- ) -> JSONList:
42
- """
43
- Read CSV content from ``path``.
44
-
45
- Parameters
46
- ----------
47
- path : Path
48
- Path to the CSV file on disk.
49
-
50
- Returns
51
- -------
52
- JSONList
53
- The list of dictionaries read from the CSV file.
54
- """
55
- return stub.read(path, format_name='DAT')
56
-
57
-
58
- def write(
59
- path: Path,
60
- data: JSONData,
61
- ) -> int:
62
- """
63
- Write ``data`` to CSV at ``path`` and return record count.
64
-
65
- Parameters
66
- ----------
67
- path : Path
68
- Path to the CSV file on disk.
69
- data : JSONData
70
- Data to write as CSV. Should be a list of dictionaries or a
71
- single dictionary.
72
-
73
- Returns
74
- -------
75
- int
76
- The number of rows written to the CSV file.
77
- """
78
- return stub.write(path, data, format_name='DAT')
etlplus/file/msgpack.py DELETED
@@ -1,78 +0,0 @@
1
- """
2
- :mod:`etlplus.file.msgpack` module.
3
-
4
- Helpers for reading/writing MessagePack (MSGPACK) files.
5
-
6
- Notes
7
- -----
8
- - A MsgPack file is a binary serialization format that is more compact than
9
- JSON.
10
- - Common cases:
11
- - Efficient data storage and transmission.
12
- - Inter-process communication.
13
- - Data serialization in performance-critical applications.
14
- - Rule of thumb:
15
- - If the file follows the MsgPack specification, use this module for
16
- reading and writing.
17
- """
18
-
19
- from __future__ import annotations
20
-
21
- from pathlib import Path
22
-
23
- from ..types import JSONData
24
- from ..types import JSONList
25
- from . import stub
26
-
27
- # SECTION: EXPORTS ========================================================== #
28
-
29
-
30
- __all__ = [
31
- 'read',
32
- 'write',
33
- ]
34
-
35
-
36
- # SECTION: FUNCTIONS ======================================================== #
37
-
38
-
39
- def read(
40
- path: Path,
41
- ) -> JSONList:
42
- """
43
- Read MsgPack content from ``path``.
44
-
45
- Parameters
46
- ----------
47
- path : Path
48
- Path to the MsgPack file on disk.
49
-
50
- Returns
51
- -------
52
- JSONList
53
- The list of dictionaries read from the MsgPack file.
54
- """
55
- return stub.read(path, format_name='MSGPACK')
56
-
57
-
58
- def write(
59
- path: Path,
60
- data: JSONData,
61
- ) -> int:
62
- """
63
- Write ``data`` to MsgPack at ``path`` and return record count.
64
-
65
- Parameters
66
- ----------
67
- path : Path
68
- Path to the MsgPack file on disk.
69
- data : JSONData
70
- Data to write as MsgPack. Should be a list of dictionaries or a
71
- single dictionary.
72
-
73
- Returns
74
- -------
75
- int
76
- The number of rows written to the MsgPack file.
77
- """
78
- return stub.write(path, data, format_name='MSGPACK')
etlplus/file/mustache.py DELETED
@@ -1,78 +0,0 @@
1
- """
2
- :mod:`etlplus.file.mustache` module.
3
-
4
- Helpers for reading/writing Mustache (MUSTACHE) template files.
5
-
6
- Notes
7
- -----
8
- - A MUSTACHE file is a text file used for generating HTML or other text formats
9
- by combining templates with data.
10
- - Common cases:
11
- - HTML templates.
12
- - Email templates.
13
- - Configuration files.
14
- - Rule of thumb:
15
- - If you need to work with Mustache template files, use this module for
16
- reading and writing.
17
- """
18
-
19
- from __future__ import annotations
20
-
21
- from pathlib import Path
22
-
23
- from ..types import JSONData
24
- from ..types import JSONList
25
- from . import stub
26
-
27
- # SECTION: EXPORTS ========================================================== #
28
-
29
-
30
- __all__ = [
31
- 'read',
32
- 'write',
33
- ]
34
-
35
-
36
- # SECTION: FUNCTIONS ======================================================== #
37
-
38
-
39
- def read(
40
- path: Path,
41
- ) -> JSONList:
42
- """
43
- Read MUSTACHE content from ``path``.
44
-
45
- Parameters
46
- ----------
47
- path : Path
48
- Path to the MUSTACHE file on disk.
49
-
50
- Returns
51
- -------
52
- JSONList
53
- The list of dictionaries read from the MUSTACHE file.
54
- """
55
- return stub.read(path, format_name='MUSTACHE')
56
-
57
-
58
- def write(
59
- path: Path,
60
- data: JSONData,
61
- ) -> int:
62
- """
63
- Write ``data`` to MUSTACHE file at ``path`` and return record count.
64
-
65
- Parameters
66
- ----------
67
- path : Path
68
- Path to the MUSTACHE file on disk.
69
- data : JSONData
70
- Data to write as MUSTACHE file. Should be a list of dictionaries or a
71
- single dictionary.
72
-
73
- Returns
74
- -------
75
- int
76
- The number of rows written to the MUSTACHE file.
77
- """
78
- return stub.write(path, data, format_name='MUSTACHE')
etlplus/file/nc.py DELETED
@@ -1,78 +0,0 @@
1
- """
2
- :mod:`etlplus.file.nc` module.
3
-
4
- Helpers for reading/writing NetCDF (NC) data files.
5
-
6
- Notes
7
- -----
8
- - A NC file is a binary file format used for array-oriented scientific data,
9
- particularly in meteorology, oceanography, and climate science.
10
- - Common cases:
11
- - Storing multi-dimensional scientific data.
12
- - Sharing large datasets in research communities.
13
- - Efficient data access and manipulation.
14
- - Rule of thumb:
15
- - If the file follows the NetCDF standard, use this module for
16
- reading and writing.
17
- """
18
-
19
- from __future__ import annotations
20
-
21
- from pathlib import Path
22
-
23
- from ..types import JSONData
24
- from ..types import JSONList
25
- from . import stub
26
-
27
- # SECTION: EXPORTS ========================================================== #
28
-
29
-
30
- __all__ = [
31
- 'read',
32
- 'write',
33
- ]
34
-
35
-
36
- # SECTION: FUNCTIONS ======================================================== #
37
-
38
-
39
- def read(
40
- path: Path,
41
- ) -> JSONList:
42
- """
43
- Read NC content from ``path``.
44
-
45
- Parameters
46
- ----------
47
- path : Path
48
- Path to the NC file on disk.
49
-
50
- Returns
51
- -------
52
- JSONList
53
- The list of dictionaries read from the NC file.
54
- """
55
- return stub.read(path, format_name='NC')
56
-
57
-
58
- def write(
59
- path: Path,
60
- data: JSONData,
61
- ) -> int:
62
- """
63
- Write ``data`` to NC file at ``path`` and return record count.
64
-
65
- Parameters
66
- ----------
67
- path : Path
68
- Path to the NC file on disk.
69
- data : JSONData
70
- Data to write as NC file. Should be a list of dictionaries or a
71
- single dictionary.
72
-
73
- Returns
74
- -------
75
- int
76
- The number of rows written to the NC file.
77
- """
78
- return stub.write(path, data, format_name='NC')
etlplus/file/ndjson.py DELETED
@@ -1,108 +0,0 @@
1
- """
2
- :mod:`etlplus.file.ndjson` module.
3
-
4
- Helpers for reading/writing Newline Delimited JSON (NDJSON) files.
5
-
6
- Notes
7
- -----
8
- - An NDJSON file is a format where each line is a separate JSON object.
9
- - Common cases:
10
- - Streaming JSON data.
11
- - Log files with JSON entries.
12
- - Large datasets that are processed line-by-line.
13
- - Rule of thumb:
14
- - If the file follows the NDJSON specification, use this module for
15
- reading and writing.
16
- """
17
-
18
- from __future__ import annotations
19
-
20
- import json
21
- from pathlib import Path
22
- from typing import cast
23
-
24
- from ..types import JSONData
25
- from ..types import JSONDict
26
- from ..types import JSONList
27
- from ..utils import count_records
28
- from ._io import normalize_records
29
-
30
- # SECTION: EXPORTS ========================================================== #
31
-
32
-
33
- __all__ = [
34
- 'read',
35
- 'write',
36
- ]
37
-
38
-
39
- # SECTION: FUNCTIONS ======================================================== #
40
-
41
-
42
- def read(
43
- path: Path,
44
- ) -> JSONList:
45
- """
46
- Read NDJSON content from ``path``.
47
-
48
- Parameters
49
- ----------
50
- path : Path
51
- Path to the NDJSON file on disk.
52
-
53
- Returns
54
- -------
55
- JSONList
56
- The list of dictionaries read from the NDJSON file.
57
-
58
- Raises
59
- ------
60
- TypeError
61
- If any line in the NDJSON file is not a JSON object (dict).
62
- """
63
- rows: JSONList = []
64
- with path.open('r', encoding='utf-8') as handle:
65
- for idx, line in enumerate(handle, start=1):
66
- text = line.strip()
67
- if not text:
68
- continue
69
- payload = json.loads(text)
70
- if not isinstance(payload, dict):
71
- raise TypeError(
72
- f'NDJSON lines must be objects (dicts) (line {idx})',
73
- )
74
- rows.append(cast(JSONDict, payload))
75
- return rows
76
-
77
-
78
- def write(
79
- path: Path,
80
- data: JSONData,
81
- ) -> int:
82
- """
83
- Write ``data`` to NDJSON at ``path``.
84
-
85
- Parameters
86
- ----------
87
- path : Path
88
- Path to the NDJSON file on disk.
89
- data : JSONData
90
- Data to write.
91
-
92
- Returns
93
- -------
94
- int
95
- Number of records written.
96
- """
97
- rows = normalize_records(data, 'NDJSON')
98
-
99
- if not rows:
100
- return 0
101
-
102
- path.parent.mkdir(parents=True, exist_ok=True)
103
- with path.open('w', encoding='utf-8') as handle:
104
- for row in rows:
105
- handle.write(json.dumps(row, ensure_ascii=False))
106
- handle.write('\n')
107
-
108
- return count_records(rows)
etlplus/file/numbers.py DELETED
@@ -1,75 +0,0 @@
1
- """
2
- :mod:`etlplus.file.numbers` module.
3
-
4
- Helpers for reading/writing Apple Numbers (NUMBERS) spreadsheet files.
5
-
6
- Notes
7
- -----
8
- - A NUMBERS file is a spreadsheet file created by Apple Numbers.
9
- - Common cases:
10
- - Spreadsheet files created by Apple Numbers.
11
- - Rule of thumb:
12
- - If you need to read/write NUMBERS files, consider converting them to
13
- more common formats like CSV or XLSX for better compatibility.
14
- """
15
-
16
- from __future__ import annotations
17
-
18
- from pathlib import Path
19
-
20
- from ..types import JSONData
21
- from ..types import JSONList
22
- from . import stub
23
-
24
- # SECTION: EXPORTS ========================================================== #
25
-
26
-
27
- __all__ = [
28
- 'read',
29
- 'write',
30
- ]
31
-
32
-
33
- # SECTION: FUNCTIONS ======================================================== #
34
-
35
-
36
- def read(
37
- path: Path,
38
- ) -> JSONList:
39
- """
40
- Read NUMBERS content from ``path``.
41
-
42
- Parameters
43
- ----------
44
- path : Path
45
- Path to the NUMBERS file on disk.
46
-
47
- Returns
48
- -------
49
- JSONList
50
- The list of dictionaries read from the NUMBERS file.
51
- """
52
- return stub.read(path, format_name='NUMBERS')
53
-
54
-
55
- def write(
56
- path: Path,
57
- data: JSONData,
58
- ) -> int:
59
- """
60
- Write ``data`` to NUMBERS file at ``path`` and return record count.
61
-
62
- Parameters
63
- ----------
64
- path : Path
65
- Path to the NUMBERS file on disk.
66
- data : JSONData
67
- Data to write as NUMBERS file. Should be a list of dictionaries or a
68
- single dictionary.
69
-
70
- Returns
71
- -------
72
- int
73
- The number of rows written to the NUMBERS file.
74
- """
75
- return stub.write(path, data, format_name='NUMBERS')