etlplus 0.9.2__py3-none-any.whl → 0.10.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. etlplus/__init__.py +26 -1
  2. etlplus/api/README.md +3 -51
  3. etlplus/api/__init__.py +0 -10
  4. etlplus/api/config.py +28 -39
  5. etlplus/api/endpoint_client.py +3 -3
  6. etlplus/api/pagination/client.py +1 -1
  7. etlplus/api/rate_limiting/config.py +1 -13
  8. etlplus/api/rate_limiting/rate_limiter.py +11 -8
  9. etlplus/api/request_manager.py +6 -11
  10. etlplus/api/transport.py +2 -14
  11. etlplus/api/types.py +6 -96
  12. etlplus/cli/commands.py +43 -76
  13. etlplus/cli/constants.py +1 -1
  14. etlplus/cli/handlers.py +12 -40
  15. etlplus/cli/io.py +2 -2
  16. etlplus/cli/main.py +1 -1
  17. etlplus/cli/state.py +7 -4
  18. etlplus/{workflow → config}/__init__.py +23 -10
  19. etlplus/{workflow → config}/connector.py +44 -58
  20. etlplus/{workflow → config}/jobs.py +32 -105
  21. etlplus/{workflow → config}/pipeline.py +51 -59
  22. etlplus/{workflow → config}/profile.py +5 -8
  23. etlplus/config/types.py +204 -0
  24. etlplus/config/utils.py +120 -0
  25. etlplus/database/ddl.py +1 -1
  26. etlplus/database/engine.py +3 -19
  27. etlplus/database/orm.py +0 -2
  28. etlplus/database/schema.py +1 -1
  29. etlplus/enums.py +288 -0
  30. etlplus/{ops/extract.py → extract.py} +99 -81
  31. etlplus/file.py +652 -0
  32. etlplus/{ops/load.py → load.py} +101 -78
  33. etlplus/{ops/run.py → run.py} +127 -159
  34. etlplus/{api/utils.py → run_helpers.py} +153 -209
  35. etlplus/{ops/transform.py → transform.py} +68 -75
  36. etlplus/types.py +4 -5
  37. etlplus/utils.py +2 -136
  38. etlplus/{ops/validate.py → validate.py} +12 -22
  39. etlplus/validation/__init__.py +44 -0
  40. etlplus/{ops → validation}/utils.py +17 -53
  41. {etlplus-0.9.2.dist-info → etlplus-0.10.2.dist-info}/METADATA +17 -210
  42. etlplus-0.10.2.dist-info/RECORD +65 -0
  43. {etlplus-0.9.2.dist-info → etlplus-0.10.2.dist-info}/WHEEL +1 -1
  44. etlplus/README.md +0 -37
  45. etlplus/api/enums.py +0 -51
  46. etlplus/cli/README.md +0 -40
  47. etlplus/database/README.md +0 -48
  48. etlplus/file/README.md +0 -105
  49. etlplus/file/__init__.py +0 -25
  50. etlplus/file/_imports.py +0 -141
  51. etlplus/file/_io.py +0 -160
  52. etlplus/file/accdb.py +0 -78
  53. etlplus/file/arrow.py +0 -78
  54. etlplus/file/avro.py +0 -176
  55. etlplus/file/bson.py +0 -77
  56. etlplus/file/cbor.py +0 -78
  57. etlplus/file/cfg.py +0 -79
  58. etlplus/file/conf.py +0 -80
  59. etlplus/file/core.py +0 -322
  60. etlplus/file/csv.py +0 -79
  61. etlplus/file/dat.py +0 -78
  62. etlplus/file/dta.py +0 -77
  63. etlplus/file/duckdb.py +0 -78
  64. etlplus/file/enums.py +0 -343
  65. etlplus/file/feather.py +0 -111
  66. etlplus/file/fwf.py +0 -77
  67. etlplus/file/gz.py +0 -123
  68. etlplus/file/hbs.py +0 -78
  69. etlplus/file/hdf5.py +0 -78
  70. etlplus/file/ini.py +0 -79
  71. etlplus/file/ion.py +0 -78
  72. etlplus/file/jinja2.py +0 -78
  73. etlplus/file/json.py +0 -98
  74. etlplus/file/log.py +0 -78
  75. etlplus/file/mat.py +0 -78
  76. etlplus/file/mdb.py +0 -78
  77. etlplus/file/msgpack.py +0 -78
  78. etlplus/file/mustache.py +0 -78
  79. etlplus/file/nc.py +0 -78
  80. etlplus/file/ndjson.py +0 -108
  81. etlplus/file/numbers.py +0 -75
  82. etlplus/file/ods.py +0 -79
  83. etlplus/file/orc.py +0 -111
  84. etlplus/file/parquet.py +0 -113
  85. etlplus/file/pb.py +0 -78
  86. etlplus/file/pbf.py +0 -77
  87. etlplus/file/properties.py +0 -78
  88. etlplus/file/proto.py +0 -77
  89. etlplus/file/psv.py +0 -79
  90. etlplus/file/rda.py +0 -78
  91. etlplus/file/rds.py +0 -78
  92. etlplus/file/sas7bdat.py +0 -78
  93. etlplus/file/sav.py +0 -77
  94. etlplus/file/sqlite.py +0 -78
  95. etlplus/file/stub.py +0 -84
  96. etlplus/file/sylk.py +0 -77
  97. etlplus/file/tab.py +0 -81
  98. etlplus/file/toml.py +0 -78
  99. etlplus/file/tsv.py +0 -80
  100. etlplus/file/txt.py +0 -102
  101. etlplus/file/vm.py +0 -78
  102. etlplus/file/wks.py +0 -77
  103. etlplus/file/xls.py +0 -88
  104. etlplus/file/xlsm.py +0 -79
  105. etlplus/file/xlsx.py +0 -99
  106. etlplus/file/xml.py +0 -185
  107. etlplus/file/xpt.py +0 -78
  108. etlplus/file/yaml.py +0 -95
  109. etlplus/file/zip.py +0 -175
  110. etlplus/file/zsav.py +0 -77
  111. etlplus/ops/README.md +0 -50
  112. etlplus/ops/__init__.py +0 -61
  113. etlplus/templates/README.md +0 -46
  114. etlplus/workflow/README.md +0 -52
  115. etlplus/workflow/dag.py +0 -105
  116. etlplus/workflow/types.py +0 -115
  117. etlplus-0.9.2.dist-info/RECORD +0 -134
  118. {etlplus-0.9.2.dist-info → etlplus-0.10.2.dist-info}/entry_points.txt +0 -0
  119. {etlplus-0.9.2.dist-info → etlplus-0.10.2.dist-info}/licenses/LICENSE +0 -0
  120. {etlplus-0.9.2.dist-info → etlplus-0.10.2.dist-info}/top_level.txt +0 -0
etlplus/file/tsv.py DELETED
@@ -1,80 +0,0 @@
1
- """
2
- :mod:`etlplus.file.tsv` module.
3
-
4
- Helpers for reading/writing Tab-Separated Values (TSV) files.
5
-
6
- Notes
7
- -----
8
- - A TSV file is a plain text file that uses the tab character (``\t``) to
9
- separate values.
10
- - Common cases:
11
- - Each line in the file represents a single record.
12
- - The first line often contains headers that define the column names.
13
- - Values may be enclosed in quotes, especially if they contain tabs
14
- or special characters.
15
- - Rule of thumb:
16
- - If the file follows the TSV specification, use this module for
17
- reading and writing.
18
- """
19
-
20
- from __future__ import annotations
21
-
22
- from pathlib import Path
23
-
24
- from ..types import JSONData
25
- from ..types import JSONList
26
- from ._io import read_delimited
27
- from ._io import write_delimited
28
-
29
- # SECTION: EXPORTS ========================================================== #
30
-
31
-
32
- __all__ = [
33
- 'read',
34
- 'write',
35
- ]
36
-
37
-
38
- # SECTION: FUNCTIONS ======================================================== #
39
-
40
-
41
- def read(
42
- path: Path,
43
- ) -> JSONList:
44
- """
45
- Read TSV content from ``path``.
46
-
47
- Parameters
48
- ----------
49
- path : Path
50
- Path to the TSV file on disk.
51
-
52
- Returns
53
- -------
54
- JSONList
55
- The list of dictionaries read from the TSV file.
56
- """
57
- return read_delimited(path, delimiter='\t')
58
-
59
-
60
- def write(
61
- path: Path,
62
- data: JSONData,
63
- ) -> int:
64
- """
65
- Write ``data`` to TSV at ``path`` and return record count.
66
-
67
- Parameters
68
- ----------
69
- path : Path
70
- Path to the TSV file on disk.
71
- data : JSONData
72
- Data to write as TSV. Should be a list of dictionaries or a
73
- single dictionary.
74
-
75
- Returns
76
- -------
77
- int
78
- The number of rows written to the TSV file.
79
- """
80
- return write_delimited(path, data, delimiter='\t')
etlplus/file/txt.py DELETED
@@ -1,102 +0,0 @@
1
- """
2
- :mod:`etlplus.file.txt` module.
3
-
4
- Helpers for reading/writing text (TXT) files.
5
-
6
- Notes
7
- -----
8
- - A TXT file is a plain text file that contains unformatted text.
9
- - Common cases:
10
- - Each line in the file represents a single piece of text.
11
- - Lines may vary in length and content.
12
- - Rule of thumb:
13
- - If the file is a simple text file without specific formatting
14
- requirements, use this module for reading and writing.
15
- """
16
-
17
- from __future__ import annotations
18
-
19
- from pathlib import Path
20
-
21
- from ..types import JSONData
22
- from ..types import JSONList
23
- from ..utils import count_records
24
- from ._io import normalize_records
25
-
26
- # SECTION: EXPORTS ========================================================== #
27
-
28
-
29
- __all__ = [
30
- 'read',
31
- 'write',
32
- ]
33
-
34
-
35
- # SECTION: FUNCTIONS ======================================================== #
36
-
37
-
38
- def read(
39
- path: Path,
40
- ) -> JSONList:
41
- """
42
- Read TXT content from ``path``.
43
-
44
- Parameters
45
- ----------
46
- path : Path
47
- Path to the TXT file on disk.
48
-
49
- Returns
50
- -------
51
- JSONList
52
- The list of dictionaries read from the TXT file.
53
- """
54
- rows: JSONList = []
55
- with path.open('r', encoding='utf-8') as handle:
56
- for line in handle:
57
- text = line.rstrip('\n')
58
- if text == '':
59
- continue
60
- rows.append({'text': text})
61
- return rows
62
-
63
-
64
- def write(
65
- path: Path,
66
- data: JSONData,
67
- ) -> int:
68
- """
69
- Write ``data`` to TXT at ``path`` and return record count.
70
-
71
- Parameters
72
- ----------
73
- path : Path
74
- Path to the TXT file on disk.
75
- data : JSONData
76
- Data to write. Expects ``{'text': '...'} `` or a list of those.
77
-
78
- Returns
79
- -------
80
- int
81
- Number of records written.
82
-
83
- Raises
84
- ------
85
- TypeError
86
- If any item in ``data`` is not a dictionary or if any dictionary
87
- does not contain a ``'text'`` key.
88
- """
89
- rows = normalize_records(data, 'TXT')
90
-
91
- if not rows:
92
- return 0
93
-
94
- path.parent.mkdir(parents=True, exist_ok=True)
95
- with path.open('w', encoding='utf-8') as handle:
96
- for row in rows:
97
- if 'text' not in row:
98
- raise TypeError('TXT payloads must include a "text" key')
99
- handle.write(str(row['text']))
100
- handle.write('\n')
101
-
102
- return count_records(rows)
etlplus/file/vm.py DELETED
@@ -1,78 +0,0 @@
1
- """
2
- :mod:`etlplus.file.vm` module.
3
-
4
- Helpers for reading/writing Apache Velocity (VM) template files.
5
-
6
- Notes
7
- -----
8
- - A VM file is a text file used for generating HTML or other text formats
9
- by combining templates with data.
10
- - Common cases:
11
- - HTML templates.
12
- - Email templates.
13
- - Configuration files.
14
- - Rule of thumb:
15
- - If you need to work with Apache Velocity template files, use this module
16
- for reading and writing.
17
- """
18
-
19
- from __future__ import annotations
20
-
21
- from pathlib import Path
22
-
23
- from ..types import JSONData
24
- from ..types import JSONList
25
- from . import stub
26
-
27
- # SECTION: EXPORTS ========================================================== #
28
-
29
-
30
- __all__ = [
31
- 'read',
32
- 'write',
33
- ]
34
-
35
-
36
- # SECTION: FUNCTIONS ======================================================== #
37
-
38
-
39
- def read(
40
- path: Path,
41
- ) -> JSONList:
42
- """
43
- Read VM content from ``path``.
44
-
45
- Parameters
46
- ----------
47
- path : Path
48
- Path to the VM file on disk.
49
-
50
- Returns
51
- -------
52
- JSONList
53
- The list of dictionaries read from the VM file.
54
- """
55
- return stub.read(path, format_name='VM')
56
-
57
-
58
- def write(
59
- path: Path,
60
- data: JSONData,
61
- ) -> int:
62
- """
63
- Write ``data`` to VM file at ``path`` and return record count.
64
-
65
- Parameters
66
- ----------
67
- path : Path
68
- Path to the VM file on disk.
69
- data : JSONData
70
- Data to write as VM file. Should be a list of dictionaries or a single
71
- dictionary.
72
-
73
- Returns
74
- -------
75
- int
76
- The number of rows written to the VM file.
77
- """
78
- return stub.write(path, data, format_name='VM')
etlplus/file/wks.py DELETED
@@ -1,77 +0,0 @@
1
- """
2
- :mod:`etlplus.file.wks` module.
3
-
4
- Helpers for reading/writing Lotus 1-2-3 (WKS) spreadsheet files.
5
-
6
- Notes
7
- -----
8
- - A WKS file is a spreadsheet file created using the Lotus 1-2-3 format.
9
- - Common cases:
10
- - Reading data from legacy Lotus 1-2-3 spreadsheets.
11
- - Writing data to Lotus 1-2-3 format for compatibility.
12
- - Converting WKS files to more modern formats.
13
- - Rule of thumb:
14
- - If you need to work with Lotus 1-2-3 spreadsheet files, use this module
15
- for reading and writing.
16
- """
17
-
18
- from __future__ import annotations
19
-
20
- from pathlib import Path
21
-
22
- from ..types import JSONData
23
- from ..types import JSONList
24
- from . import stub
25
-
26
- # SECTION: EXPORTS ========================================================== #
27
-
28
-
29
- __all__ = [
30
- 'read',
31
- 'write',
32
- ]
33
-
34
-
35
- # SECTION: FUNCTIONS ======================================================== #
36
-
37
-
38
- def read(
39
- path: Path,
40
- ) -> JSONList:
41
- """
42
- Read WKS content from ``path``.
43
-
44
- Parameters
45
- ----------
46
- path : Path
47
- Path to the WKS file on disk.
48
-
49
- Returns
50
- -------
51
- JSONList
52
- The list of dictionaries read from the WKS file.
53
- """
54
- return stub.read(path, format_name='WKS')
55
-
56
-
57
- def write(
58
- path: Path,
59
- data: JSONData,
60
- ) -> int:
61
- """
62
- Write ``data`` to WKS file at ``path`` and return record count.
63
-
64
- Parameters
65
- ----------
66
- path : Path
67
- Path to the WKS file on disk.
68
- data : JSONData
69
- Data to write as WKS file. Should be a list of dictionaries or a
70
- single dictionary.
71
-
72
- Returns
73
- -------
74
- int
75
- The number of rows written to the WKS file.
76
- """
77
- return stub.write(path, data, format_name='WKS')
etlplus/file/xls.py DELETED
@@ -1,88 +0,0 @@
1
- """
2
- :mod:`etlplus.file.xls` module.
3
-
4
- Helpers for reading/writing Excel XLS files.
5
- """
6
-
7
- from __future__ import annotations
8
-
9
- from pathlib import Path
10
- from typing import cast
11
-
12
- from ..types import JSONData
13
- from ..types import JSONList
14
- from ._imports import get_pandas
15
-
16
- # SECTION: EXPORTS ========================================================== #
17
-
18
-
19
- __all__ = [
20
- 'read',
21
- 'write',
22
- ]
23
-
24
-
25
- # SECTION: FUNCTIONS ======================================================== #
26
-
27
-
28
- def read(
29
- path: Path,
30
- ) -> JSONList:
31
- """
32
- Read XLS content from ``path``.
33
-
34
- Parameters
35
- ----------
36
- path : Path
37
- Path to the XLS file on disk.
38
-
39
- Returns
40
- -------
41
- JSONList
42
- The list of dictionaries read from the XLS file.
43
-
44
- Raises
45
- ------
46
- ImportError
47
- If the optional dependency "xlrd" is not installed.
48
- """
49
- pandas = get_pandas('XLS')
50
- try:
51
- frame = pandas.read_excel(path, engine='xlrd')
52
- except ImportError as e: # pragma: no cover
53
- raise ImportError(
54
- 'XLS support requires optional dependency "xlrd".\n'
55
- 'Install with: pip install xlrd',
56
- ) from e
57
- return cast(JSONList, frame.to_dict(orient='records'))
58
-
59
-
60
- def write(
61
- path: Path,
62
- data: JSONData,
63
- ) -> int:
64
- """
65
- Write ``data`` to XLS at ``path`` and return record count.
66
-
67
- Notes
68
- -----
69
- XLS writing is not supported by pandas 2.x. Use XLSX for writes.
70
-
71
- Parameters
72
- ----------
73
- path : Path
74
- Path to the XLS file on disk.
75
- data : JSONData
76
- Data to write.
77
-
78
- Returns
79
- -------
80
- int
81
- Number of records written.
82
-
83
- Raises
84
- ------
85
- RuntimeError
86
- If XLS writing is attempted.
87
- """
88
- raise RuntimeError('XLS write is not supported; use XLSX instead')
etlplus/file/xlsm.py DELETED
@@ -1,79 +0,0 @@
1
- """
2
- :mod:`etlplus.file.xlsm` module.
3
-
4
- Helpers for reading/writing Microsoft Excel Macro-Enabled (XLSM) spreadsheet
5
- files.
6
-
7
- Notes
8
- -----
9
- - An XLSM file is a spreadsheet file created using the Microsoft Excel Macro-
10
- Enabled (Open XML) format.
11
- - Common cases:
12
- - Reading data from Excel Macro-Enabled spreadsheets.
13
- - Writing data to Excel Macro-Enabled format for compatibility.
14
- - Converting XLSM files to more modern formats.
15
- - Rule of thumb:
16
- - If you need to work with Excel Macro-Enabled spreadsheet files, use this
17
- module for reading and writing.
18
- """
19
-
20
- from __future__ import annotations
21
-
22
- from pathlib import Path
23
-
24
- from ..types import JSONData
25
- from ..types import JSONList
26
- from . import stub
27
-
28
- # SECTION: EXPORTS ========================================================== #
29
-
30
-
31
- __all__ = [
32
- 'read',
33
- 'write',
34
- ]
35
-
36
-
37
- # SECTION: FUNCTIONS ======================================================== #
38
-
39
-
40
- def read(
41
- path: Path,
42
- ) -> JSONList:
43
- """
44
- Read XLSM content from ``path``.
45
-
46
- Parameters
47
- ----------
48
- path : Path
49
- Path to the XLSM file on disk.
50
-
51
- Returns
52
- -------
53
- JSONList
54
- The list of dictionaries read from the XLSM file.
55
- """
56
- return stub.read(path, format_name='XLSM')
57
-
58
-
59
- def write(
60
- path: Path,
61
- data: JSONData,
62
- ) -> int:
63
- """
64
- Write ``data`` to XLSM file at ``path`` and return record count.
65
-
66
- Parameters
67
- ----------
68
- path : Path
69
- Path to the XLSM file on disk.
70
- data : JSONData
71
- Data to write as XLSM file. Should be a list of dictionaries or a
72
- single dictionary.
73
-
74
- Returns
75
- -------
76
- int
77
- The number of rows written to the XLSM file.
78
- """
79
- return stub.write(path, data, format_name='XLSM')
etlplus/file/xlsx.py DELETED
@@ -1,99 +0,0 @@
1
- """
2
- :mod:`etlplus.file.xlsx` module.
3
-
4
- Helpers for reading/writing Excel XLSX files.
5
- """
6
-
7
- from __future__ import annotations
8
-
9
- from pathlib import Path
10
- from typing import cast
11
-
12
- from ..types import JSONData
13
- from ..types import JSONList
14
- from ._imports import get_pandas
15
- from ._io import normalize_records
16
-
17
- # SECTION: EXPORTS ========================================================== #
18
-
19
-
20
- __all__ = [
21
- 'read',
22
- 'write',
23
- ]
24
-
25
-
26
- # SECTION: FUNCTIONS ======================================================== #
27
-
28
-
29
- def read(
30
- path: Path,
31
- ) -> JSONList:
32
- """
33
- Read XLSX content from ``path``.
34
-
35
- Parameters
36
- ----------
37
- path : Path
38
- Path to the XLSX file on disk.
39
-
40
- Returns
41
- -------
42
- JSONList
43
- The list of dictionaries read from the XLSX file.
44
-
45
- Raises
46
- ------
47
- ImportError
48
- If optional dependencies for XLSX support are missing.
49
- """
50
- pandas = get_pandas('XLSX')
51
- try:
52
- frame = pandas.read_excel(path)
53
- except ImportError as e: # pragma: no cover
54
- raise ImportError(
55
- 'XLSX support requires optional dependency "openpyxl".\n'
56
- 'Install with: pip install openpyxl',
57
- ) from e
58
- return cast(JSONList, frame.to_dict(orient='records'))
59
-
60
-
61
- def write(
62
- path: Path,
63
- data: JSONData,
64
- ) -> int:
65
- """
66
- Write ``data`` to XLSX at ``path`` and return record count.
67
-
68
- Parameters
69
- ----------
70
- path : Path
71
- Path to the XLSX file on disk.
72
- data : JSONData
73
- Data to write.
74
-
75
- Returns
76
- -------
77
- int
78
- Number of records written.
79
-
80
- Raises
81
- ------
82
- ImportError
83
- If optional dependencies for XLSX support are missing.
84
- """
85
- records = normalize_records(data, 'XLSX')
86
- if not records:
87
- return 0
88
-
89
- pandas = get_pandas('XLSX')
90
- path.parent.mkdir(parents=True, exist_ok=True)
91
- frame = pandas.DataFrame.from_records(records)
92
- try:
93
- frame.to_excel(path, index=False)
94
- except ImportError as e: # pragma: no cover
95
- raise ImportError(
96
- 'XLSX support requires optional dependency "openpyxl".\n'
97
- 'Install with: pip install openpyxl',
98
- ) from e
99
- return len(records)