dataspector 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataspector/__init__.py +64 -0
- dataspector/__main__.py +10 -0
- dataspector/_connectors.py +260 -0
- dataspector/_reporting.py +480 -0
- dataspector/checkers/__init__.py +85 -0
- dataspector/checkers/record.py +671 -0
- dataspector/checkers/source.py +344 -0
- dataspector/checkers/utils.py +342 -0
- dataspector/cli.py +207 -0
- dataspector/coboller/__init__.py +123 -0
- dataspector/coboller/codecs.py +841 -0
- dataspector/coboller/constants.py +324 -0
- dataspector/coboller/copybook.py +1448 -0
- dataspector/coboller/parser.py +201 -0
- dataspector/core.py +740 -0
- dataspector/py.typed +0 -0
- dataspector/settings.py +101 -0
- dataspector-1.0.0.dist-info/METADATA +181 -0
- dataspector-1.0.0.dist-info/RECORD +22 -0
- dataspector-1.0.0.dist-info/WHEEL +4 -0
- dataspector-1.0.0.dist-info/entry_points.txt +3 -0
- dataspector-1.0.0.dist-info/licenses/LICENSE +21 -0
dataspector/cli.py
ADDED
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
"""Command-line interfaces for ``dataspector`` and ``coboller`` powered by ``jsonargparse``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import sys
|
|
6
|
+
from collections.abc import Sequence
|
|
7
|
+
|
|
8
|
+
from jsonargparse import auto_cli
|
|
9
|
+
from loguru import logger
|
|
10
|
+
|
|
11
|
+
from dataspector.coboller import Copybook
|
|
12
|
+
from dataspector.core import (
|
|
13
|
+
load_and_run_checks,
|
|
14
|
+
pretty_print_results,
|
|
15
|
+
version_adder,
|
|
16
|
+
write_excel_report,
|
|
17
|
+
)
|
|
18
|
+
from dataspector.settings import (
|
|
19
|
+
CobollerConvertSettings,
|
|
20
|
+
CobollerInspectSettings,
|
|
21
|
+
DataSpectorSettings,
|
|
22
|
+
LogLevel,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _configure_logging(level: LogLevel) -> None:
|
|
27
|
+
"""Configure loguru stderr sink at *level*."""
|
|
28
|
+
logger.remove()
|
|
29
|
+
logger.add(
|
|
30
|
+
sys.stderr,
|
|
31
|
+
level=level,
|
|
32
|
+
format="{time:YYYY-MM-DD HH:mm:ss} | {level:<8} | {name} - {message}",
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def run_dataspector(settings: DataSpectorSettings) -> int:
|
|
37
|
+
"""Execute the DataSpector batch check workflow from *settings*.
|
|
38
|
+
|
|
39
|
+
Parameters
|
|
40
|
+
----------
|
|
41
|
+
settings
|
|
42
|
+
Parsed CLI / config settings.
|
|
43
|
+
|
|
44
|
+
Returns
|
|
45
|
+
-------
|
|
46
|
+
int
|
|
47
|
+
Exit status code (``0`` if all checks pass, ``1`` otherwise).
|
|
48
|
+
"""
|
|
49
|
+
_configure_logging(settings.log_level)
|
|
50
|
+
|
|
51
|
+
if settings.driver is None:
|
|
52
|
+
logger.error("No driver workbook supplied. Pass --driver <path.xlsx>.")
|
|
53
|
+
return 1
|
|
54
|
+
|
|
55
|
+
if not settings.driver.exists():
|
|
56
|
+
logger.error("Could not find driver file {}. Stopping DataSpector.", settings.driver)
|
|
57
|
+
return 1
|
|
58
|
+
|
|
59
|
+
results = load_and_run_checks(
|
|
60
|
+
settings.driver,
|
|
61
|
+
use_multiprocessing=settings.use_multiprocess,
|
|
62
|
+
)
|
|
63
|
+
pretty_print_results(results)
|
|
64
|
+
|
|
65
|
+
if settings.output is not None:
|
|
66
|
+
out_target = settings.output
|
|
67
|
+
if settings.overwrite and out_target.exists():
|
|
68
|
+
out_target.unlink()
|
|
69
|
+
fout = (
|
|
70
|
+
str(out_target) if settings.overwrite else version_adder(out_target, extension=".xlsx")
|
|
71
|
+
)
|
|
72
|
+
written = write_excel_report(results, fout)
|
|
73
|
+
print(f"Writing Excel report to {written.resolve()}")
|
|
74
|
+
|
|
75
|
+
if any(not res.all_passed for res in results):
|
|
76
|
+
print("Some checks failed to pass. Exiting with status code 1.")
|
|
77
|
+
return 1
|
|
78
|
+
|
|
79
|
+
return 0
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def run_coboller_inspect(settings: CobollerInspectSettings) -> int:
|
|
83
|
+
"""Print a copybook's field layout, PIC/USAGE types, and byte offsets.
|
|
84
|
+
|
|
85
|
+
Parameters
|
|
86
|
+
----------
|
|
87
|
+
settings
|
|
88
|
+
Parsed CLI settings for the ``coboller inspect`` subcommand.
|
|
89
|
+
|
|
90
|
+
Returns
|
|
91
|
+
-------
|
|
92
|
+
int
|
|
93
|
+
Exit status code (``0`` on success, ``1`` if the copybook is missing).
|
|
94
|
+
"""
|
|
95
|
+
_configure_logging(settings.log_level)
|
|
96
|
+
|
|
97
|
+
if not settings.copybook.exists():
|
|
98
|
+
print(f"Error: copybook not found: {settings.copybook}")
|
|
99
|
+
return 1
|
|
100
|
+
|
|
101
|
+
cbl = Copybook(settings.copybook)
|
|
102
|
+
table = cbl.summary(
|
|
103
|
+
include_redefines=settings.include_redefines,
|
|
104
|
+
include_filler=not settings.omit_filler,
|
|
105
|
+
)
|
|
106
|
+
print(
|
|
107
|
+
f"Copybook: {settings.copybook} "
|
|
108
|
+
f"(record_length={cbl.record_length} bytes, fields={len(table)})"
|
|
109
|
+
)
|
|
110
|
+
print(table.to_string(index=False))
|
|
111
|
+
return 0
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def run_coboller_convert(settings: CobollerConvertSettings) -> int:
|
|
115
|
+
"""Preview or convert a COBOL data file into a human-readable format.
|
|
116
|
+
|
|
117
|
+
Parameters
|
|
118
|
+
----------
|
|
119
|
+
settings
|
|
120
|
+
Parsed CLI settings for the ``coboller convert`` subcommand.
|
|
121
|
+
|
|
122
|
+
Returns
|
|
123
|
+
-------
|
|
124
|
+
int
|
|
125
|
+
Exit status code (``0`` on success, ``1`` if an input file is missing).
|
|
126
|
+
"""
|
|
127
|
+
_configure_logging(settings.log_level)
|
|
128
|
+
|
|
129
|
+
if not settings.copybook.exists():
|
|
130
|
+
print(f"Error: copybook not found: {settings.copybook}")
|
|
131
|
+
return 1
|
|
132
|
+
if not settings.datafile.exists():
|
|
133
|
+
print(f"Error: data file not found: {settings.datafile}")
|
|
134
|
+
return 1
|
|
135
|
+
|
|
136
|
+
cbl = Copybook(settings.copybook)
|
|
137
|
+
|
|
138
|
+
if settings.output is not None:
|
|
139
|
+
written = cbl.convert(
|
|
140
|
+
settings.datafile,
|
|
141
|
+
settings.output,
|
|
142
|
+
encoding=settings.encoding,
|
|
143
|
+
float_format=settings.float_format,
|
|
144
|
+
byteorder=settings.byteorder,
|
|
145
|
+
include_redefines=settings.include_redefines,
|
|
146
|
+
include_filler=settings.include_filler,
|
|
147
|
+
)
|
|
148
|
+
print(f"Converted {settings.datafile} -> {written}")
|
|
149
|
+
return 0
|
|
150
|
+
|
|
151
|
+
df = cbl.to_dataframe(
|
|
152
|
+
settings.datafile,
|
|
153
|
+
nrows=settings.nrows,
|
|
154
|
+
encoding=settings.encoding,
|
|
155
|
+
float_format=settings.float_format,
|
|
156
|
+
byteorder=settings.byteorder,
|
|
157
|
+
include_redefines=settings.include_redefines,
|
|
158
|
+
include_filler=settings.include_filler,
|
|
159
|
+
)
|
|
160
|
+
print(df.to_string(index=False))
|
|
161
|
+
return 0
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
165
|
+
"""Entry point for the ``dataspector`` CLI.
|
|
166
|
+
|
|
167
|
+
Parameters
|
|
168
|
+
----------
|
|
169
|
+
argv
|
|
170
|
+
Optional argument list (defaults to ``sys.argv[1:]``).
|
|
171
|
+
|
|
172
|
+
Returns
|
|
173
|
+
-------
|
|
174
|
+
int
|
|
175
|
+
Exit status code (``0`` if all checks pass, ``1`` otherwise).
|
|
176
|
+
"""
|
|
177
|
+
settings = auto_cli(
|
|
178
|
+
DataSpectorSettings,
|
|
179
|
+
args=list(argv) if argv is not None else None,
|
|
180
|
+
as_positional=False,
|
|
181
|
+
)
|
|
182
|
+
return run_dataspector(settings)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def coboller_main(argv: Sequence[str] | None = None) -> int:
|
|
186
|
+
"""Entry point for the ``coboller`` CLI (``inspect`` and ``convert`` subcommands).
|
|
187
|
+
|
|
188
|
+
Parameters
|
|
189
|
+
----------
|
|
190
|
+
argv
|
|
191
|
+
Optional argument list (defaults to ``sys.argv[1:]``).
|
|
192
|
+
|
|
193
|
+
Returns
|
|
194
|
+
-------
|
|
195
|
+
int
|
|
196
|
+
Exit status code (``0`` on success, ``1`` on error).
|
|
197
|
+
"""
|
|
198
|
+
settings = auto_cli(
|
|
199
|
+
{
|
|
200
|
+
"inspect": CobollerInspectSettings,
|
|
201
|
+
"convert": CobollerConvertSettings,
|
|
202
|
+
},
|
|
203
|
+
args=list(argv) if argv is not None else None,
|
|
204
|
+
)
|
|
205
|
+
if isinstance(settings, CobollerInspectSettings):
|
|
206
|
+
return run_coboller_inspect(settings)
|
|
207
|
+
return run_coboller_convert(settings)
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
"""COBOL copybook parser, data decoder, and modern format converter.
|
|
2
|
+
|
|
3
|
+
Organized into four focused submodules:
|
|
4
|
+
|
|
5
|
+
* :mod:`dataspector.coboller.copybook` —
|
|
6
|
+
:class:`~dataspector.coboller.copybook.Copybook`,
|
|
7
|
+
:class:`~dataspector.coboller.copybook.GroupItem`,
|
|
8
|
+
:class:`~dataspector.coboller.copybook.ElementaryItem`,
|
|
9
|
+
:class:`~dataspector.coboller.copybook.ConditionName`,
|
|
10
|
+
:class:`~dataspector.coboller.copybook.ResolvedItem`, and
|
|
11
|
+
:func:`~dataspector.coboller.copybook.parse_copybook`
|
|
12
|
+
* :mod:`dataspector.coboller.codecs` — pure binary/text pack & unpack codecs
|
|
13
|
+
(``COMP-3``, ``COMP-6``, ``COMP``/``COMP-5``/``COMP-X``, ``COMP-1``/``COMP-2``
|
|
14
|
+
IEEE & IBM HFP, zoned decimal, numeric-edited, external float, ``USAGE BIT``)
|
|
15
|
+
* :mod:`dataspector.coboller.parser` — fixed-format & free-format COBOL line
|
|
16
|
+
cleaning and clause parsing
|
|
17
|
+
* :mod:`dataspector.coboller.constants` —
|
|
18
|
+
:class:`~dataspector.coboller.constants.Usage`,
|
|
19
|
+
:class:`~dataspector.coboller.constants.SignMode`,
|
|
20
|
+
:class:`~dataspector.coboller.constants.DataCategory`, compiled regexes, and
|
|
21
|
+
lookup tables
|
|
22
|
+
|
|
23
|
+
References
|
|
24
|
+
----------
|
|
25
|
+
* IBM Enterprise COBOL Language Reference — PICTURE & USAGE clauses:
|
|
26
|
+
https://www.ibm.com/docs/en/cobol-zos/latest?topic=division-data-description-entry
|
|
27
|
+
* IBM Enterprise COBOL — UTF-8 (``PIC U`` / ``BYTE-LENGTH``) & National (``PIC N``):
|
|
28
|
+
https://www.ibm.com/docs/en/cobol-zos/latest?topic=clause-usage
|
|
29
|
+
* IBM z/Architecture Hexadecimal Floating-Point (HFP):
|
|
30
|
+
https://en.wikipedia.org/wiki/IBM_hexadecimal_floating-point
|
|
31
|
+
* GnuCOBOL Programmer's Guide (``COMP-6``, ``COMP-X``, ``BINARY-*``, ``FLOAT-*``, ``USAGE BIT``):
|
|
32
|
+
https://gnucobol.sourceforge.io/guides.html
|
|
33
|
+
* Reading COBOL Layouts:
|
|
34
|
+
http://www.3480-3590-data-conversion.com/article-reading-cobol-layouts-1.html
|
|
35
|
+
* COBOL Code Format:
|
|
36
|
+
https://en.wikipedia.org/wiki/COBOL#Code_format
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
from __future__ import annotations
|
|
40
|
+
|
|
41
|
+
from dataspector.coboller.codecs import (
|
|
42
|
+
bit_pack,
|
|
43
|
+
bit_unpack,
|
|
44
|
+
comp3_pack,
|
|
45
|
+
comp3_unpack,
|
|
46
|
+
comp3decompress,
|
|
47
|
+
comp6_pack,
|
|
48
|
+
comp6_unpack,
|
|
49
|
+
comp_float_pack,
|
|
50
|
+
comp_float_unpack,
|
|
51
|
+
comp_pack,
|
|
52
|
+
comp_unpack,
|
|
53
|
+
expand_pic,
|
|
54
|
+
external_float_unpack,
|
|
55
|
+
float_to_ibm_hfp,
|
|
56
|
+
ibm_hfp_to_float,
|
|
57
|
+
parse_numeric_edited,
|
|
58
|
+
predict_encoding,
|
|
59
|
+
zoned_pack,
|
|
60
|
+
zoned_unpack,
|
|
61
|
+
)
|
|
62
|
+
from dataspector.coboller.constants import (
|
|
63
|
+
PIC_FLOAT,
|
|
64
|
+
PIC_INT,
|
|
65
|
+
PIC_REPEAT,
|
|
66
|
+
DataCategory,
|
|
67
|
+
FieldKind,
|
|
68
|
+
SignMode,
|
|
69
|
+
Usage,
|
|
70
|
+
)
|
|
71
|
+
from dataspector.coboller.copybook import (
|
|
72
|
+
Condition,
|
|
73
|
+
ConditionName,
|
|
74
|
+
CopyBook,
|
|
75
|
+
Copybook,
|
|
76
|
+
ElementaryItem,
|
|
77
|
+
Field,
|
|
78
|
+
Group,
|
|
79
|
+
GroupItem,
|
|
80
|
+
ResolvedField,
|
|
81
|
+
ResolvedItem,
|
|
82
|
+
parse_copybook,
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
__all__ = [
|
|
86
|
+
"PIC_FLOAT",
|
|
87
|
+
"PIC_INT",
|
|
88
|
+
"PIC_REPEAT",
|
|
89
|
+
"Condition",
|
|
90
|
+
"ConditionName",
|
|
91
|
+
"CopyBook",
|
|
92
|
+
"Copybook",
|
|
93
|
+
"DataCategory",
|
|
94
|
+
"ElementaryItem",
|
|
95
|
+
"Field",
|
|
96
|
+
"FieldKind",
|
|
97
|
+
"Group",
|
|
98
|
+
"GroupItem",
|
|
99
|
+
"ResolvedField",
|
|
100
|
+
"ResolvedItem",
|
|
101
|
+
"SignMode",
|
|
102
|
+
"Usage",
|
|
103
|
+
"bit_pack",
|
|
104
|
+
"bit_unpack",
|
|
105
|
+
"comp3_pack",
|
|
106
|
+
"comp3_unpack",
|
|
107
|
+
"comp3decompress",
|
|
108
|
+
"comp6_pack",
|
|
109
|
+
"comp6_unpack",
|
|
110
|
+
"comp_float_pack",
|
|
111
|
+
"comp_float_unpack",
|
|
112
|
+
"comp_pack",
|
|
113
|
+
"comp_unpack",
|
|
114
|
+
"expand_pic",
|
|
115
|
+
"external_float_unpack",
|
|
116
|
+
"float_to_ibm_hfp",
|
|
117
|
+
"ibm_hfp_to_float",
|
|
118
|
+
"parse_copybook",
|
|
119
|
+
"parse_numeric_edited",
|
|
120
|
+
"predict_encoding",
|
|
121
|
+
"zoned_pack",
|
|
122
|
+
"zoned_unpack",
|
|
123
|
+
]
|