dataspector 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataspector/__init__.py +64 -0
- dataspector/__main__.py +10 -0
- dataspector/_connectors.py +260 -0
- dataspector/_reporting.py +480 -0
- dataspector/checkers/__init__.py +85 -0
- dataspector/checkers/record.py +671 -0
- dataspector/checkers/source.py +344 -0
- dataspector/checkers/utils.py +342 -0
- dataspector/cli.py +207 -0
- dataspector/coboller/__init__.py +123 -0
- dataspector/coboller/codecs.py +841 -0
- dataspector/coboller/constants.py +324 -0
- dataspector/coboller/copybook.py +1448 -0
- dataspector/coboller/parser.py +201 -0
- dataspector/core.py +740 -0
- dataspector/py.typed +0 -0
- dataspector/settings.py +101 -0
- dataspector-1.0.0.dist-info/METADATA +181 -0
- dataspector-1.0.0.dist-info/RECORD +22 -0
- dataspector-1.0.0.dist-info/WHEEL +4 -0
- dataspector-1.0.0.dist-info/entry_points.txt +3 -0
- dataspector-1.0.0.dist-info/licenses/LICENSE +21 -0
dataspector/__init__.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""DataSpector: format-agnostic data-integrity checking and COBOL copybook conversion."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
6
|
+
|
|
7
|
+
from dataspector import checkers, coboller, settings
|
|
8
|
+
from dataspector.checkers import CheckOutcome, requires_all_data
|
|
9
|
+
from dataspector.cli import (
|
|
10
|
+
coboller_main,
|
|
11
|
+
main,
|
|
12
|
+
run_coboller_convert,
|
|
13
|
+
run_coboller_inspect,
|
|
14
|
+
run_dataspector,
|
|
15
|
+
)
|
|
16
|
+
from dataspector.core import (
|
|
17
|
+
CheckeredHistory,
|
|
18
|
+
GenericData,
|
|
19
|
+
available_checks,
|
|
20
|
+
connect_to_copybook,
|
|
21
|
+
connect_to_embedded_db,
|
|
22
|
+
connect_to_location,
|
|
23
|
+
connect_to_server,
|
|
24
|
+
load_and_run_checks,
|
|
25
|
+
load_driverfile,
|
|
26
|
+
parse_cell_to_dict,
|
|
27
|
+
pretty_print_results,
|
|
28
|
+
run_checks_on_location,
|
|
29
|
+
version_adder,
|
|
30
|
+
write_excel_report,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
try:
|
|
34
|
+
__version__ = version("dataspector")
|
|
35
|
+
except PackageNotFoundError:
|
|
36
|
+
__version__ = "0.0.0.dev0"
|
|
37
|
+
|
|
38
|
+
__all__ = [
|
|
39
|
+
"CheckOutcome",
|
|
40
|
+
"CheckeredHistory",
|
|
41
|
+
"GenericData",
|
|
42
|
+
"__version__",
|
|
43
|
+
"available_checks",
|
|
44
|
+
"checkers",
|
|
45
|
+
"coboller",
|
|
46
|
+
"coboller_main",
|
|
47
|
+
"connect_to_copybook",
|
|
48
|
+
"connect_to_embedded_db",
|
|
49
|
+
"connect_to_location",
|
|
50
|
+
"connect_to_server",
|
|
51
|
+
"load_and_run_checks",
|
|
52
|
+
"load_driverfile",
|
|
53
|
+
"main",
|
|
54
|
+
"parse_cell_to_dict",
|
|
55
|
+
"pretty_print_results",
|
|
56
|
+
"requires_all_data",
|
|
57
|
+
"run_checks_on_location",
|
|
58
|
+
"run_coboller_convert",
|
|
59
|
+
"run_coboller_inspect",
|
|
60
|
+
"run_dataspector",
|
|
61
|
+
"settings",
|
|
62
|
+
"version_adder",
|
|
63
|
+
"write_excel_report",
|
|
64
|
+
]
|
dataspector/__main__.py
ADDED
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
"""Path normalization and data-source connection helpers for :mod:`dataspector.core`."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import importlib
|
|
6
|
+
import pathlib
|
|
7
|
+
import re
|
|
8
|
+
import urllib.parse
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from loguru import logger
|
|
12
|
+
from sqlalchemy import create_engine
|
|
13
|
+
from sqlalchemy import inspect as sa_inspect
|
|
14
|
+
from sqlalchemy.engine import Engine
|
|
15
|
+
|
|
16
|
+
from dataspector import coboller
|
|
17
|
+
|
|
18
|
+
pyodbc: Any
|
|
19
|
+
try:
|
|
20
|
+
pyodbc = importlib.import_module("pyodbc")
|
|
21
|
+
except ImportError: # pragma: no cover - optional dependency
|
|
22
|
+
logger.trace("Optional dependency 'pyodbc' is not installed; ODBC sources are unavailable.")
|
|
23
|
+
pyodbc = None
|
|
24
|
+
|
|
25
|
+
_JET_CONNECTIONS: tuple[str, ...] = (
|
|
26
|
+
"DRIVER={Microsoft Access Driver (*.mdb, *.accdb)};DBQ=%s",
|
|
27
|
+
"DRIVER={Microsoft Access Driver (*.mdb)};DBQ=%s",
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
_SQL_MATCH = re.compile(
|
|
31
|
+
r"^\[(?P<server>.+?)\].*?\[[^\S\n]*(?P<database>.+?)\].*?\[[^\S\n]*(?P<table>.+?)\].*$",
|
|
32
|
+
re.I,
|
|
33
|
+
)
|
|
34
|
+
_EMBEDDED_DB_MATCH = re.compile(
|
|
35
|
+
r"^\[[^\S\n]*(?P<database>.+?)\].*?\[[^\S\n]*(?P<table>.+?)\].*$",
|
|
36
|
+
re.I,
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def connect_to_copybook(
|
|
41
|
+
layout: str | pathlib.Path,
|
|
42
|
+
data: str | pathlib.Path | None = None,
|
|
43
|
+
) -> tuple[coboller.Copybook | None, pathlib.Path | None]:
|
|
44
|
+
"""Attempt to parse *layout* as a COBOL copybook and resolve *data*.
|
|
45
|
+
|
|
46
|
+
Parameters
|
|
47
|
+
----------
|
|
48
|
+
layout
|
|
49
|
+
Path to the COBOL copybook file.
|
|
50
|
+
data
|
|
51
|
+
Optional path to the accompanying COBOL data file.
|
|
52
|
+
|
|
53
|
+
Returns
|
|
54
|
+
-------
|
|
55
|
+
tuple[Copybook | None, pathlib.Path | None]
|
|
56
|
+
Parsed copybook and resolved data file path, or ``None`` entries if invalid/missing.
|
|
57
|
+
"""
|
|
58
|
+
layout_path = resolve_cross_platform_path(layout)
|
|
59
|
+
data_path = resolve_cross_platform_path(data) if data is not None else None
|
|
60
|
+
|
|
61
|
+
copybook: coboller.Copybook | None = None
|
|
62
|
+
if layout_path.is_file() and layout_path.suffix.lower() in {".cbl", ".cpy", ".cob"}:
|
|
63
|
+
try:
|
|
64
|
+
parsed = coboller.Copybook(layout_path)
|
|
65
|
+
if parsed.records:
|
|
66
|
+
copybook = parsed
|
|
67
|
+
except Exception as exc:
|
|
68
|
+
logger.trace("Could not parse {} as COBOL copybook: {}", layout_path, exc)
|
|
69
|
+
|
|
70
|
+
resolved_data = (
|
|
71
|
+
data_path
|
|
72
|
+
if (copybook is not None and data_path is not None and data_path.exists())
|
|
73
|
+
else None
|
|
74
|
+
)
|
|
75
|
+
return copybook, resolved_data
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def connect_to_embedded_db(database: str | pathlib.Path) -> Any:
|
|
79
|
+
"""Connect to an embedded SQLite or MS Access database file.
|
|
80
|
+
|
|
81
|
+
Parameters
|
|
82
|
+
----------
|
|
83
|
+
database
|
|
84
|
+
Path to the embedded database file (``.sqlite``, ``.db``, ``.mdb``, ``.accdb``).
|
|
85
|
+
|
|
86
|
+
Returns
|
|
87
|
+
-------
|
|
88
|
+
Engine | pyodbc.Connection | None
|
|
89
|
+
Active database handle, or ``None`` if unsupported/unavailable.
|
|
90
|
+
"""
|
|
91
|
+
db_path = resolve_cross_platform_path(database)
|
|
92
|
+
if db_path.suffix.lower() in {".sqlite", ".sqlite3", ".db"}:
|
|
93
|
+
try:
|
|
94
|
+
engine = create_engine(f"sqlite:///{db_path}")
|
|
95
|
+
sa_inspect(engine).get_table_names()
|
|
96
|
+
return engine
|
|
97
|
+
except Exception as exc:
|
|
98
|
+
logger.trace("SQLite connection failed for {}: {}", db_path, exc)
|
|
99
|
+
return None
|
|
100
|
+
|
|
101
|
+
if pyodbc is not None and db_path.suffix.lower() in {".mdb", ".accdb"}:
|
|
102
|
+
for base in _JET_CONNECTIONS:
|
|
103
|
+
connstr = base % str(db_path)
|
|
104
|
+
try:
|
|
105
|
+
return pyodbc.connect(connstr)
|
|
106
|
+
except Exception as exc:
|
|
107
|
+
logger.trace("MS Access ODBC connection failed: {}", exc)
|
|
108
|
+
return None
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def connect_to_location(
|
|
112
|
+
location: str | pathlib.Path | tuple[Any, ...],
|
|
113
|
+
) -> tuple[pathlib.Path | coboller.Copybook | None, Any, str | None]:
|
|
114
|
+
"""Resolve *location* into a ``(path, engine, table)`` triple.
|
|
115
|
+
|
|
116
|
+
Parameters
|
|
117
|
+
----------
|
|
118
|
+
location
|
|
119
|
+
File/directory path, bracketed locator (``'[file][table]'``,
|
|
120
|
+
``'[layout.cbl][data.dat]'``, ``'[server][db][table]'``), or tuple.
|
|
121
|
+
|
|
122
|
+
Returns
|
|
123
|
+
-------
|
|
124
|
+
tuple[pathlib.Path | Copybook | None, Any, str | None]
|
|
125
|
+
``(path, engine, table)`` triple describing the resolved data source.
|
|
126
|
+
"""
|
|
127
|
+
if isinstance(location, pathlib.Path):
|
|
128
|
+
location = str(location)
|
|
129
|
+
|
|
130
|
+
if isinstance(location, str):
|
|
131
|
+
loc_path = resolve_cross_platform_path(location)
|
|
132
|
+
try:
|
|
133
|
+
loc_exists = loc_path.exists()
|
|
134
|
+
except OSError:
|
|
135
|
+
loc_exists = False
|
|
136
|
+
|
|
137
|
+
if loc_exists:
|
|
138
|
+
copybook, _ = connect_to_copybook(loc_path)
|
|
139
|
+
return (copybook if copybook is not None else loc_path), None, None
|
|
140
|
+
|
|
141
|
+
sql_m = _SQL_MATCH.match(location)
|
|
142
|
+
emb_m = _EMBEDDED_DB_MATCH.match(location)
|
|
143
|
+
|
|
144
|
+
if sql_m is not None:
|
|
145
|
+
server = sql_m.group("server")
|
|
146
|
+
database = sql_m.group("database")
|
|
147
|
+
table = sql_m.group("table")
|
|
148
|
+
return None, connect_to_server(server, database), table
|
|
149
|
+
|
|
150
|
+
if emb_m is not None:
|
|
151
|
+
raw_db = emb_m.group("database")
|
|
152
|
+
raw_tbl = emb_m.group("table")
|
|
153
|
+
db_path = resolve_cross_platform_path(raw_db)
|
|
154
|
+
copybook, datapath = connect_to_copybook(db_path, raw_tbl)
|
|
155
|
+
if copybook is not None:
|
|
156
|
+
return copybook, datapath, None
|
|
157
|
+
if db_path.exists():
|
|
158
|
+
return db_path, connect_to_embedded_db(db_path), raw_tbl
|
|
159
|
+
|
|
160
|
+
return loc_path, None, None
|
|
161
|
+
|
|
162
|
+
if isinstance(location, tuple):
|
|
163
|
+
if len(location) >= 2:
|
|
164
|
+
copybook, datapath = connect_to_copybook(location[0], location[1])
|
|
165
|
+
if copybook is not None:
|
|
166
|
+
return copybook, datapath, None
|
|
167
|
+
|
|
168
|
+
first_path = resolve_cross_platform_path(location[0])
|
|
169
|
+
if first_path.exists():
|
|
170
|
+
return first_path, connect_to_embedded_db(first_path), str(location[1])
|
|
171
|
+
|
|
172
|
+
engine = connect_to_server(str(location[0]), str(location[1]))
|
|
173
|
+
table = str(location[2]) if len(location) > 2 else None
|
|
174
|
+
return None, engine, table
|
|
175
|
+
|
|
176
|
+
if len(location) == 1:
|
|
177
|
+
return connect_to_location(location[0])
|
|
178
|
+
|
|
179
|
+
return None, None, None
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def connect_to_server(server: str, database: str) -> Engine | None:
|
|
183
|
+
"""Try ODBC connection templates to connect to a remote SQL Server database.
|
|
184
|
+
|
|
185
|
+
Parameters
|
|
186
|
+
----------
|
|
187
|
+
server
|
|
188
|
+
Database server hostname or instance name.
|
|
189
|
+
database
|
|
190
|
+
Target database name.
|
|
191
|
+
|
|
192
|
+
Returns
|
|
193
|
+
-------
|
|
194
|
+
Engine | None
|
|
195
|
+
Connected SQLAlchemy engine, or ``None`` if no template succeeded.
|
|
196
|
+
"""
|
|
197
|
+
if pyodbc is None:
|
|
198
|
+
logger.trace("pyodbc is not installed; skipping remote SQL Server connection.")
|
|
199
|
+
return None
|
|
200
|
+
|
|
201
|
+
ad_odbc = urllib.parse.quote_plus(
|
|
202
|
+
f"DRIVER={{ODBC Driver 18 for SQL Server}};SERVER={server};DATABASE={database};"
|
|
203
|
+
"Authentication=ActiveDirectoryIntegrated;"
|
|
204
|
+
)
|
|
205
|
+
templates = (
|
|
206
|
+
f"mssql+pyodbc:///?odbc_connect={ad_odbc}",
|
|
207
|
+
f"mssql+pyodbc://{server}/{database}?driver=ODBC+Driver+17+for+SQL+Server",
|
|
208
|
+
f"mssql+pyodbc://{server}/{database}?driver=SQL+Server+Native+Client+11.0",
|
|
209
|
+
)
|
|
210
|
+
for url in templates:
|
|
211
|
+
try:
|
|
212
|
+
engine = create_engine(url)
|
|
213
|
+
sa_inspect(engine).get_table_names()
|
|
214
|
+
return engine
|
|
215
|
+
except Exception as exc:
|
|
216
|
+
logger.trace("Server connection template failed: {}", exc)
|
|
217
|
+
return None
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def is_pyodbc_connection(obj: Any) -> bool:
|
|
221
|
+
"""Return ``True`` if *obj* is a :class:`pyodbc.Connection` instance.
|
|
222
|
+
|
|
223
|
+
Parameters
|
|
224
|
+
----------
|
|
225
|
+
obj
|
|
226
|
+
Candidate database handle.
|
|
227
|
+
|
|
228
|
+
Returns
|
|
229
|
+
-------
|
|
230
|
+
bool
|
|
231
|
+
``True`` when ``pyodbc`` is installed and *obj* is a connection.
|
|
232
|
+
"""
|
|
233
|
+
return pyodbc is not None and isinstance(obj, pyodbc.Connection)
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def resolve_cross_platform_path(raw: str | pathlib.Path) -> pathlib.Path:
|
|
237
|
+
"""Resolve *raw* into a :class:`pathlib.Path`, normalizing Windows backslashes on POSIX.
|
|
238
|
+
|
|
239
|
+
Parameters
|
|
240
|
+
----------
|
|
241
|
+
raw
|
|
242
|
+
Filesystem path string or :class:`pathlib.Path`.
|
|
243
|
+
|
|
244
|
+
Returns
|
|
245
|
+
-------
|
|
246
|
+
pathlib.Path
|
|
247
|
+
Normalized path (preferring forward-slash fallback if it exists).
|
|
248
|
+
"""
|
|
249
|
+
if isinstance(raw, pathlib.Path):
|
|
250
|
+
if raw.exists() or "\\" not in str(raw):
|
|
251
|
+
return raw
|
|
252
|
+
candidate = pathlib.Path(str(raw).replace("\\", "/"))
|
|
253
|
+
return candidate if candidate.exists() else raw
|
|
254
|
+
|
|
255
|
+
path = pathlib.Path(raw)
|
|
256
|
+
if "\\" in raw and not path.exists():
|
|
257
|
+
candidate = pathlib.Path(raw.replace("\\", "/"))
|
|
258
|
+
if candidate.exists():
|
|
259
|
+
return candidate
|
|
260
|
+
return path
|