dataspector 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,64 @@
1
+ """DataSpector: format-agnostic data-integrity checking and COBOL copybook conversion."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from importlib.metadata import PackageNotFoundError, version
6
+
7
+ from dataspector import checkers, coboller, settings
8
+ from dataspector.checkers import CheckOutcome, requires_all_data
9
+ from dataspector.cli import (
10
+ coboller_main,
11
+ main,
12
+ run_coboller_convert,
13
+ run_coboller_inspect,
14
+ run_dataspector,
15
+ )
16
+ from dataspector.core import (
17
+ CheckeredHistory,
18
+ GenericData,
19
+ available_checks,
20
+ connect_to_copybook,
21
+ connect_to_embedded_db,
22
+ connect_to_location,
23
+ connect_to_server,
24
+ load_and_run_checks,
25
+ load_driverfile,
26
+ parse_cell_to_dict,
27
+ pretty_print_results,
28
+ run_checks_on_location,
29
+ version_adder,
30
+ write_excel_report,
31
+ )
32
+
33
+ try:
34
+ __version__ = version("dataspector")
35
+ except PackageNotFoundError:
36
+ __version__ = "0.0.0.dev0"
37
+
38
+ __all__ = [
39
+ "CheckOutcome",
40
+ "CheckeredHistory",
41
+ "GenericData",
42
+ "__version__",
43
+ "available_checks",
44
+ "checkers",
45
+ "coboller",
46
+ "coboller_main",
47
+ "connect_to_copybook",
48
+ "connect_to_embedded_db",
49
+ "connect_to_location",
50
+ "connect_to_server",
51
+ "load_and_run_checks",
52
+ "load_driverfile",
53
+ "main",
54
+ "parse_cell_to_dict",
55
+ "pretty_print_results",
56
+ "requires_all_data",
57
+ "run_checks_on_location",
58
+ "run_coboller_convert",
59
+ "run_coboller_inspect",
60
+ "run_dataspector",
61
+ "settings",
62
+ "version_adder",
63
+ "write_excel_report",
64
+ ]
@@ -0,0 +1,10 @@
1
+ """Enable ``python -m dataspector`` execution."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import sys
6
+
7
+ from dataspector.cli import main
8
+
9
+ if __name__ == "__main__":
10
+ sys.exit(main())
@@ -0,0 +1,260 @@
1
+ """Path normalization and data-source connection helpers for :mod:`dataspector.core`."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import importlib
6
+ import pathlib
7
+ import re
8
+ import urllib.parse
9
+ from typing import Any
10
+
11
+ from loguru import logger
12
+ from sqlalchemy import create_engine
13
+ from sqlalchemy import inspect as sa_inspect
14
+ from sqlalchemy.engine import Engine
15
+
16
+ from dataspector import coboller
17
+
18
+ pyodbc: Any
19
+ try:
20
+ pyodbc = importlib.import_module("pyodbc")
21
+ except ImportError: # pragma: no cover - optional dependency
22
+ logger.trace("Optional dependency 'pyodbc' is not installed; ODBC sources are unavailable.")
23
+ pyodbc = None
24
+
25
+ _JET_CONNECTIONS: tuple[str, ...] = (
26
+ "DRIVER={Microsoft Access Driver (*.mdb, *.accdb)};DBQ=%s",
27
+ "DRIVER={Microsoft Access Driver (*.mdb)};DBQ=%s",
28
+ )
29
+
30
+ _SQL_MATCH = re.compile(
31
+ r"^\[(?P<server>.+?)\].*?\[[^\S\n]*(?P<database>.+?)\].*?\[[^\S\n]*(?P<table>.+?)\].*$",
32
+ re.I,
33
+ )
34
+ _EMBEDDED_DB_MATCH = re.compile(
35
+ r"^\[[^\S\n]*(?P<database>.+?)\].*?\[[^\S\n]*(?P<table>.+?)\].*$",
36
+ re.I,
37
+ )
38
+
39
+
40
+ def connect_to_copybook(
41
+ layout: str | pathlib.Path,
42
+ data: str | pathlib.Path | None = None,
43
+ ) -> tuple[coboller.Copybook | None, pathlib.Path | None]:
44
+ """Attempt to parse *layout* as a COBOL copybook and resolve *data*.
45
+
46
+ Parameters
47
+ ----------
48
+ layout
49
+ Path to the COBOL copybook file.
50
+ data
51
+ Optional path to the accompanying COBOL data file.
52
+
53
+ Returns
54
+ -------
55
+ tuple[Copybook | None, pathlib.Path | None]
56
+ Parsed copybook and resolved data file path, or ``None`` entries if invalid/missing.
57
+ """
58
+ layout_path = resolve_cross_platform_path(layout)
59
+ data_path = resolve_cross_platform_path(data) if data is not None else None
60
+
61
+ copybook: coboller.Copybook | None = None
62
+ if layout_path.is_file() and layout_path.suffix.lower() in {".cbl", ".cpy", ".cob"}:
63
+ try:
64
+ parsed = coboller.Copybook(layout_path)
65
+ if parsed.records:
66
+ copybook = parsed
67
+ except Exception as exc:
68
+ logger.trace("Could not parse {} as COBOL copybook: {}", layout_path, exc)
69
+
70
+ resolved_data = (
71
+ data_path
72
+ if (copybook is not None and data_path is not None and data_path.exists())
73
+ else None
74
+ )
75
+ return copybook, resolved_data
76
+
77
+
78
+ def connect_to_embedded_db(database: str | pathlib.Path) -> Any:
79
+ """Connect to an embedded SQLite or MS Access database file.
80
+
81
+ Parameters
82
+ ----------
83
+ database
84
+ Path to the embedded database file (``.sqlite``, ``.db``, ``.mdb``, ``.accdb``).
85
+
86
+ Returns
87
+ -------
88
+ Engine | pyodbc.Connection | None
89
+ Active database handle, or ``None`` if unsupported/unavailable.
90
+ """
91
+ db_path = resolve_cross_platform_path(database)
92
+ if db_path.suffix.lower() in {".sqlite", ".sqlite3", ".db"}:
93
+ try:
94
+ engine = create_engine(f"sqlite:///{db_path}")
95
+ sa_inspect(engine).get_table_names()
96
+ return engine
97
+ except Exception as exc:
98
+ logger.trace("SQLite connection failed for {}: {}", db_path, exc)
99
+ return None
100
+
101
+ if pyodbc is not None and db_path.suffix.lower() in {".mdb", ".accdb"}:
102
+ for base in _JET_CONNECTIONS:
103
+ connstr = base % str(db_path)
104
+ try:
105
+ return pyodbc.connect(connstr)
106
+ except Exception as exc:
107
+ logger.trace("MS Access ODBC connection failed: {}", exc)
108
+ return None
109
+
110
+
111
+ def connect_to_location(
112
+ location: str | pathlib.Path | tuple[Any, ...],
113
+ ) -> tuple[pathlib.Path | coboller.Copybook | None, Any, str | None]:
114
+ """Resolve *location* into a ``(path, engine, table)`` triple.
115
+
116
+ Parameters
117
+ ----------
118
+ location
119
+ File/directory path, bracketed locator (``'[file][table]'``,
120
+ ``'[layout.cbl][data.dat]'``, ``'[server][db][table]'``), or tuple.
121
+
122
+ Returns
123
+ -------
124
+ tuple[pathlib.Path | Copybook | None, Any, str | None]
125
+ ``(path, engine, table)`` triple describing the resolved data source.
126
+ """
127
+ if isinstance(location, pathlib.Path):
128
+ location = str(location)
129
+
130
+ if isinstance(location, str):
131
+ loc_path = resolve_cross_platform_path(location)
132
+ try:
133
+ loc_exists = loc_path.exists()
134
+ except OSError:
135
+ loc_exists = False
136
+
137
+ if loc_exists:
138
+ copybook, _ = connect_to_copybook(loc_path)
139
+ return (copybook if copybook is not None else loc_path), None, None
140
+
141
+ sql_m = _SQL_MATCH.match(location)
142
+ emb_m = _EMBEDDED_DB_MATCH.match(location)
143
+
144
+ if sql_m is not None:
145
+ server = sql_m.group("server")
146
+ database = sql_m.group("database")
147
+ table = sql_m.group("table")
148
+ return None, connect_to_server(server, database), table
149
+
150
+ if emb_m is not None:
151
+ raw_db = emb_m.group("database")
152
+ raw_tbl = emb_m.group("table")
153
+ db_path = resolve_cross_platform_path(raw_db)
154
+ copybook, datapath = connect_to_copybook(db_path, raw_tbl)
155
+ if copybook is not None:
156
+ return copybook, datapath, None
157
+ if db_path.exists():
158
+ return db_path, connect_to_embedded_db(db_path), raw_tbl
159
+
160
+ return loc_path, None, None
161
+
162
+ if isinstance(location, tuple):
163
+ if len(location) >= 2:
164
+ copybook, datapath = connect_to_copybook(location[0], location[1])
165
+ if copybook is not None:
166
+ return copybook, datapath, None
167
+
168
+ first_path = resolve_cross_platform_path(location[0])
169
+ if first_path.exists():
170
+ return first_path, connect_to_embedded_db(first_path), str(location[1])
171
+
172
+ engine = connect_to_server(str(location[0]), str(location[1]))
173
+ table = str(location[2]) if len(location) > 2 else None
174
+ return None, engine, table
175
+
176
+ if len(location) == 1:
177
+ return connect_to_location(location[0])
178
+
179
+ return None, None, None
180
+
181
+
182
+ def connect_to_server(server: str, database: str) -> Engine | None:
183
+ """Try ODBC connection templates to connect to a remote SQL Server database.
184
+
185
+ Parameters
186
+ ----------
187
+ server
188
+ Database server hostname or instance name.
189
+ database
190
+ Target database name.
191
+
192
+ Returns
193
+ -------
194
+ Engine | None
195
+ Connected SQLAlchemy engine, or ``None`` if no template succeeded.
196
+ """
197
+ if pyodbc is None:
198
+ logger.trace("pyodbc is not installed; skipping remote SQL Server connection.")
199
+ return None
200
+
201
+ ad_odbc = urllib.parse.quote_plus(
202
+ f"DRIVER={{ODBC Driver 18 for SQL Server}};SERVER={server};DATABASE={database};"
203
+ "Authentication=ActiveDirectoryIntegrated;"
204
+ )
205
+ templates = (
206
+ f"mssql+pyodbc:///?odbc_connect={ad_odbc}",
207
+ f"mssql+pyodbc://{server}/{database}?driver=ODBC+Driver+17+for+SQL+Server",
208
+ f"mssql+pyodbc://{server}/{database}?driver=SQL+Server+Native+Client+11.0",
209
+ )
210
+ for url in templates:
211
+ try:
212
+ engine = create_engine(url)
213
+ sa_inspect(engine).get_table_names()
214
+ return engine
215
+ except Exception as exc:
216
+ logger.trace("Server connection template failed: {}", exc)
217
+ return None
218
+
219
+
220
+ def is_pyodbc_connection(obj: Any) -> bool:
221
+ """Return ``True`` if *obj* is a :class:`pyodbc.Connection` instance.
222
+
223
+ Parameters
224
+ ----------
225
+ obj
226
+ Candidate database handle.
227
+
228
+ Returns
229
+ -------
230
+ bool
231
+ ``True`` when ``pyodbc`` is installed and *obj* is a connection.
232
+ """
233
+ return pyodbc is not None and isinstance(obj, pyodbc.Connection)
234
+
235
+
236
+ def resolve_cross_platform_path(raw: str | pathlib.Path) -> pathlib.Path:
237
+ """Resolve *raw* into a :class:`pathlib.Path`, normalizing Windows backslashes on POSIX.
238
+
239
+ Parameters
240
+ ----------
241
+ raw
242
+ Filesystem path string or :class:`pathlib.Path`.
243
+
244
+ Returns
245
+ -------
246
+ pathlib.Path
247
+ Normalized path (preferring forward-slash fallback if it exists).
248
+ """
249
+ if isinstance(raw, pathlib.Path):
250
+ if raw.exists() or "\\" not in str(raw):
251
+ return raw
252
+ candidate = pathlib.Path(str(raw).replace("\\", "/"))
253
+ return candidate if candidate.exists() else raw
254
+
255
+ path = pathlib.Path(raw)
256
+ if "\\" in raw and not path.exists():
257
+ candidate = pathlib.Path(raw.replace("\\", "/"))
258
+ if candidate.exists():
259
+ return candidate
260
+ return path