odsbox-diff 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- odsbox_diff/__init__.py +32 -0
- odsbox_diff/__main__.py +5 -0
- odsbox_diff/api.py +245 -0
- odsbox_diff/connection/__init__.py +14 -0
- odsbox_diff/connection/config.py +96 -0
- odsbox_diff/connection/factory.py +52 -0
- odsbox_diff/connection/manager.py +150 -0
- odsbox_diff/diff.py +676 -0
- odsbox_diff/ods_diff_hierarchy/__init__.py +16 -0
- odsbox_diff/ods_diff_hierarchy/collect.py +639 -0
- odsbox_diff/ods_diff_hierarchy/diff.py +35 -0
- odsbox_diff/ods_diff_hierarchy/rel_to_name.py +74 -0
- odsbox_diff-1.0.0.dist-info/METADATA +274 -0
- odsbox_diff-1.0.0.dist-info/RECORD +17 -0
- odsbox_diff-1.0.0.dist-info/WHEEL +4 -0
- odsbox_diff-1.0.0.dist-info/entry_points.txt +3 -0
- odsbox_diff-1.0.0.dist-info/licenses/LICENSE +200 -0
odsbox_diff/diff.py
ADDED
|
@@ -0,0 +1,676 @@
|
|
|
1
|
+
"""diff tool to compare two tests, test steps or measurements"""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
import logging
|
|
6
|
+
import sys
|
|
7
|
+
from typing import Any, cast
|
|
8
|
+
|
|
9
|
+
import urllib3
|
|
10
|
+
|
|
11
|
+
from deepdiff import DeepDiff
|
|
12
|
+
|
|
13
|
+
from .connection import ServerConfig, create_connection, load_config
|
|
14
|
+
from .ods_diff_hierarchy.collect import collect, load_collect_results, save_collect_results
|
|
15
|
+
from .ods_diff_hierarchy.diff import diff_dictionaries, dump_diff_as_json
|
|
16
|
+
|
|
17
|
+
urllib3.disable_warnings()
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _parse_id_string(id_string: str | int, queries: list[dict[str, Any]] | None) -> int | dict[str, Any] | str:
|
|
21
|
+
if isinstance(id_string, str):
|
|
22
|
+
if id_string.isdigit():
|
|
23
|
+
return int(id_string)
|
|
24
|
+
|
|
25
|
+
query_string: str | None = None
|
|
26
|
+
if id_string.strip().startswith("{") and id_string.strip().endswith("}"):
|
|
27
|
+
query_string = id_string.strip()
|
|
28
|
+
else:
|
|
29
|
+
if queries:
|
|
30
|
+
for query in queries:
|
|
31
|
+
if query.get("name") == id_string:
|
|
32
|
+
query_string = cast(str, query.get("condition"))
|
|
33
|
+
if not query_string:
|
|
34
|
+
raise ValueError(f"ID string '{id_string}' is not a valid integer, JSON condition, or named query.")
|
|
35
|
+
|
|
36
|
+
try:
|
|
37
|
+
logging.warning("*** Parsing JSON condition string: %s", query_string)
|
|
38
|
+
if isinstance(query_string, dict):
|
|
39
|
+
return query_string
|
|
40
|
+
return json.loads(query_string) # type: ignore
|
|
41
|
+
except json.JSONDecodeError as e:
|
|
42
|
+
raise ValueError(f"Invalid JSON condition string: {id_string}") from e
|
|
43
|
+
elif isinstance(id_string, dict):
|
|
44
|
+
return id_string
|
|
45
|
+
else:
|
|
46
|
+
return int(id_string)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _parse_server_id(
|
|
50
|
+
id_str: str, servers: dict[str, ServerConfig], queries: list[dict[str, Any]] | None, multi_server: bool
|
|
51
|
+
) -> tuple[ServerConfig, int | str | dict[str, Any]]:
|
|
52
|
+
"""Parse 'server:id' or 'id' format, returning (ServerConfig, id_int).
|
|
53
|
+
|
|
54
|
+
Args:
|
|
55
|
+
id_str: String like '5' or 'prod:5'
|
|
56
|
+
servers: Available servers dict
|
|
57
|
+
multi_server: True if multiple servers configured
|
|
58
|
+
|
|
59
|
+
Raises:
|
|
60
|
+
ValueError: If format is invalid or server not found
|
|
61
|
+
"""
|
|
62
|
+
if ":" in id_str and not id_str.strip().startswith("{"):
|
|
63
|
+
server_name, id_part = id_str.split(":", 1)
|
|
64
|
+
if server_name not in servers:
|
|
65
|
+
raise ValueError(f"Server '{server_name}' not found. Available: {', '.join(servers.keys())}")
|
|
66
|
+
try:
|
|
67
|
+
instance_id = _parse_id_string(id_part, queries)
|
|
68
|
+
except ValueError:
|
|
69
|
+
raise ValueError(f"Invalid ID '{id_part}' after colon; must be an integer, JSON condition, or named query.")
|
|
70
|
+
return servers[server_name], instance_id
|
|
71
|
+
else:
|
|
72
|
+
if multi_server:
|
|
73
|
+
raise ValueError(
|
|
74
|
+
f"Multiple servers configured ({', '.join(servers.keys())}). "
|
|
75
|
+
f"Specify server as 'server:id' (e.g., 'prod:5')."
|
|
76
|
+
)
|
|
77
|
+
sole = next(iter(servers.values()))
|
|
78
|
+
try:
|
|
79
|
+
instance_id = _parse_id_string(id_str, queries)
|
|
80
|
+
except ValueError:
|
|
81
|
+
raise ValueError(f"Invalid ID '{id_str}'; must be an integer")
|
|
82
|
+
return sole, instance_id
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _parse_id_or_file(
|
|
86
|
+
id_str: str, servers: dict[str, ServerConfig], queries: list[dict[str, Any]] | None, multi_server: bool
|
|
87
|
+
) -> tuple[ServerConfig | None, int | str | dict[str, Any] | None, str | None]:
|
|
88
|
+
"""Parse an instance reference that may be a file path or a server:id.
|
|
89
|
+
|
|
90
|
+
Args:
|
|
91
|
+
id_str: One of ``"file:path.json"``, ``"42"``, or ``"server:42"``.
|
|
92
|
+
servers: Available servers dict (may be empty for file sources).
|
|
93
|
+
multi_server: True if multiple servers configured.
|
|
94
|
+
|
|
95
|
+
Returns:
|
|
96
|
+
``(None, None, file_path)`` when the input starts with ``file:``,
|
|
97
|
+
otherwise ``(ServerConfig, instance_id, None)`` via :func:`_parse_server_id`.
|
|
98
|
+
"""
|
|
99
|
+
if id_str.startswith("file:"):
|
|
100
|
+
return None, None, id_str[5:]
|
|
101
|
+
cfg, iid = _parse_server_id(id_str, servers, queries, multi_server)
|
|
102
|
+
return cfg, iid, None
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def diff_ods_tests(
|
|
106
|
+
server1_cfg: ServerConfig | None,
|
|
107
|
+
server2_cfg: ServerConfig | None,
|
|
108
|
+
entity_name: str,
|
|
109
|
+
inst1_condition: int | str | dict[str, Any] | None,
|
|
110
|
+
inst2_condition: int | str | dict[str, Any] | None,
|
|
111
|
+
result_file: str,
|
|
112
|
+
dump_dictionaries: bool,
|
|
113
|
+
exclude_regex_paths: list[str],
|
|
114
|
+
exclude_paths: list[str],
|
|
115
|
+
no_bulk: bool,
|
|
116
|
+
bulk_progress_bar: bool,
|
|
117
|
+
cached_related: list[str] | None = None,
|
|
118
|
+
file1_path: str | None = None,
|
|
119
|
+
file2_path: str | None = None,
|
|
120
|
+
) -> int:
|
|
121
|
+
"""Collect two ODS hierarchies and write a structural diff to ``result_file``.
|
|
122
|
+
|
|
123
|
+
Each side can be collected live from a server (when ``server*_cfg`` and
|
|
124
|
+
``inst*_id`` are given) or loaded from a previously saved JSON/ZIP file
|
|
125
|
+
(when ``file*_path`` is given).
|
|
126
|
+
|
|
127
|
+
Args:
|
|
128
|
+
server1_cfg: Configuration for the server hosting ``inst1_id``.
|
|
129
|
+
``None`` when loading side 1 from a file.
|
|
130
|
+
server2_cfg: Configuration for the server hosting ``inst2_id``.
|
|
131
|
+
``None`` when loading side 2 from a file. May be the same object
|
|
132
|
+
as ``server1_cfg`` to reuse a single connection.
|
|
133
|
+
entity_name: ODS entity name (e.g. ``"TestStep"``) of the root instances.
|
|
134
|
+
inst1_id: Instance ID on ``server1_cfg``. ``None`` when using a file.
|
|
135
|
+
inst2_id: Instance ID on ``server2_cfg``. ``None`` when using a file.
|
|
136
|
+
result_file: Path to write the diff JSON to. If empty, no file is written.
|
|
137
|
+
dump_dictionaries: Also write each collected hierarchy as
|
|
138
|
+
``<result_file>.inst1.json`` / ``.inst2.json``.
|
|
139
|
+
exclude_regex_paths: Extra regex patterns appended to the default exclusions.
|
|
140
|
+
exclude_paths: Extra explicit DeepDiff paths to exclude.
|
|
141
|
+
no_bulk: If ``True``, skip hashing of bulk LocalColumn data.
|
|
142
|
+
bulk_progress_bar: Show a textual progress bar during bulk hashing.
|
|
143
|
+
cached_related: Entity names whose IDs should be resolved to names in the
|
|
144
|
+
output for cleaner diffs.
|
|
145
|
+
file1_path: Path to a previously saved hierarchy JSON/ZIP for side 1.
|
|
146
|
+
When set, ``server1_cfg`` and ``inst1_id`` are ignored.
|
|
147
|
+
file2_path: Path to a previously saved hierarchy JSON/ZIP for side 2.
|
|
148
|
+
When set, ``server2_cfg`` and ``inst2_id`` are ignored.
|
|
149
|
+
|
|
150
|
+
Returns:
|
|
151
|
+
``0`` if no differences were found, ``100`` if differences were found.
|
|
152
|
+
"""
|
|
153
|
+
log = logging.getLogger(__name__)
|
|
154
|
+
log.info("Comparing '%s' id=%s vs id=%s", entity_name, inst1_condition or file1_path, inst2_condition or file2_path)
|
|
155
|
+
|
|
156
|
+
inst1_dict = None
|
|
157
|
+
inst2_dict = None
|
|
158
|
+
|
|
159
|
+
# --- Side 1 ---
|
|
160
|
+
if file1_path is not None:
|
|
161
|
+
log.info("[1/2] ------- Loading from file: %s", file1_path)
|
|
162
|
+
inst1_dict = load_collect_results(file1_path)
|
|
163
|
+
log.info("[1/2] ------- Loaded from file.")
|
|
164
|
+
# --- Side 2 ---
|
|
165
|
+
if file2_path is not None:
|
|
166
|
+
log.info("[2/2] ------- Loading from file: %s", file2_path)
|
|
167
|
+
inst2_dict = load_collect_results(file2_path)
|
|
168
|
+
log.info("[2/2] ------- Loaded from file.")
|
|
169
|
+
|
|
170
|
+
# --- Server-based collection for sides that are NOT file-based ---
|
|
171
|
+
if inst1_dict is None or inst2_dict is None:
|
|
172
|
+
# Determine which sides need server collection
|
|
173
|
+
need_1 = inst1_dict is None
|
|
174
|
+
need_2 = inst2_dict is None
|
|
175
|
+
same_server = need_1 and need_2 and server1_cfg is server2_cfg
|
|
176
|
+
|
|
177
|
+
if same_server:
|
|
178
|
+
assert server1_cfg is not None
|
|
179
|
+
log.info("Connecting to server: %s", server1_cfg.url)
|
|
180
|
+
with create_connection(server1_cfg) as con_i:
|
|
181
|
+
if need_1:
|
|
182
|
+
assert inst1_condition is not None
|
|
183
|
+
log.info("[1/2] ------- Collecting '%s' id=%s ...", entity_name, inst1_condition)
|
|
184
|
+
inst1_dict = collect(
|
|
185
|
+
con_i,
|
|
186
|
+
entity_name,
|
|
187
|
+
inst1_condition,
|
|
188
|
+
calculate_bulk_hash=not no_bulk,
|
|
189
|
+
show_progress=bulk_progress_bar,
|
|
190
|
+
cached_related_entities=cached_related,
|
|
191
|
+
)[0]
|
|
192
|
+
log.info("[1/2] ------- Finished collecting '%s' id=%s.", entity_name, inst1_condition)
|
|
193
|
+
if need_2:
|
|
194
|
+
assert inst2_condition is not None
|
|
195
|
+
log.info("[2/2] ------- Collecting '%s' id=%s ...", entity_name, inst2_condition)
|
|
196
|
+
inst2_dict = collect(
|
|
197
|
+
con_i,
|
|
198
|
+
entity_name,
|
|
199
|
+
inst2_condition,
|
|
200
|
+
calculate_bulk_hash=not no_bulk,
|
|
201
|
+
show_progress=bulk_progress_bar,
|
|
202
|
+
cached_related_entities=cached_related,
|
|
203
|
+
)[0]
|
|
204
|
+
log.info("[2/2] ------- Finished collecting '%s' id=%s.", entity_name, inst2_condition)
|
|
205
|
+
log.info("Connection closed")
|
|
206
|
+
else:
|
|
207
|
+
if need_1:
|
|
208
|
+
assert server1_cfg is not None
|
|
209
|
+
assert inst1_condition is not None
|
|
210
|
+
log.info("Connecting to server1: %s", server1_cfg.url)
|
|
211
|
+
with create_connection(server1_cfg) as con_i1:
|
|
212
|
+
log.info("[1/2] ------- Collecting '%s' id=%s ...", entity_name, inst1_condition)
|
|
213
|
+
inst1_dict = collect(
|
|
214
|
+
con_i1,
|
|
215
|
+
entity_name,
|
|
216
|
+
inst1_condition,
|
|
217
|
+
calculate_bulk_hash=not no_bulk,
|
|
218
|
+
show_progress=bulk_progress_bar,
|
|
219
|
+
cached_related_entities=cached_related,
|
|
220
|
+
)[0]
|
|
221
|
+
log.info("[1/2] ------- Finished collecting '%s' id=%s.", entity_name, inst1_condition)
|
|
222
|
+
log.info("Connection to server1 closed")
|
|
223
|
+
if need_2:
|
|
224
|
+
assert server2_cfg is not None
|
|
225
|
+
assert inst2_condition is not None
|
|
226
|
+
log.info("Connecting to server2: %s", server2_cfg.url)
|
|
227
|
+
with create_connection(server2_cfg) as con_i2:
|
|
228
|
+
log.info("[2/2] ------- Collecting '%s' id=%s ...", entity_name, inst2_condition)
|
|
229
|
+
inst2_dict = collect(
|
|
230
|
+
con_i2,
|
|
231
|
+
entity_name,
|
|
232
|
+
inst2_condition,
|
|
233
|
+
calculate_bulk_hash=not no_bulk,
|
|
234
|
+
show_progress=bulk_progress_bar,
|
|
235
|
+
cached_related_entities=cached_related,
|
|
236
|
+
)[0]
|
|
237
|
+
log.info("[2/2] ------- Finished collecting '%s' id=%s.", entity_name, inst2_condition)
|
|
238
|
+
log.info("Connection to server2 closed")
|
|
239
|
+
|
|
240
|
+
if dump_dictionaries and result_file is not None and "" != result_file:
|
|
241
|
+
log.info("Dumping collected dictionaries alongside result file")
|
|
242
|
+
if file1_path is None:
|
|
243
|
+
with open(f"{result_file}.inst1.json", "w", encoding="utf-8") as f:
|
|
244
|
+
json.dump(inst1_dict, f, indent=2, default=str)
|
|
245
|
+
if file2_path is None:
|
|
246
|
+
with open(f"{result_file}.inst2.json", "w", encoding="utf-8") as f:
|
|
247
|
+
json.dump(inst2_dict, f, indent=2, default=str)
|
|
248
|
+
|
|
249
|
+
assert inst1_dict is not None
|
|
250
|
+
assert inst2_dict is not None
|
|
251
|
+
log.info("Running diff ...")
|
|
252
|
+
diff_result: DeepDiff = diff_dictionaries(inst1_dict, inst2_dict, exclude_regex_paths, exclude_paths)
|
|
253
|
+
|
|
254
|
+
if not diff_result:
|
|
255
|
+
log.info("Result: no differences found")
|
|
256
|
+
else:
|
|
257
|
+
n = sum(len(v) if hasattr(v, "__len__") else 1 for v in diff_result.values())
|
|
258
|
+
log.info("Result: %s difference(s) found", n)
|
|
259
|
+
|
|
260
|
+
if result_file is not None and "" != result_file:
|
|
261
|
+
log.info("Writing result file: %s", result_file)
|
|
262
|
+
dump_diff_as_json(result_file, diff_result)
|
|
263
|
+
|
|
264
|
+
return 0 if not diff_result else 100
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
268
|
+
parser = argparse.ArgumentParser(
|
|
269
|
+
prog="odsbox-diff",
|
|
270
|
+
description="Compare two Hierarchy instances of an ASAM ODS server and write a difference result file.",
|
|
271
|
+
epilog="Returns 0 if no changes where found.",
|
|
272
|
+
)
|
|
273
|
+
parser.add_argument(
|
|
274
|
+
"-c",
|
|
275
|
+
"--config",
|
|
276
|
+
dest="config",
|
|
277
|
+
type=str,
|
|
278
|
+
required=True,
|
|
279
|
+
help="Path to TOML or JSON config file with server connection and default settings.",
|
|
280
|
+
)
|
|
281
|
+
parser.add_argument(
|
|
282
|
+
"-entity",
|
|
283
|
+
"--entity",
|
|
284
|
+
dest="entity_name",
|
|
285
|
+
type=str,
|
|
286
|
+
required=True,
|
|
287
|
+
nargs="?",
|
|
288
|
+
help="Entity to collect instance tree for.",
|
|
289
|
+
)
|
|
290
|
+
parser.add_argument(
|
|
291
|
+
"-id1",
|
|
292
|
+
"--inst1_id",
|
|
293
|
+
dest="inst1_id",
|
|
294
|
+
type=str,
|
|
295
|
+
required=True,
|
|
296
|
+
nargs="?",
|
|
297
|
+
help="Instance ID ('42'), 'server:id' ('prod:5'), or 'file:path.json' to load from disk.",
|
|
298
|
+
)
|
|
299
|
+
parser.add_argument(
|
|
300
|
+
"-id2",
|
|
301
|
+
"--inst2_id",
|
|
302
|
+
dest="inst2_id",
|
|
303
|
+
type=str,
|
|
304
|
+
required=True,
|
|
305
|
+
nargs="?",
|
|
306
|
+
help="Instance ID ('42'), 'server:id' ('staging:5'), or 'file:path.json' to load from disk.",
|
|
307
|
+
)
|
|
308
|
+
parser.add_argument(
|
|
309
|
+
"-rf",
|
|
310
|
+
"--result_file",
|
|
311
|
+
dest="result_file",
|
|
312
|
+
help="File storing the results if Tests differs. Overrides config default.",
|
|
313
|
+
default=None,
|
|
314
|
+
)
|
|
315
|
+
parser.add_argument(
|
|
316
|
+
"-ep",
|
|
317
|
+
"--exclude_path",
|
|
318
|
+
dest="exclude_paths",
|
|
319
|
+
type=str,
|
|
320
|
+
action="append",
|
|
321
|
+
help="Add path to exclude from diff. Can be used multiple times. Extends config defaults.",
|
|
322
|
+
)
|
|
323
|
+
parser.add_argument(
|
|
324
|
+
"-erp",
|
|
325
|
+
"--exclude_regex_path",
|
|
326
|
+
dest="exclude_regex_paths",
|
|
327
|
+
type=str,
|
|
328
|
+
action="append",
|
|
329
|
+
help="Add regex to exclude paths from diff. Can be used multiple times. Extends config defaults.",
|
|
330
|
+
)
|
|
331
|
+
parser.add_argument(
|
|
332
|
+
"-dd",
|
|
333
|
+
"--dump_dictionaries",
|
|
334
|
+
dest="dump_dictionaries",
|
|
335
|
+
action="store_true",
|
|
336
|
+
default=None,
|
|
337
|
+
help="Dump collected dictionaries to JSON files alongside the result file.",
|
|
338
|
+
)
|
|
339
|
+
parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", default=None)
|
|
340
|
+
parser.add_argument(
|
|
341
|
+
"-q",
|
|
342
|
+
"--quiet",
|
|
343
|
+
dest="quiet",
|
|
344
|
+
action="store_true",
|
|
345
|
+
default=None,
|
|
346
|
+
help="Suppress all output.",
|
|
347
|
+
)
|
|
348
|
+
parser.add_argument(
|
|
349
|
+
"-bn",
|
|
350
|
+
"--no_bulk",
|
|
351
|
+
dest="no_bulk",
|
|
352
|
+
action="store_true",
|
|
353
|
+
default=None,
|
|
354
|
+
help="If given the bulk values are not hashed.",
|
|
355
|
+
)
|
|
356
|
+
parser.add_argument(
|
|
357
|
+
"-bpb",
|
|
358
|
+
"--bulk_progress_bar",
|
|
359
|
+
dest="bulk_progress_bar",
|
|
360
|
+
action="store_true",
|
|
361
|
+
default=None,
|
|
362
|
+
help="Show a progress bar while calculating bulk hash values.",
|
|
363
|
+
)
|
|
364
|
+
parser.add_argument(
|
|
365
|
+
"--cached-related",
|
|
366
|
+
dest="cached_related",
|
|
367
|
+
type=str,
|
|
368
|
+
nargs="+",
|
|
369
|
+
default=None,
|
|
370
|
+
metavar="ENTITY",
|
|
371
|
+
help="Entity names whose IDs are resolved to names in the diff output (e.g. AoUnit Classification). Extends config defaults.",
|
|
372
|
+
)
|
|
373
|
+
return parser
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def cli() -> None:
|
|
377
|
+
"""Console script entry point for ``uv run odsbox-diff``.
|
|
378
|
+
|
|
379
|
+
Parses CLI arguments, loads the config file, applies CLI-over-config
|
|
380
|
+
precedence to all options, resolves ``server:id`` instance references and
|
|
381
|
+
delegates to :func:`diff_ods_tests`. Exits with the diff return code
|
|
382
|
+
(``0`` no differences, ``100`` differences found, ``-1`` on uncaught
|
|
383
|
+
exception, ``1`` on argument validation errors).
|
|
384
|
+
"""
|
|
385
|
+
# Dispatch to collect subcommand if requested
|
|
386
|
+
if len(sys.argv) > 1 and sys.argv[1] == "collect":
|
|
387
|
+
_cli_collect(sys.argv[2:])
|
|
388
|
+
return
|
|
389
|
+
|
|
390
|
+
parser = _build_parser()
|
|
391
|
+
args = parser.parse_args()
|
|
392
|
+
|
|
393
|
+
# Load config (connection + defaults)
|
|
394
|
+
app_config = load_config(args.config)
|
|
395
|
+
defaults = app_config.defaults
|
|
396
|
+
|
|
397
|
+
# CLI-over-config precedence: explicit CLI flags override config defaults
|
|
398
|
+
verbose = args.verbose if args.verbose is not None else defaults.verbose
|
|
399
|
+
quiet = args.quiet if args.quiet is not None else defaults.quiet
|
|
400
|
+
if quiet:
|
|
401
|
+
logging.disable(logging.CRITICAL)
|
|
402
|
+
elif verbose:
|
|
403
|
+
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
|
|
404
|
+
else:
|
|
405
|
+
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
|
406
|
+
|
|
407
|
+
result_file = args.result_file if args.result_file is not None else defaults.result_file
|
|
408
|
+
dump_dicts = args.dump_dictionaries if args.dump_dictionaries is not None else defaults.dump_dictionaries
|
|
409
|
+
no_bulk = args.no_bulk if args.no_bulk is not None else defaults.no_bulk
|
|
410
|
+
bulk_progress_bar = args.bulk_progress_bar if args.bulk_progress_bar is not None else defaults.bulk_progress_bar
|
|
411
|
+
|
|
412
|
+
# Extend config defaults with any extra CLI exclusions
|
|
413
|
+
exclude_regex_paths = list(defaults.exclude_regex_paths)
|
|
414
|
+
if args.exclude_regex_paths:
|
|
415
|
+
exclude_regex_paths.extend(args.exclude_regex_paths)
|
|
416
|
+
|
|
417
|
+
exclude_paths = list(defaults.exclude_paths)
|
|
418
|
+
if args.exclude_paths:
|
|
419
|
+
exclude_paths.extend(args.exclude_paths)
|
|
420
|
+
|
|
421
|
+
cached_related = list(defaults.cached_related)
|
|
422
|
+
if args.cached_related:
|
|
423
|
+
cached_related.extend(args.cached_related)
|
|
424
|
+
|
|
425
|
+
log = logging.getLogger(__name__)
|
|
426
|
+
|
|
427
|
+
# Resolve server configs and instance IDs from 'server:id' or 'file:path' format
|
|
428
|
+
servers = app_config.servers
|
|
429
|
+
multi_server = len(servers) > 1
|
|
430
|
+
queries = app_config.queries
|
|
431
|
+
|
|
432
|
+
try:
|
|
433
|
+
server1_cfg, inst1_condition, file1_path = _parse_id_or_file(args.inst1_id, servers, queries, multi_server)
|
|
434
|
+
server2_cfg, inst2_condition, file2_path = _parse_id_or_file(args.inst2_id, servers, queries, multi_server)
|
|
435
|
+
except ValueError as e:
|
|
436
|
+
log.error("%s", e)
|
|
437
|
+
sys.exit(1)
|
|
438
|
+
|
|
439
|
+
try:
|
|
440
|
+
return_value = diff_ods_tests(
|
|
441
|
+
server1_cfg=server1_cfg,
|
|
442
|
+
server2_cfg=server2_cfg,
|
|
443
|
+
entity_name=args.entity_name,
|
|
444
|
+
inst1_condition=inst1_condition,
|
|
445
|
+
inst2_condition=inst2_condition,
|
|
446
|
+
result_file=result_file,
|
|
447
|
+
dump_dictionaries=dump_dicts,
|
|
448
|
+
exclude_regex_paths=exclude_regex_paths,
|
|
449
|
+
exclude_paths=exclude_paths,
|
|
450
|
+
no_bulk=no_bulk,
|
|
451
|
+
bulk_progress_bar=bulk_progress_bar,
|
|
452
|
+
cached_related=cached_related,
|
|
453
|
+
file1_path=file1_path,
|
|
454
|
+
file2_path=file2_path,
|
|
455
|
+
)
|
|
456
|
+
log.info("Finished with result code: %s", return_value)
|
|
457
|
+
sys.exit(return_value)
|
|
458
|
+
except Exception as e:
|
|
459
|
+
log.exception("Exception: %s", e)
|
|
460
|
+
sys.exit(-1)
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def collect_ods_test(
|
|
464
|
+
server_cfg: ServerConfig,
|
|
465
|
+
entity_name: str,
|
|
466
|
+
inst_id: int | str | dict[str, Any],
|
|
467
|
+
output_file: str,
|
|
468
|
+
no_bulk: bool,
|
|
469
|
+
bulk_progress_bar: bool,
|
|
470
|
+
cached_related: list[str] | None = None,
|
|
471
|
+
validate: bool = False,
|
|
472
|
+
validate_result_file: str = "collect_validate_result.json",
|
|
473
|
+
) -> int:
|
|
474
|
+
"""Collect an ODS hierarchy and save it to a file.
|
|
475
|
+
|
|
476
|
+
Optionally performs a round-trip validation by reloading the file and
|
|
477
|
+
diffing it against the in-memory data.
|
|
478
|
+
|
|
479
|
+
Args:
|
|
480
|
+
server_cfg: Configuration for the server to collect from.
|
|
481
|
+
entity_name: ODS entity name (e.g. ``"TestStep"``).
|
|
482
|
+
inst_id: Instance ID to collect.
|
|
483
|
+
output_file: Path to write the collected hierarchy (``.json`` or ``.zip``).
|
|
484
|
+
no_bulk: If ``True``, skip hashing of bulk LocalColumn data.
|
|
485
|
+
bulk_progress_bar: Show a progress bar during bulk hashing.
|
|
486
|
+
cached_related: Entity names whose IDs should be resolved to names.
|
|
487
|
+
validate: If ``True``, reload the saved file and self-diff to verify
|
|
488
|
+
round-trip fidelity.
|
|
489
|
+
validate_result_file: Path to write the self-diff result when
|
|
490
|
+
``validate=True``.
|
|
491
|
+
|
|
492
|
+
Returns:
|
|
493
|
+
``0`` if successful (or self-diff found no differences),
|
|
494
|
+
``100`` if self-diff found unexpected differences.
|
|
495
|
+
"""
|
|
496
|
+
log = logging.getLogger(__name__)
|
|
497
|
+
log.info("Collecting '%s' id=%s from %s", entity_name, inst_id, server_cfg.url)
|
|
498
|
+
|
|
499
|
+
with create_connection(server_cfg) as con_i:
|
|
500
|
+
result_dict = collect(
|
|
501
|
+
con_i,
|
|
502
|
+
entity_name,
|
|
503
|
+
inst_id,
|
|
504
|
+
calculate_bulk_hash=not no_bulk,
|
|
505
|
+
show_progress=bulk_progress_bar,
|
|
506
|
+
cached_related_entities=cached_related,
|
|
507
|
+
)[0]
|
|
508
|
+
log.info("Connection closed")
|
|
509
|
+
|
|
510
|
+
log.info("Saving collected hierarchy to: %s", output_file)
|
|
511
|
+
save_collect_results(output_file, result_dict)
|
|
512
|
+
|
|
513
|
+
if not validate:
|
|
514
|
+
return 0
|
|
515
|
+
|
|
516
|
+
log.info("Validating round-trip fidelity ...")
|
|
517
|
+
reloaded = load_collect_results(output_file)
|
|
518
|
+
diff_result: DeepDiff = diff_dictionaries(reloaded, result_dict, [], [])
|
|
519
|
+
|
|
520
|
+
if not diff_result:
|
|
521
|
+
log.info("Validation passed: no differences after round-trip.")
|
|
522
|
+
return 0
|
|
523
|
+
else:
|
|
524
|
+
n = sum(len(v) if hasattr(v, "__len__") else 1 for v in diff_result.values())
|
|
525
|
+
log.warning("Validation found %s unexpected difference(s)!", n)
|
|
526
|
+
dump_diff_as_json(validate_result_file, diff_result)
|
|
527
|
+
log.info("Self-diff written to: %s", validate_result_file)
|
|
528
|
+
return 100
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
def _build_collect_parser() -> argparse.ArgumentParser:
|
|
532
|
+
parser = argparse.ArgumentParser(
|
|
533
|
+
prog="odsbox-diff collect",
|
|
534
|
+
description="Collect an ODS instance hierarchy and save it to a JSON or ZIP file.",
|
|
535
|
+
)
|
|
536
|
+
parser.add_argument(
|
|
537
|
+
"-c",
|
|
538
|
+
"--config",
|
|
539
|
+
dest="config",
|
|
540
|
+
type=str,
|
|
541
|
+
required=True,
|
|
542
|
+
help="Path to TOML or JSON config file with server connection and default settings.",
|
|
543
|
+
)
|
|
544
|
+
parser.add_argument(
|
|
545
|
+
"-entity",
|
|
546
|
+
"--entity",
|
|
547
|
+
dest="entity_name",
|
|
548
|
+
type=str,
|
|
549
|
+
required=True,
|
|
550
|
+
help="Entity to collect instance tree for.",
|
|
551
|
+
)
|
|
552
|
+
parser.add_argument(
|
|
553
|
+
"-id",
|
|
554
|
+
"--inst_id",
|
|
555
|
+
dest="inst_id",
|
|
556
|
+
type=str,
|
|
557
|
+
required=True,
|
|
558
|
+
help="Instance ID or 'server:id' (e.g. 'prod:42').",
|
|
559
|
+
)
|
|
560
|
+
parser.add_argument(
|
|
561
|
+
"-o",
|
|
562
|
+
"--output",
|
|
563
|
+
dest="output",
|
|
564
|
+
type=str,
|
|
565
|
+
required=True,
|
|
566
|
+
help="Output file path (.json or .zip).",
|
|
567
|
+
)
|
|
568
|
+
parser.add_argument(
|
|
569
|
+
"--validate",
|
|
570
|
+
dest="validate",
|
|
571
|
+
action="store_true",
|
|
572
|
+
default=False,
|
|
573
|
+
help="After saving, reload and self-diff to verify round-trip fidelity.",
|
|
574
|
+
)
|
|
575
|
+
parser.add_argument(
|
|
576
|
+
"-rf",
|
|
577
|
+
"--result_file",
|
|
578
|
+
dest="result_file",
|
|
579
|
+
help="File storing the self-diff result when --validate is used.",
|
|
580
|
+
default="collect_validate_result.json",
|
|
581
|
+
)
|
|
582
|
+
parser.add_argument(
|
|
583
|
+
"-bn",
|
|
584
|
+
"--no_bulk",
|
|
585
|
+
dest="no_bulk",
|
|
586
|
+
action="store_true",
|
|
587
|
+
default=None,
|
|
588
|
+
help="If given the bulk values are not hashed.",
|
|
589
|
+
)
|
|
590
|
+
parser.add_argument(
|
|
591
|
+
"-bpb",
|
|
592
|
+
"--bulk_progress_bar",
|
|
593
|
+
dest="bulk_progress_bar",
|
|
594
|
+
action="store_true",
|
|
595
|
+
default=None,
|
|
596
|
+
help="Show a progress bar while calculating bulk hash values.",
|
|
597
|
+
)
|
|
598
|
+
parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", default=None)
|
|
599
|
+
parser.add_argument(
|
|
600
|
+
"-q",
|
|
601
|
+
"--quiet",
|
|
602
|
+
dest="quiet",
|
|
603
|
+
action="store_true",
|
|
604
|
+
default=None,
|
|
605
|
+
help="Suppress all output.",
|
|
606
|
+
)
|
|
607
|
+
parser.add_argument(
|
|
608
|
+
"--cached-related",
|
|
609
|
+
dest="cached_related",
|
|
610
|
+
type=str,
|
|
611
|
+
nargs="+",
|
|
612
|
+
default=None,
|
|
613
|
+
metavar="ENTITY",
|
|
614
|
+
help="Entity names whose IDs are resolved to names in the output.",
|
|
615
|
+
)
|
|
616
|
+
return parser
|
|
617
|
+
|
|
618
|
+
|
|
619
|
+
def _cli_collect(raw_args: list[str]) -> None:
|
|
620
|
+
"""Parse and execute the ``odsbox-diff collect`` subcommand."""
|
|
621
|
+
parser = _build_collect_parser()
|
|
622
|
+
args = parser.parse_args(raw_args)
|
|
623
|
+
|
|
624
|
+
app_config = load_config(args.config)
|
|
625
|
+
defaults = app_config.defaults
|
|
626
|
+
|
|
627
|
+
verbose = args.verbose if args.verbose is not None else defaults.verbose
|
|
628
|
+
quiet = args.quiet if args.quiet is not None else defaults.quiet
|
|
629
|
+
if quiet:
|
|
630
|
+
logging.disable(logging.CRITICAL)
|
|
631
|
+
elif verbose:
|
|
632
|
+
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
|
|
633
|
+
else:
|
|
634
|
+
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
|
635
|
+
|
|
636
|
+
no_bulk = args.no_bulk if args.no_bulk is not None else defaults.no_bulk
|
|
637
|
+
bulk_progress_bar = args.bulk_progress_bar if args.bulk_progress_bar is not None else defaults.bulk_progress_bar
|
|
638
|
+
|
|
639
|
+
cached_related = list(defaults.cached_related)
|
|
640
|
+
if args.cached_related:
|
|
641
|
+
cached_related.extend(args.cached_related)
|
|
642
|
+
|
|
643
|
+
log = logging.getLogger(__name__)
|
|
644
|
+
|
|
645
|
+
servers = app_config.servers
|
|
646
|
+
multi_server = len(servers) > 1
|
|
647
|
+
|
|
648
|
+
queries = app_config.queries
|
|
649
|
+
|
|
650
|
+
try:
|
|
651
|
+
server_cfg, inst_id = _parse_server_id(args.inst_id, servers, queries, multi_server)
|
|
652
|
+
except ValueError as e:
|
|
653
|
+
log.error("%s", e)
|
|
654
|
+
sys.exit(1)
|
|
655
|
+
|
|
656
|
+
try:
|
|
657
|
+
return_value = collect_ods_test(
|
|
658
|
+
server_cfg=server_cfg,
|
|
659
|
+
entity_name=args.entity_name,
|
|
660
|
+
inst_id=inst_id,
|
|
661
|
+
output_file=args.output,
|
|
662
|
+
no_bulk=no_bulk,
|
|
663
|
+
bulk_progress_bar=bulk_progress_bar,
|
|
664
|
+
cached_related=cached_related,
|
|
665
|
+
validate=args.validate,
|
|
666
|
+
validate_result_file=args.result_file,
|
|
667
|
+
)
|
|
668
|
+
log.info("Finished with result code: %s", return_value)
|
|
669
|
+
sys.exit(return_value)
|
|
670
|
+
except Exception as e:
|
|
671
|
+
log.exception("Exception: %s", e)
|
|
672
|
+
sys.exit(-1)
|
|
673
|
+
|
|
674
|
+
|
|
675
|
+
if __name__ == "__main__":
|
|
676
|
+
cli()
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""ods_diff_hierarchy"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from .collect import collect, load_collect_results, save_collect_results
|
|
6
|
+
from .diff import diff_dictionaries, dump_diff_as_json
|
|
7
|
+
|
|
8
|
+
__version__ = "0.1.0"
|
|
9
|
+
|
|
10
|
+
__all__ = [
|
|
11
|
+
"collect",
|
|
12
|
+
"load_collect_results",
|
|
13
|
+
"save_collect_results",
|
|
14
|
+
"diff_dictionaries",
|
|
15
|
+
"dump_diff_as_json",
|
|
16
|
+
]
|