reqtable 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- reqtable/__init__.py +3 -0
- reqtable/__main__.py +3 -0
- reqtable/baselines.py +920 -0
- reqtable/cli.py +631 -0
- reqtable/comparison.py +63 -0
- reqtable/core.py +1448 -0
- reqtable/drafts.py +166 -0
- reqtable/findings.py +605 -0
- reqtable/gate_checks.py +84 -0
- reqtable/gates.py +348 -0
- reqtable/glossary.py +162 -0
- reqtable/gui.py +952 -0
- reqtable/gui_lookups.py +192 -0
- reqtable/gui_mapping.py +789 -0
- reqtable/gui_page.py +691 -0
- reqtable/gui_page_gates.py +173 -0
- reqtable/gui_page_mapping.py +390 -0
- reqtable/gui_server.py +467 -0
- reqtable/html_report.py +2595 -0
- reqtable/markdown.py +127 -0
- reqtable/package.py +163 -0
- reqtable/quality.py +422 -0
- reqtable/review.py +318 -0
- reqtable/signoff.py +179 -0
- reqtable/traceability.py +1670 -0
- reqtable/values.py +192 -0
- reqtable/waivers.py +109 -0
- reqtable/xlsx.py +1108 -0
- reqtable-0.1.0.dist-info/METADATA +106 -0
- reqtable-0.1.0.dist-info/RECORD +33 -0
- reqtable-0.1.0.dist-info/WHEEL +4 -0
- reqtable-0.1.0.dist-info/entry_points.txt +2 -0
- reqtable-0.1.0.dist-info/licenses/LICENSE +21 -0
reqtable/core.py
ADDED
|
@@ -0,0 +1,1448 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import csv
|
|
4
|
+
import difflib
|
|
5
|
+
import html
|
|
6
|
+
import io
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import re
|
|
10
|
+
import tempfile
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from datetime import datetime, timezone
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any, Iterable, Sequence
|
|
15
|
+
|
|
16
|
+
from .markdown import render_markdown
|
|
17
|
+
from .traceability import TRACE_COLUMNS
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class ReqtableError(Exception):
|
|
21
|
+
"""Base class for actionable application errors."""
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class ConfigError(ReqtableError):
|
|
25
|
+
pass
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class InputError(ReqtableError):
|
|
29
|
+
pass
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class OutputError(ReqtableError):
|
|
33
|
+
pass
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
RECORD_TYPES = (
|
|
37
|
+
"System",
|
|
38
|
+
"Requirement",
|
|
39
|
+
"Need",
|
|
40
|
+
"Source",
|
|
41
|
+
"Term",
|
|
42
|
+
"Value",
|
|
43
|
+
"Field",
|
|
44
|
+
"Test",
|
|
45
|
+
"Test plan",
|
|
46
|
+
"Change request",
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _preset_gate(
|
|
51
|
+
id: str,
|
|
52
|
+
name: str,
|
|
53
|
+
produces: str,
|
|
54
|
+
checks: list[dict[str, Any]],
|
|
55
|
+
reviewers: int = 2,
|
|
56
|
+
**extra: Any,
|
|
57
|
+
) -> dict[str, Any]:
|
|
58
|
+
return {
|
|
59
|
+
"id": id,
|
|
60
|
+
"name": name,
|
|
61
|
+
"produces": produces,
|
|
62
|
+
"scope": "table",
|
|
63
|
+
"includes": ["Requirement", "System"],
|
|
64
|
+
"captures_status": False,
|
|
65
|
+
"checks": checks,
|
|
66
|
+
"signoff": [
|
|
67
|
+
{"role": "Reviewer", "min": reviewers},
|
|
68
|
+
{"role": "Approver", "min": 1},
|
|
69
|
+
],
|
|
70
|
+
**extra,
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _check(id: str, description: str, **condition: Any) -> dict[str, Any]:
|
|
75
|
+
return {"id": id, "description": description, **condition}
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _no_findings(prefix: str) -> dict[str, Any]:
|
|
79
|
+
return _check(
|
|
80
|
+
f"{prefix}-no-findings", "There are no trace findings", no_findings=True
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _change_control(prefix: str) -> dict[str, Any]:
|
|
85
|
+
return _check(
|
|
86
|
+
f"{prefix}-change-control",
|
|
87
|
+
"Every change since the last baseline has an approved change request",
|
|
88
|
+
unchanged_since=True,
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
_COMPLIANT = {"Compliance": ["Compliant", "Waived"]}
|
|
93
|
+
_TESTS = {"type": "Test"}
|
|
94
|
+
|
|
95
|
+
PRESET_GATES = [
|
|
96
|
+
_preset_gate(
|
|
97
|
+
"SRR",
|
|
98
|
+
"System Requirements Review",
|
|
99
|
+
"Functional baseline",
|
|
100
|
+
[
|
|
101
|
+
_check(
|
|
102
|
+
"srr-text",
|
|
103
|
+
"Every requirement has its requirement text",
|
|
104
|
+
filled="Requirement",
|
|
105
|
+
),
|
|
106
|
+
_check(
|
|
107
|
+
"srr-method",
|
|
108
|
+
"Every requirement has a verification method",
|
|
109
|
+
filled="Verification Method",
|
|
110
|
+
),
|
|
111
|
+
_check(
|
|
112
|
+
"srr-priority", "Every requirement has a priority", filled="Priority"
|
|
113
|
+
),
|
|
114
|
+
_no_findings("srr"),
|
|
115
|
+
],
|
|
116
|
+
),
|
|
117
|
+
_preset_gate(
|
|
118
|
+
"SFR",
|
|
119
|
+
"System Functional Review",
|
|
120
|
+
"Functional baseline",
|
|
121
|
+
[
|
|
122
|
+
_check(
|
|
123
|
+
"sfr-allocated",
|
|
124
|
+
"Every requirement is allocated to a system",
|
|
125
|
+
linked="Allocated to",
|
|
126
|
+
),
|
|
127
|
+
_no_findings("sfr"),
|
|
128
|
+
_change_control("sfr"),
|
|
129
|
+
],
|
|
130
|
+
),
|
|
131
|
+
_preset_gate(
|
|
132
|
+
"PDR",
|
|
133
|
+
"Preliminary Design Review",
|
|
134
|
+
"Allocated baseline",
|
|
135
|
+
[
|
|
136
|
+
_check(
|
|
137
|
+
"pdr-allocated-leaf",
|
|
138
|
+
"Every requirement is allocated to a leaf system",
|
|
139
|
+
linked="Allocated to",
|
|
140
|
+
leaf=True,
|
|
141
|
+
),
|
|
142
|
+
_check(
|
|
143
|
+
"pdr-level",
|
|
144
|
+
"Every requirement has a verification level",
|
|
145
|
+
filled="Verification Level",
|
|
146
|
+
),
|
|
147
|
+
_no_findings("pdr"),
|
|
148
|
+
_change_control("pdr"),
|
|
149
|
+
],
|
|
150
|
+
),
|
|
151
|
+
_preset_gate(
|
|
152
|
+
"DDR",
|
|
153
|
+
"Detailed Design Review",
|
|
154
|
+
"Subsystem design baseline",
|
|
155
|
+
[
|
|
156
|
+
_check(
|
|
157
|
+
"ddr-verified",
|
|
158
|
+
"Every requirement is linked to a verifying test",
|
|
159
|
+
linked="Verified by",
|
|
160
|
+
),
|
|
161
|
+
_no_findings("ddr"),
|
|
162
|
+
_change_control("ddr"),
|
|
163
|
+
],
|
|
164
|
+
reviewers=1,
|
|
165
|
+
scope="system",
|
|
166
|
+
),
|
|
167
|
+
_preset_gate(
|
|
168
|
+
"CDR",
|
|
169
|
+
"Critical Design Review",
|
|
170
|
+
"Product baseline",
|
|
171
|
+
[
|
|
172
|
+
_check(
|
|
173
|
+
"cdr-verified",
|
|
174
|
+
"Every requirement is linked to a verifying test",
|
|
175
|
+
linked="Verified by",
|
|
176
|
+
),
|
|
177
|
+
_no_findings("cdr"),
|
|
178
|
+
_change_control("cdr"),
|
|
179
|
+
],
|
|
180
|
+
),
|
|
181
|
+
_preset_gate(
|
|
182
|
+
"TRR",
|
|
183
|
+
"Test Readiness Review",
|
|
184
|
+
"Test baseline",
|
|
185
|
+
[
|
|
186
|
+
_check(
|
|
187
|
+
"trr-test-plan",
|
|
188
|
+
"Every test belongs to a test plan",
|
|
189
|
+
applies_to=_TESTS,
|
|
190
|
+
linked="Test plan",
|
|
191
|
+
),
|
|
192
|
+
_check(
|
|
193
|
+
"trr-test-method",
|
|
194
|
+
"Every test has a verification method",
|
|
195
|
+
applies_to=_TESTS,
|
|
196
|
+
filled="Verification Method",
|
|
197
|
+
),
|
|
198
|
+
_no_findings("trr"),
|
|
199
|
+
_change_control("trr"),
|
|
200
|
+
],
|
|
201
|
+
includes=["Requirement", "System", "Test", "Test plan"],
|
|
202
|
+
),
|
|
203
|
+
_preset_gate(
|
|
204
|
+
"SVR",
|
|
205
|
+
"System Verification Review",
|
|
206
|
+
"Verified baseline",
|
|
207
|
+
[
|
|
208
|
+
_check(
|
|
209
|
+
"svr-results",
|
|
210
|
+
"Every test has passed",
|
|
211
|
+
applies_to=_TESTS,
|
|
212
|
+
**{"in": {"Result": ["Passed"]}},
|
|
213
|
+
),
|
|
214
|
+
_check(
|
|
215
|
+
"svr-compliance",
|
|
216
|
+
"Every requirement is compliant or waived",
|
|
217
|
+
**{"in": _COMPLIANT},
|
|
218
|
+
),
|
|
219
|
+
_no_findings("svr"),
|
|
220
|
+
_change_control("svr"),
|
|
221
|
+
],
|
|
222
|
+
captures_status=True,
|
|
223
|
+
),
|
|
224
|
+
_preset_gate(
|
|
225
|
+
"FCA",
|
|
226
|
+
"Functional Configuration Audit",
|
|
227
|
+
"Functional audit baseline",
|
|
228
|
+
[
|
|
229
|
+
_check(
|
|
230
|
+
"fca-compliance",
|
|
231
|
+
"Every requirement is compliant or waived",
|
|
232
|
+
**{"in": _COMPLIANT},
|
|
233
|
+
),
|
|
234
|
+
_no_findings("fca"),
|
|
235
|
+
_change_control("fca"),
|
|
236
|
+
],
|
|
237
|
+
captures_status=True,
|
|
238
|
+
),
|
|
239
|
+
_preset_gate(
|
|
240
|
+
"PCA",
|
|
241
|
+
"Physical Configuration Audit",
|
|
242
|
+
"As-built product baseline",
|
|
243
|
+
[
|
|
244
|
+
_check(
|
|
245
|
+
"pca-compliance",
|
|
246
|
+
"Every requirement is compliant or waived",
|
|
247
|
+
**{"in": _COMPLIANT},
|
|
248
|
+
),
|
|
249
|
+
_no_findings("pca"),
|
|
250
|
+
_change_control("pca"),
|
|
251
|
+
],
|
|
252
|
+
captures_status=True,
|
|
253
|
+
),
|
|
254
|
+
]
|
|
255
|
+
|
|
256
|
+
DEFAULT_CONFIG: dict[str, Any] = {
|
|
257
|
+
"id_column": None,
|
|
258
|
+
"title": "Requirements",
|
|
259
|
+
"document_mode": "review",
|
|
260
|
+
"traceability": False,
|
|
261
|
+
"csv": {"encoding": "utf-8-sig", "delimiter": ",", "strict_columns": True},
|
|
262
|
+
"columns": {
|
|
263
|
+
"hidden": [],
|
|
264
|
+
"order": [],
|
|
265
|
+
"labels": {},
|
|
266
|
+
"widths": {},
|
|
267
|
+
"markdown": None,
|
|
268
|
+
},
|
|
269
|
+
"diff": {
|
|
270
|
+
"initial_view": "unified",
|
|
271
|
+
"initial_changes_only": False,
|
|
272
|
+
"include_unchanged": True,
|
|
273
|
+
"ignore_columns": [],
|
|
274
|
+
"ignore_trailing_whitespace": False,
|
|
275
|
+
},
|
|
276
|
+
"comments": {
|
|
277
|
+
"enabled": True,
|
|
278
|
+
"reviewer_initials": None,
|
|
279
|
+
"csv_column": "Comments",
|
|
280
|
+
"csv_format": "json",
|
|
281
|
+
},
|
|
282
|
+
"quality": {
|
|
283
|
+
"requirement_column": None,
|
|
284
|
+
"required_columns": [],
|
|
285
|
+
"obligation": "shall",
|
|
286
|
+
},
|
|
287
|
+
"xlsx": {"sources": []},
|
|
288
|
+
"baselines": {
|
|
289
|
+
"store": "git",
|
|
290
|
+
"git": {"tag_prefix": "baseline/"},
|
|
291
|
+
"files": {"directory": "baselines"},
|
|
292
|
+
"enforcement": "advisory",
|
|
293
|
+
"status_columns": ["Compliance", "Result"],
|
|
294
|
+
"change_control": True,
|
|
295
|
+
},
|
|
296
|
+
"gates": PRESET_GATES,
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
@dataclass(frozen=True)
|
|
301
|
+
class CsvSource:
|
|
302
|
+
"""A CSV to read: a file on disk (text is None) or already-read text with a display label."""
|
|
303
|
+
|
|
304
|
+
label: str
|
|
305
|
+
text: str | None
|
|
306
|
+
path: Path
|
|
307
|
+
warnings: tuple[str, ...] = ()
|
|
308
|
+
comments: tuple[dict[str, Any], ...] = ()
|
|
309
|
+
|
|
310
|
+
@property
|
|
311
|
+
def filename(self) -> str:
|
|
312
|
+
return self.path.name if self.text is None else self.label
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
@dataclass
|
|
316
|
+
class CsvTable:
|
|
317
|
+
label: str
|
|
318
|
+
filename: str
|
|
319
|
+
headers: list[str]
|
|
320
|
+
rows: list[dict[str, str]]
|
|
321
|
+
ids: list[str]
|
|
322
|
+
by_id: dict[str, dict[str, str]]
|
|
323
|
+
warnings: tuple[str, ...] = ()
|
|
324
|
+
comments: tuple[dict[str, Any], ...] = ()
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def _merge_known(
|
|
328
|
+
base: dict[str, Any], override: dict[str, Any], prefix: str = ""
|
|
329
|
+
) -> dict[str, Any]:
|
|
330
|
+
result = json.loads(json.dumps(base))
|
|
331
|
+
for key, value in override.items():
|
|
332
|
+
where = f"{prefix}.{key}" if prefix else key
|
|
333
|
+
if key not in base:
|
|
334
|
+
raise ConfigError(f"unknown configuration key: {where}")
|
|
335
|
+
if isinstance(base[key], dict):
|
|
336
|
+
if not isinstance(value, dict):
|
|
337
|
+
raise ConfigError(f"configuration value {where} must be an object")
|
|
338
|
+
result[key] = (
|
|
339
|
+
dict(value) if not base[key] else _merge_known(base[key], value, where)
|
|
340
|
+
)
|
|
341
|
+
else:
|
|
342
|
+
result[key] = value
|
|
343
|
+
return result
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def load_config(
|
|
347
|
+
path: str | None, overrides: dict[str, Any] | None = None
|
|
348
|
+
) -> dict[str, Any]:
|
|
349
|
+
supplied: dict[str, Any] = {}
|
|
350
|
+
if path:
|
|
351
|
+
try:
|
|
352
|
+
supplied = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
353
|
+
except (OSError, UnicodeError) as exc:
|
|
354
|
+
raise ConfigError(f"cannot read configuration {path}: {exc}") from exc
|
|
355
|
+
except json.JSONDecodeError as exc:
|
|
356
|
+
raise ConfigError(
|
|
357
|
+
f"invalid JSON configuration {path}:{exc.lineno}:{exc.colno}: {exc.msg}"
|
|
358
|
+
) from exc
|
|
359
|
+
if not isinstance(supplied, dict):
|
|
360
|
+
raise ConfigError("configuration root must be a JSON object")
|
|
361
|
+
config = _merge_known(DEFAULT_CONFIG, supplied)
|
|
362
|
+
for dotted, value in (overrides or {}).items():
|
|
363
|
+
if value is None:
|
|
364
|
+
continue
|
|
365
|
+
parts = dotted.split(".")
|
|
366
|
+
cursor = config
|
|
367
|
+
for part in parts[:-1]:
|
|
368
|
+
cursor = cursor[part]
|
|
369
|
+
cursor[parts[-1]] = value
|
|
370
|
+
validate_config(config)
|
|
371
|
+
return config
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def validate_config(config: dict[str, Any]) -> None:
|
|
375
|
+
if not isinstance(config["traceability"], bool):
|
|
376
|
+
raise ConfigError("traceability must be true or false")
|
|
377
|
+
if config["traceability"] and config["id_column"] in TRACE_COLUMNS:
|
|
378
|
+
raise ConfigError("the ID column cannot be a reserved traceability column")
|
|
379
|
+
quality = config["quality"]
|
|
380
|
+
for field in ("requirement_column", "obligation"):
|
|
381
|
+
value = quality[field]
|
|
382
|
+
if value is not None and (not isinstance(value, str) or not value.strip()):
|
|
383
|
+
raise ConfigError(f"quality.{field} must be null or a non-empty string")
|
|
384
|
+
if not isinstance(quality["required_columns"], list) or not all(
|
|
385
|
+
isinstance(column, str) and column.strip()
|
|
386
|
+
for column in quality["required_columns"]
|
|
387
|
+
):
|
|
388
|
+
raise ConfigError(
|
|
389
|
+
"quality.required_columns must be an array of non-empty column names"
|
|
390
|
+
)
|
|
391
|
+
if config["id_column"] is not None and not isinstance(config["id_column"], str):
|
|
392
|
+
raise ConfigError("id_column must be null or a string")
|
|
393
|
+
if not isinstance(config["title"], str):
|
|
394
|
+
raise ConfigError("title must be a string")
|
|
395
|
+
if config["document_mode"] not in {"review", "develop"}:
|
|
396
|
+
raise ConfigError("document_mode must be 'review' or 'develop'")
|
|
397
|
+
if not isinstance(config["csv"]["encoding"], str) or not config["csv"]["encoding"]:
|
|
398
|
+
raise ConfigError("csv.encoding must be a non-empty string")
|
|
399
|
+
delimiter = config["csv"]["delimiter"]
|
|
400
|
+
if (
|
|
401
|
+
not isinstance(delimiter, str)
|
|
402
|
+
or len(delimiter) != 1
|
|
403
|
+
or ord(delimiter) < 32
|
|
404
|
+
or ord(delimiter) == 127
|
|
405
|
+
):
|
|
406
|
+
raise ConfigError("csv.delimiter must be one printable character")
|
|
407
|
+
if not isinstance(config["csv"]["strict_columns"], bool):
|
|
408
|
+
raise ConfigError("csv.strict_columns must be true or false")
|
|
409
|
+
for field in ("hidden", "order"):
|
|
410
|
+
value = config["columns"][field]
|
|
411
|
+
if not isinstance(value, list) or not all(isinstance(x, str) for x in value):
|
|
412
|
+
raise ConfigError(f"columns.{field} must be an array of strings")
|
|
413
|
+
markdown = config["columns"]["markdown"]
|
|
414
|
+
if markdown is not None and (
|
|
415
|
+
not isinstance(markdown, list) or not all(isinstance(x, str) for x in markdown)
|
|
416
|
+
):
|
|
417
|
+
raise ConfigError("columns.markdown must be null or an array of strings")
|
|
418
|
+
for field in ("labels", "widths"):
|
|
419
|
+
value = config["columns"][field]
|
|
420
|
+
if not isinstance(value, dict) or not all(
|
|
421
|
+
isinstance(k, str) and isinstance(v, str) for k, v in value.items()
|
|
422
|
+
):
|
|
423
|
+
raise ConfigError(f"columns.{field} must be an object of string values")
|
|
424
|
+
width_pattern = re.compile(r"^(?:[1-9]\d*(?:\.\d+)?)(?:px|rem|em|ch|%)$")
|
|
425
|
+
for column, width in config["columns"]["widths"].items():
|
|
426
|
+
if not width_pattern.fullmatch(width):
|
|
427
|
+
raise ConfigError(
|
|
428
|
+
f"unsafe or invalid width for column {column!r}: {width!r}"
|
|
429
|
+
)
|
|
430
|
+
if config["diff"]["initial_view"] not in {"unified", "side-by-side"}:
|
|
431
|
+
raise ConfigError("diff.initial_view must be 'unified' or 'side-by-side'")
|
|
432
|
+
for field in (
|
|
433
|
+
"initial_changes_only",
|
|
434
|
+
"include_unchanged",
|
|
435
|
+
"ignore_trailing_whitespace",
|
|
436
|
+
):
|
|
437
|
+
if not isinstance(config["diff"][field], bool):
|
|
438
|
+
raise ConfigError(f"diff.{field} must be true or false")
|
|
439
|
+
ignored = config["diff"]["ignore_columns"]
|
|
440
|
+
if not isinstance(ignored, list) or not all(isinstance(x, str) for x in ignored):
|
|
441
|
+
raise ConfigError("diff.ignore_columns must be an array of strings")
|
|
442
|
+
if config["comments"]["csv_format"] not in {"json", "text"}:
|
|
443
|
+
raise ConfigError("comments.csv_format must be 'json' or 'text'")
|
|
444
|
+
if not isinstance(config["comments"]["enabled"], bool):
|
|
445
|
+
raise ConfigError("comments.enabled must be true or false")
|
|
446
|
+
if (
|
|
447
|
+
not isinstance(config["comments"]["csv_column"], str)
|
|
448
|
+
or not config["comments"]["csv_column"]
|
|
449
|
+
):
|
|
450
|
+
raise ConfigError("comments.csv_column must be a non-empty string")
|
|
451
|
+
initials = config["comments"]["reviewer_initials"]
|
|
452
|
+
if initials is not None and not valid_initials(initials):
|
|
453
|
+
raise ConfigError(
|
|
454
|
+
"comments.reviewer_initials must contain 1-16 non-control characters"
|
|
455
|
+
)
|
|
456
|
+
validate_xlsx_sources(config["xlsx"]["sources"])
|
|
457
|
+
validate_baselines(config["baselines"])
|
|
458
|
+
validate_gates(config["gates"])
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
def validate_baselines(baselines: dict[str, Any]) -> None:
|
|
462
|
+
if baselines["store"] not in {"git", "files"}:
|
|
463
|
+
raise ConfigError("baselines.store must be 'git' or 'files'")
|
|
464
|
+
prefix = baselines["git"]["tag_prefix"]
|
|
465
|
+
if not isinstance(prefix, str) or not prefix.endswith("/") or prefix == "/":
|
|
466
|
+
raise ConfigError(
|
|
467
|
+
"baselines.git.tag_prefix must be a non-empty string that ends with '/'"
|
|
468
|
+
)
|
|
469
|
+
directory = baselines["files"]["directory"]
|
|
470
|
+
if not isinstance(directory, str) or not directory.strip():
|
|
471
|
+
raise ConfigError("baselines.files.directory must be a non-empty string")
|
|
472
|
+
validate_enforcement(baselines["enforcement"], "baselines.enforcement")
|
|
473
|
+
columns = baselines["status_columns"]
|
|
474
|
+
if not isinstance(columns, list) or not all(isinstance(x, str) for x in columns):
|
|
475
|
+
raise ConfigError("baselines.status_columns must be an array of strings")
|
|
476
|
+
if not isinstance(baselines["change_control"], bool):
|
|
477
|
+
raise ConfigError("baselines.change_control must be true or false")
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
ENFORCEMENT_LEVELS = ("advisory", "strict")
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
def validate_enforcement(value: Any, where: str) -> None:
|
|
484
|
+
if value not in ENFORCEMENT_LEVELS:
|
|
485
|
+
raise ConfigError(f"{where} must be 'advisory' or 'strict'")
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
def resolve_gate_enforcement(gate: dict[str, Any], default: str) -> dict[str, Any]:
|
|
489
|
+
"""The gate with every check carrying its effective enforcement: the check's, else the gate's, else the default."""
|
|
490
|
+
level = gate.get("enforcement", default)
|
|
491
|
+
return {
|
|
492
|
+
**gate,
|
|
493
|
+
"checks": [
|
|
494
|
+
{**check, "enforcement": check.get("enforcement", level)}
|
|
495
|
+
for check in gate.get("checks", [])
|
|
496
|
+
],
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
_GATE_KEYS = {
|
|
501
|
+
"id",
|
|
502
|
+
"name",
|
|
503
|
+
"produces",
|
|
504
|
+
"scope",
|
|
505
|
+
"includes",
|
|
506
|
+
"captures_status",
|
|
507
|
+
"checks",
|
|
508
|
+
"signoff",
|
|
509
|
+
"enforcement",
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def validate_gates(gates: Any) -> None:
|
|
514
|
+
from .gates import validate_checks
|
|
515
|
+
from .signoff import validate_roles
|
|
516
|
+
|
|
517
|
+
if not isinstance(gates, list) or not all(isinstance(g, dict) for g in gates):
|
|
518
|
+
raise ConfigError("gates must be an array of objects")
|
|
519
|
+
seen: set[str] = set()
|
|
520
|
+
for index, gate in enumerate(gates):
|
|
521
|
+
where = f"gates[{index}]"
|
|
522
|
+
unknown = sorted(set(gate) - _GATE_KEYS)
|
|
523
|
+
if unknown:
|
|
524
|
+
raise ConfigError(f"unknown configuration key: {where}.{unknown[0]}")
|
|
525
|
+
gate_id = gate.get("id")
|
|
526
|
+
if not isinstance(gate_id, str) or not gate_id.strip():
|
|
527
|
+
raise ConfigError(f"{where}.id must be a non-empty string")
|
|
528
|
+
where = f"gates[{index}] ({gate_id})"
|
|
529
|
+
if "/" in gate_id:
|
|
530
|
+
raise ConfigError(f"{where}: gate ID cannot contain '/'")
|
|
531
|
+
if re.search(r"-\d+$", gate_id):
|
|
532
|
+
raise ConfigError(
|
|
533
|
+
f"{where}: gate ID cannot end in '-' and a number; numbers are reserved for re-baselines"
|
|
534
|
+
)
|
|
535
|
+
if gate_id in seen:
|
|
536
|
+
raise ConfigError(f"{where}: duplicate gate ID")
|
|
537
|
+
seen.add(gate_id)
|
|
538
|
+
for key in ("name", "produces"):
|
|
539
|
+
if key in gate and not isinstance(gate[key], str):
|
|
540
|
+
raise ConfigError(f"{where}.{key} must be a string")
|
|
541
|
+
if gate.get("scope", "table") not in {"table", "system"}:
|
|
542
|
+
raise ConfigError(f"{where}.scope must be 'table' or 'system'")
|
|
543
|
+
includes = gate.get("includes", ["Requirement", "System"])
|
|
544
|
+
if (
|
|
545
|
+
not isinstance(includes, list)
|
|
546
|
+
or not includes
|
|
547
|
+
or not all(x in RECORD_TYPES for x in includes)
|
|
548
|
+
):
|
|
549
|
+
raise ConfigError(
|
|
550
|
+
f"{where}.includes must be a non-empty array of: {', '.join(RECORD_TYPES)}"
|
|
551
|
+
)
|
|
552
|
+
if not isinstance(gate.get("captures_status", False), bool):
|
|
553
|
+
raise ConfigError(f"{where}.captures_status must be true or false")
|
|
554
|
+
if "enforcement" in gate:
|
|
555
|
+
validate_enforcement(gate["enforcement"], f"{where}.enforcement")
|
|
556
|
+
validate_roles(gate.get("signoff", []), where)
|
|
557
|
+
validate_checks(gate.get("checks", []), where)
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
_XLSX_SOURCE_KEYS = {
|
|
561
|
+
"table",
|
|
562
|
+
"sheet",
|
|
563
|
+
"header_row",
|
|
564
|
+
"columns",
|
|
565
|
+
"exclude",
|
|
566
|
+
"type",
|
|
567
|
+
"id_prefix",
|
|
568
|
+
"set",
|
|
569
|
+
"id_from",
|
|
570
|
+
"lists",
|
|
571
|
+
"headings",
|
|
572
|
+
"merge",
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
def validate_xlsx_sources(sources: Any) -> None:
|
|
577
|
+
if not isinstance(sources, list) or not all(isinstance(s, dict) for s in sources):
|
|
578
|
+
raise ConfigError("xlsx.sources must be an array of objects")
|
|
579
|
+
for index, source in enumerate(sources):
|
|
580
|
+
where = f"xlsx.sources[{index}]"
|
|
581
|
+
unknown = sorted(set(source) - _XLSX_SOURCE_KEYS)
|
|
582
|
+
if unknown:
|
|
583
|
+
raise ConfigError(f"unknown configuration key: {where}.{unknown[0]}")
|
|
584
|
+
has_table, has_sheet = "table" in source, "sheet" in source
|
|
585
|
+
if has_table == has_sheet:
|
|
586
|
+
raise ConfigError(f"{where} needs exactly one of 'table' or 'sheet'")
|
|
587
|
+
for key in ("table", "sheet", "type", "id_prefix"):
|
|
588
|
+
if key in source and (
|
|
589
|
+
not isinstance(source[key], str) or not source[key].strip()
|
|
590
|
+
):
|
|
591
|
+
raise ConfigError(f"{where}.{key} must be a non-empty string")
|
|
592
|
+
if "header_row" in source:
|
|
593
|
+
row = source["header_row"]
|
|
594
|
+
if not has_sheet:
|
|
595
|
+
raise ConfigError(f"{where}.header_row applies only with 'sheet'")
|
|
596
|
+
if isinstance(row, bool) or not isinstance(row, int) or row < 1:
|
|
597
|
+
raise ConfigError(
|
|
598
|
+
f"{where}.header_row must be a whole number of 1 or greater"
|
|
599
|
+
)
|
|
600
|
+
columns = source.get("columns", {})
|
|
601
|
+
if not isinstance(columns, dict) or not all(
|
|
602
|
+
isinstance(k, str) and isinstance(v, str) and v.strip()
|
|
603
|
+
for k, v in columns.items()
|
|
604
|
+
):
|
|
605
|
+
raise ConfigError(
|
|
606
|
+
f"{where}.columns must be an object of source header to output header strings"
|
|
607
|
+
)
|
|
608
|
+
exclude = source.get("exclude", [])
|
|
609
|
+
if not isinstance(exclude, list) or not all(
|
|
610
|
+
isinstance(x, str) for x in exclude
|
|
611
|
+
):
|
|
612
|
+
raise ConfigError(f"{where}.exclude must be an array of strings")
|
|
613
|
+
constants = source.get("set", {})
|
|
614
|
+
if not isinstance(constants, dict) or not all(
|
|
615
|
+
isinstance(k, str) and k.strip() and isinstance(v, str)
|
|
616
|
+
for k, v in constants.items()
|
|
617
|
+
):
|
|
618
|
+
raise ConfigError(
|
|
619
|
+
f"{where}.set must be an object of output header to constant string"
|
|
620
|
+
)
|
|
621
|
+
if "type" in source and "Type" in constants:
|
|
622
|
+
raise ConfigError(f"{where} gives both 'type' and set.Type; use one")
|
|
623
|
+
if "merge" in source:
|
|
624
|
+
if not isinstance(source["merge"], bool):
|
|
625
|
+
raise ConfigError(f"{where}.merge must be true or false")
|
|
626
|
+
if source["merge"]:
|
|
627
|
+
given = [
|
|
628
|
+
key
|
|
629
|
+
for key in ("type", "id_prefix", "id_from", "headings")
|
|
630
|
+
if key in source
|
|
631
|
+
]
|
|
632
|
+
given += ["set.Type"] if "Type" in constants else []
|
|
633
|
+
if given:
|
|
634
|
+
raise ConfigError(
|
|
635
|
+
f"{where} is a merge source, which adds no records, so it cannot give {', '.join(given)}"
|
|
636
|
+
)
|
|
637
|
+
if "id_from" in source:
|
|
638
|
+
id_from = source["id_from"]
|
|
639
|
+
if (
|
|
640
|
+
not isinstance(id_from, list)
|
|
641
|
+
or not id_from
|
|
642
|
+
or not all(isinstance(x, str) and x.strip() for x in id_from)
|
|
643
|
+
):
|
|
644
|
+
raise ConfigError(
|
|
645
|
+
f"{where}.id_from must be a non-empty array of column names"
|
|
646
|
+
)
|
|
647
|
+
if "id_prefix" not in source:
|
|
648
|
+
raise ConfigError(f"{where}.id_from needs id_prefix")
|
|
649
|
+
if "headings" in source:
|
|
650
|
+
validate_headings(source["headings"], f"{where}.headings")
|
|
651
|
+
lists = source.get("lists", {})
|
|
652
|
+
if not isinstance(lists, dict) or not all(
|
|
653
|
+
isinstance(k, str) and isinstance(v, str) and v for k, v in lists.items()
|
|
654
|
+
):
|
|
655
|
+
raise ConfigError(
|
|
656
|
+
f'{where}.lists must be an object of output header to separator string (or "newline")'
|
|
657
|
+
)
|
|
658
|
+
if sources and all(source.get("merge") is True for source in sources):
|
|
659
|
+
raise ConfigError(
|
|
660
|
+
"xlsx.sources: every source is a merge source; at least one must add the records they fill in"
|
|
661
|
+
)
|
|
662
|
+
|
|
663
|
+
|
|
664
|
+
def validate_headings(headings: Any, where: str) -> None:
|
|
665
|
+
if not isinstance(headings, dict):
|
|
666
|
+
raise ConfigError(f"{where} must be an object with column, values and into")
|
|
667
|
+
unknown = sorted(set(headings) - {"column", "values", "text", "into"})
|
|
668
|
+
if unknown:
|
|
669
|
+
raise ConfigError(f"unknown configuration key: {where}.{unknown[0]}")
|
|
670
|
+
for key in ("column", "into"):
|
|
671
|
+
if not isinstance(headings.get(key), str) or not headings[key].strip():
|
|
672
|
+
raise ConfigError(
|
|
673
|
+
f"{where}.{key} is required and must be a non-empty string"
|
|
674
|
+
)
|
|
675
|
+
if "text" in headings and (
|
|
676
|
+
not isinstance(headings["text"], str) or not headings["text"].strip()
|
|
677
|
+
):
|
|
678
|
+
raise ConfigError(f"{where}.text must be a non-empty string")
|
|
679
|
+
values = headings.get("values")
|
|
680
|
+
if (
|
|
681
|
+
not isinstance(values, list)
|
|
682
|
+
or not values
|
|
683
|
+
or not all(isinstance(v, str) and v.strip() for v in values)
|
|
684
|
+
):
|
|
685
|
+
raise ConfigError(
|
|
686
|
+
f"{where}.values must be a non-empty array of the type values that mark a heading row"
|
|
687
|
+
)
|
|
688
|
+
|
|
689
|
+
|
|
690
|
+
def valid_initials(value: Any) -> bool:
|
|
691
|
+
return (
|
|
692
|
+
isinstance(value, str)
|
|
693
|
+
and 1 <= len(value.strip()) <= 16
|
|
694
|
+
and not any(ord(c) < 32 or ord(c) == 127 for c in value)
|
|
695
|
+
)
|
|
696
|
+
|
|
697
|
+
|
|
698
|
+
def read_csv_table(path: str, config: dict[str, Any]) -> CsvTable:
|
|
699
|
+
return read_csv_source(CsvSource(str(Path(path)), None, Path(path)), config)
|
|
700
|
+
|
|
701
|
+
|
|
702
|
+
def read_csv_source(csv_source: CsvSource, config: dict[str, Any]) -> CsvTable:
|
|
703
|
+
source = csv_source.label
|
|
704
|
+
encoding = config["csv"]["encoding"]
|
|
705
|
+
delimiter = config["csv"]["delimiter"]
|
|
706
|
+
if csv_source.text is not None:
|
|
707
|
+
handle: Any = io.StringIO(csv_source.text, newline="")
|
|
708
|
+
else:
|
|
709
|
+
try:
|
|
710
|
+
handle = csv_source.path.open("r", encoding=encoding, newline="")
|
|
711
|
+
except (OSError, LookupError) as exc:
|
|
712
|
+
raise InputError(f"cannot open {source}: {exc}") from exc
|
|
713
|
+
with handle:
|
|
714
|
+
reader = csv.reader(handle, delimiter=delimiter, strict=True)
|
|
715
|
+
try:
|
|
716
|
+
headers = next(reader)
|
|
717
|
+
except StopIteration as exc:
|
|
718
|
+
raise InputError(f"{source}: CSV file is empty") from exc
|
|
719
|
+
except csv.Error as exc:
|
|
720
|
+
raise InputError(f"{source}: invalid CSV header: {exc}") from exc
|
|
721
|
+
if not headers:
|
|
722
|
+
raise InputError(f"{source}: CSV header is empty")
|
|
723
|
+
if any(h == "" for h in headers):
|
|
724
|
+
raise InputError(f"{source}: empty column name in header")
|
|
725
|
+
duplicates = sorted({h for h in headers if headers.count(h) > 1})
|
|
726
|
+
if duplicates:
|
|
727
|
+
raise InputError(
|
|
728
|
+
f"{source}: duplicate column name(s): {', '.join(duplicates)}"
|
|
729
|
+
)
|
|
730
|
+
id_column = config.get("id_column")
|
|
731
|
+
if not id_column:
|
|
732
|
+
raise ConfigError(
|
|
733
|
+
"an ID column is required; use --id-column or set id_column in the config"
|
|
734
|
+
)
|
|
735
|
+
if id_column not in headers:
|
|
736
|
+
raise InputError(f"{source}: ID column {id_column!r} is not present")
|
|
737
|
+
rows: list[dict[str, str]] = []
|
|
738
|
+
ids: list[str] = []
|
|
739
|
+
by_id: dict[str, dict[str, str]] = {}
|
|
740
|
+
record_by_id: dict[str, int] = {}
|
|
741
|
+
try:
|
|
742
|
+
for record_number, values in enumerate(reader, start=2):
|
|
743
|
+
if values == []:
|
|
744
|
+
continue
|
|
745
|
+
if config["csv"]["strict_columns"] and len(values) != len(headers):
|
|
746
|
+
raise InputError(
|
|
747
|
+
f"{source}: record {record_number} has {len(values)} fields; expected {len(headers)}"
|
|
748
|
+
)
|
|
749
|
+
if len(values) < len(headers):
|
|
750
|
+
values += [""] * (len(headers) - len(values))
|
|
751
|
+
elif len(values) > len(headers):
|
|
752
|
+
values = values[: len(headers)]
|
|
753
|
+
row = {
|
|
754
|
+
header: value.replace("\r\n", "\n").replace("\r", "\n")
|
|
755
|
+
for header, value in zip(headers, values)
|
|
756
|
+
}
|
|
757
|
+
normalized_id = row[id_column].strip()
|
|
758
|
+
if not normalized_id:
|
|
759
|
+
raise InputError(
|
|
760
|
+
f"{source}: record {record_number} has an empty ID in {id_column!r}"
|
|
761
|
+
)
|
|
762
|
+
if normalized_id in by_id:
|
|
763
|
+
raise InputError(
|
|
764
|
+
f"{source}: duplicate ID {normalized_id!r} at records "
|
|
765
|
+
f"{record_by_id[normalized_id]} and {record_number}"
|
|
766
|
+
)
|
|
767
|
+
rows.append(row)
|
|
768
|
+
ids.append(normalized_id)
|
|
769
|
+
by_id[normalized_id] = row
|
|
770
|
+
record_by_id[normalized_id] = record_number
|
|
771
|
+
except (csv.Error, UnicodeError) as exc:
|
|
772
|
+
raise InputError(
|
|
773
|
+
f"{source}: invalid CSV near record {reader.line_num}: {exc}"
|
|
774
|
+
) from exc
|
|
775
|
+
return CsvTable(
|
|
776
|
+
source,
|
|
777
|
+
csv_source.filename,
|
|
778
|
+
headers,
|
|
779
|
+
rows,
|
|
780
|
+
ids,
|
|
781
|
+
by_id,
|
|
782
|
+
csv_source.warnings,
|
|
783
|
+
csv_source.comments,
|
|
784
|
+
)
|
|
785
|
+
|
|
786
|
+
|
|
787
|
+
def effective_columns(
|
|
788
|
+
old: CsvTable | None, new: CsvTable, config: dict[str, Any]
|
|
789
|
+
) -> list[str]:
|
|
790
|
+
headers = list(new.headers)
|
|
791
|
+
if old:
|
|
792
|
+
headers.extend(h for h in old.headers if h not in headers)
|
|
793
|
+
configured = config["columns"]["order"]
|
|
794
|
+
missing = [h for h in configured if h not in headers]
|
|
795
|
+
if missing:
|
|
796
|
+
raise ConfigError(
|
|
797
|
+
f"configured ordered column(s) not found: {', '.join(missing)}"
|
|
798
|
+
)
|
|
799
|
+
return configured + [h for h in headers if h not in configured]
|
|
800
|
+
|
|
801
|
+
|
|
802
|
+
def _comparison_value(value: str, config: dict[str, Any]) -> str:
|
|
803
|
+
if config["diff"]["ignore_trailing_whitespace"]:
|
|
804
|
+
return "\n".join(line.rstrip() for line in value.split("\n"))
|
|
805
|
+
return value
|
|
806
|
+
|
|
807
|
+
|
|
808
|
+
def _tokens(line: str) -> list[str]:
|
|
809
|
+
return re.findall(r"\s+|\w+|[^\w\s]", line, flags=re.UNICODE)
|
|
810
|
+
|
|
811
|
+
|
|
812
|
+
def _word_pair(old: str, new: str) -> tuple[str, str]:
|
|
813
|
+
a, b = _tokens(old), _tokens(new)
|
|
814
|
+
matcher = difflib.SequenceMatcher(None, a, b, autojunk=False)
|
|
815
|
+
old_parts: list[str] = []
|
|
816
|
+
new_parts: list[str] = []
|
|
817
|
+
for tag, i1, i2, j1, j2 in matcher.get_opcodes():
|
|
818
|
+
left = html.escape("".join(a[i1:i2]))
|
|
819
|
+
right = html.escape("".join(b[j1:j2]))
|
|
820
|
+
if tag == "equal":
|
|
821
|
+
old_parts.append(left)
|
|
822
|
+
new_parts.append(right)
|
|
823
|
+
else:
|
|
824
|
+
if left:
|
|
825
|
+
old_parts.append(f'<span class="diff-word removed-word">{left}</span>')
|
|
826
|
+
if right:
|
|
827
|
+
new_parts.append(f'<span class="diff-word added-word">{right}</span>')
|
|
828
|
+
return "".join(old_parts), "".join(new_parts)
|
|
829
|
+
|
|
830
|
+
|
|
831
|
+
def render_diff(old: str, new: str) -> dict[str, str]:
|
|
832
|
+
# Do not manufacture a highlighted blank line for a genuinely empty side.
|
|
833
|
+
# This is especially important for added and removed rows in unified view.
|
|
834
|
+
if old == "" and new != "":
|
|
835
|
+
new_html = "".join(
|
|
836
|
+
f'<div class="diff-line added-line">{html.escape(line) or " "}</div>'
|
|
837
|
+
for line in new.splitlines()
|
|
838
|
+
)
|
|
839
|
+
return {"oldHtml": "", "newHtml": new_html, "unifiedHtml": new_html}
|
|
840
|
+
if new == "" and old != "":
|
|
841
|
+
old_html = "".join(
|
|
842
|
+
f'<div class="diff-line removed-line">{html.escape(line) or " "}</div>'
|
|
843
|
+
for line in old.splitlines()
|
|
844
|
+
)
|
|
845
|
+
return {"oldHtml": old_html, "newHtml": "", "unifiedHtml": old_html}
|
|
846
|
+
if max(len(old), len(new)) > 100_000:
|
|
847
|
+
old_html = html.escape(old)
|
|
848
|
+
new_html = html.escape(new)
|
|
849
|
+
return {
|
|
850
|
+
"oldHtml": f'<div class="diff-line removed-line">{old_html}</div>',
|
|
851
|
+
"newHtml": f'<div class="diff-line added-line">{new_html}</div>',
|
|
852
|
+
"unifiedHtml": (
|
|
853
|
+
f'<div class="diff-line removed-line">{old_html}</div>'
|
|
854
|
+
f'<div class="diff-line added-line">{new_html}</div>'
|
|
855
|
+
),
|
|
856
|
+
}
|
|
857
|
+
a = old.splitlines() or ([""] if old == "" else [])
|
|
858
|
+
b = new.splitlines() or ([""] if new == "" else [])
|
|
859
|
+
matcher = difflib.SequenceMatcher(None, a, b, autojunk=False)
|
|
860
|
+
old_out: list[str] = []
|
|
861
|
+
new_out: list[str] = []
|
|
862
|
+
unified: list[str] = []
|
|
863
|
+
for tag, i1, i2, j1, j2 in matcher.get_opcodes():
|
|
864
|
+
if tag == "equal":
|
|
865
|
+
for line in a[i1:i2]:
|
|
866
|
+
escaped = html.escape(line) or " "
|
|
867
|
+
block = f'<div class="diff-line context-line">{escaped}</div>'
|
|
868
|
+
old_out.append(block)
|
|
869
|
+
new_out.append(block)
|
|
870
|
+
unified.append(block)
|
|
871
|
+
elif tag == "delete":
|
|
872
|
+
for line in a[i1:i2]:
|
|
873
|
+
block = f'<div class="diff-line removed-line">{html.escape(line) or " "}</div>'
|
|
874
|
+
old_out.append(block)
|
|
875
|
+
unified.append(block)
|
|
876
|
+
elif tag == "insert":
|
|
877
|
+
for line in b[j1:j2]:
|
|
878
|
+
block = f'<div class="diff-line added-line">{html.escape(line) or " "}</div>'
|
|
879
|
+
new_out.append(block)
|
|
880
|
+
unified.append(block)
|
|
881
|
+
else:
|
|
882
|
+
old_lines, new_lines = a[i1:i2], b[j1:j2]
|
|
883
|
+
count = max(len(old_lines), len(new_lines))
|
|
884
|
+
for index in range(count):
|
|
885
|
+
left = old_lines[index] if index < len(old_lines) else None
|
|
886
|
+
right = new_lines[index] if index < len(new_lines) else None
|
|
887
|
+
if left is not None and right is not None:
|
|
888
|
+
left_html, right_html = _word_pair(left, right)
|
|
889
|
+
old_block = f'<div class="diff-line removed-line">{left_html or " "}</div>'
|
|
890
|
+
new_block = f'<div class="diff-line added-line">{right_html or " "}</div>'
|
|
891
|
+
old_out.append(old_block)
|
|
892
|
+
new_out.append(new_block)
|
|
893
|
+
unified.extend((old_block, new_block))
|
|
894
|
+
elif left is not None:
|
|
895
|
+
block = f'<div class="diff-line removed-line">{html.escape(left) or " "}</div>'
|
|
896
|
+
old_out.append(block)
|
|
897
|
+
unified.append(block)
|
|
898
|
+
elif right is not None:
|
|
899
|
+
block = f'<div class="diff-line added-line">{html.escape(right) or " "}</div>'
|
|
900
|
+
new_out.append(block)
|
|
901
|
+
unified.append(block)
|
|
902
|
+
return {
|
|
903
|
+
"oldHtml": "".join(old_out),
|
|
904
|
+
"newHtml": "".join(new_out),
|
|
905
|
+
"unifiedHtml": "".join(unified),
|
|
906
|
+
}
|
|
907
|
+
|
|
908
|
+
|
|
909
|
+
def _estimated_column_width(name: str, rows: list[dict[str, Any]]) -> int:
|
|
910
|
+
"""Estimate a compact initial pixel width without letting outliers dominate."""
|
|
911
|
+
lengths = [len(name)]
|
|
912
|
+
multiline = False
|
|
913
|
+
for row in rows[:1000]:
|
|
914
|
+
cell = row["cells"][name]
|
|
915
|
+
for value in (cell.get("old"), cell.get("new")):
|
|
916
|
+
if value is None:
|
|
917
|
+
continue
|
|
918
|
+
text = str(value)
|
|
919
|
+
multiline = multiline or "\n" in text
|
|
920
|
+
lengths.extend(len(line) for line in text.splitlines() or [""])
|
|
921
|
+
lengths.sort()
|
|
922
|
+
percentile = lengths[min(len(lengths) - 1, int((len(lengths) - 1) * 0.90))]
|
|
923
|
+
useful_chars = max(len(name) + 2, percentile)
|
|
924
|
+
cap = 44 if multiline or useful_chars > 36 else 32
|
|
925
|
+
return max(82, min(460, 30 + min(useful_chars, cap) * 7))
|
|
926
|
+
|
|
927
|
+
|
|
928
|
+
def _ordered_diff_ids(old: CsvTable, new: CsvTable) -> list[str]:
|
|
929
|
+
result = list(new.ids)
|
|
930
|
+
old_positions = {value: index for index, value in enumerate(old.ids)}
|
|
931
|
+
removed = [value for value in old.ids if value not in new.by_id]
|
|
932
|
+
for removed_id in removed:
|
|
933
|
+
following = next(
|
|
934
|
+
(
|
|
935
|
+
value
|
|
936
|
+
for value in old.ids[old_positions[removed_id] + 1 :]
|
|
937
|
+
if value in result
|
|
938
|
+
),
|
|
939
|
+
None,
|
|
940
|
+
)
|
|
941
|
+
if following is None:
|
|
942
|
+
result.append(removed_id)
|
|
943
|
+
else:
|
|
944
|
+
result.insert(result.index(following), removed_id)
|
|
945
|
+
return result
|
|
946
|
+
|
|
947
|
+
|
|
948
|
+
def _cell(
|
|
949
|
+
column: str,
|
|
950
|
+
old_value: str | None,
|
|
951
|
+
new_value: str | None,
|
|
952
|
+
changed: bool,
|
|
953
|
+
markdown: bool,
|
|
954
|
+
) -> dict[str, Any]:
|
|
955
|
+
old_text = "" if old_value is None else old_value
|
|
956
|
+
new_text = "" if new_value is None else new_value
|
|
957
|
+
result: dict[str, Any] = {
|
|
958
|
+
"column": column,
|
|
959
|
+
"old": old_value,
|
|
960
|
+
"new": new_value,
|
|
961
|
+
"changed": changed,
|
|
962
|
+
"search": f"{old_text}\n{new_text}".casefold(),
|
|
963
|
+
}
|
|
964
|
+
if changed:
|
|
965
|
+
result.update(render_diff(old_text, new_text))
|
|
966
|
+
else:
|
|
967
|
+
display = new_text if new_value is not None else old_text
|
|
968
|
+
rendered = (
|
|
969
|
+
render_markdown(display)
|
|
970
|
+
if markdown
|
|
971
|
+
else html.escape(display).replace("\n", "<br>")
|
|
972
|
+
)
|
|
973
|
+
result.update(
|
|
974
|
+
{"oldHtml": rendered, "newHtml": rendered, "unifiedHtml": rendered}
|
|
975
|
+
)
|
|
976
|
+
return result
|
|
977
|
+
|
|
978
|
+
|
|
979
|
+
def generation_time() -> str:
|
|
980
|
+
epoch = os.environ.get("SOURCE_DATE_EPOCH")
|
|
981
|
+
if epoch is not None:
|
|
982
|
+
try:
|
|
983
|
+
return (
|
|
984
|
+
datetime.fromtimestamp(int(epoch), timezone.utc)
|
|
985
|
+
.astimezone()
|
|
986
|
+
.isoformat(timespec="seconds")
|
|
987
|
+
)
|
|
988
|
+
except (ValueError, OverflowError, OSError) as exc:
|
|
989
|
+
raise ConfigError(
|
|
990
|
+
"SOURCE_DATE_EPOCH must be a valid integer timestamp"
|
|
991
|
+
) from exc
|
|
992
|
+
return datetime.now().astimezone().isoformat(timespec="seconds")
|
|
993
|
+
|
|
994
|
+
|
|
995
|
+
def prepare_trace_table(
|
|
996
|
+
table: CsvTable, config: dict[str, Any]
|
|
997
|
+
) -> tuple[CsvTable, list[dict[str, Any]]]:
|
|
998
|
+
comments_config = config["comments"]
|
|
999
|
+
comment_column = comments_config["csv_column"]
|
|
1000
|
+
if comments_config["enabled"] and comment_column in (
|
|
1001
|
+
*TRACE_COLUMNS,
|
|
1002
|
+
config["id_column"],
|
|
1003
|
+
):
|
|
1004
|
+
raise ConfigError("the comments column cannot also store traceability data")
|
|
1005
|
+
managed_comments = (
|
|
1006
|
+
comments_config["enabled"]
|
|
1007
|
+
and comments_config["csv_format"] == "json"
|
|
1008
|
+
and comment_column in table.headers
|
|
1009
|
+
)
|
|
1010
|
+
headers = [
|
|
1011
|
+
name for name in table.headers if not managed_comments or name != comment_column
|
|
1012
|
+
]
|
|
1013
|
+
headers.extend(name for name in TRACE_COLUMNS if name not in headers)
|
|
1014
|
+
rows = [{name: row.get(name, "") for name in headers} for row in table.rows]
|
|
1015
|
+
comments = []
|
|
1016
|
+
if managed_comments:
|
|
1017
|
+
for rid, row in zip(table.ids, table.rows):
|
|
1018
|
+
message = f"{table.label}: expected exported JSON comments in {comment_column!r} for {rid!r}"
|
|
1019
|
+
try:
|
|
1020
|
+
records = json.loads(row[comment_column] or "[]")
|
|
1021
|
+
except ValueError as exc:
|
|
1022
|
+
raise InputError(message) from exc
|
|
1023
|
+
if not isinstance(records, list):
|
|
1024
|
+
raise InputError(message)
|
|
1025
|
+
for record in records:
|
|
1026
|
+
if (
|
|
1027
|
+
not isinstance(record, dict)
|
|
1028
|
+
or record.get("column") not in headers
|
|
1029
|
+
or record["column"] == config["id_column"]
|
|
1030
|
+
or not valid_initials(record.get("initials"))
|
|
1031
|
+
or not all(
|
|
1032
|
+
isinstance(record.get(key), str) for key in ("created", "text")
|
|
1033
|
+
)
|
|
1034
|
+
or (
|
|
1035
|
+
record.get("modified") is not None
|
|
1036
|
+
and not isinstance(record["modified"], str)
|
|
1037
|
+
)
|
|
1038
|
+
):
|
|
1039
|
+
raise InputError(message)
|
|
1040
|
+
comments.append(
|
|
1041
|
+
{
|
|
1042
|
+
"id": f"imported-{len(comments)}",
|
|
1043
|
+
"requirementId": rid,
|
|
1044
|
+
**{
|
|
1045
|
+
key: record.get(key)
|
|
1046
|
+
for key in (
|
|
1047
|
+
"column",
|
|
1048
|
+
"initials",
|
|
1049
|
+
"created",
|
|
1050
|
+
"modified",
|
|
1051
|
+
"text",
|
|
1052
|
+
)
|
|
1053
|
+
},
|
|
1054
|
+
}
|
|
1055
|
+
)
|
|
1056
|
+
prepared = CsvTable(
|
|
1057
|
+
table.label,
|
|
1058
|
+
table.filename,
|
|
1059
|
+
headers,
|
|
1060
|
+
rows,
|
|
1061
|
+
table.ids,
|
|
1062
|
+
dict(zip(table.ids, rows)),
|
|
1063
|
+
table.warnings,
|
|
1064
|
+
table.comments,
|
|
1065
|
+
)
|
|
1066
|
+
return prepared, comments
|
|
1067
|
+
|
|
1068
|
+
|
|
1069
|
+
def build_report_data(
|
|
1070
|
+
new: CsvTable,
|
|
1071
|
+
config: dict[str, Any],
|
|
1072
|
+
output_name: str,
|
|
1073
|
+
old: CsvTable | None = None,
|
|
1074
|
+
baselines: Sequence[dict[str, Any]] = (),
|
|
1075
|
+
controls: dict[str, Any] | None = None,
|
|
1076
|
+
) -> dict[str, Any]:
|
|
1077
|
+
comments = []
|
|
1078
|
+
traceability = config["traceability"]
|
|
1079
|
+
if traceability:
|
|
1080
|
+
new, comments = prepare_trace_table(new, config)
|
|
1081
|
+
if old:
|
|
1082
|
+
old, old_comments = prepare_trace_table(old, config)
|
|
1083
|
+
comments.extend(
|
|
1084
|
+
{**comment, "id": f"old-{comment['id']}"}
|
|
1085
|
+
for comment in old_comments
|
|
1086
|
+
if comment["requirementId"] not in new.by_id
|
|
1087
|
+
)
|
|
1088
|
+
if config["comments"]["enabled"]:
|
|
1089
|
+
comments.extend(c for c in new.comments if c["column"] in new.headers)
|
|
1090
|
+
if old:
|
|
1091
|
+
comments.extend(
|
|
1092
|
+
{**c, "id": f"old-{c['id']}"}
|
|
1093
|
+
for c in old.comments
|
|
1094
|
+
if c["requirementId"] not in new.by_id and c["column"] in old.headers
|
|
1095
|
+
)
|
|
1096
|
+
columns = effective_columns(old, new, config)
|
|
1097
|
+
quality = config["quality"]
|
|
1098
|
+
quality_columns = set(quality["required_columns"])
|
|
1099
|
+
if quality["requirement_column"] is not None:
|
|
1100
|
+
quality_columns.add(quality["requirement_column"])
|
|
1101
|
+
invalid_quality_columns = quality_columns.difference(
|
|
1102
|
+
set(new.headers) - {config["id_column"]}
|
|
1103
|
+
)
|
|
1104
|
+
if invalid_quality_columns:
|
|
1105
|
+
names = ", ".join(sorted(invalid_quality_columns))
|
|
1106
|
+
raise ConfigError(
|
|
1107
|
+
f"quality columns must be non-ID columns in the current CSV: {names}"
|
|
1108
|
+
)
|
|
1109
|
+
markdown_columns = config["columns"]["markdown"]
|
|
1110
|
+
markdown_set = set(columns if markdown_columns is None else markdown_columns)
|
|
1111
|
+
for name in markdown_set:
|
|
1112
|
+
if name not in columns:
|
|
1113
|
+
raise ConfigError(f"configured Markdown column not found: {name}")
|
|
1114
|
+
ignored = set(config["diff"]["ignore_columns"])
|
|
1115
|
+
unknown_ignored = ignored.difference(columns)
|
|
1116
|
+
if unknown_ignored:
|
|
1117
|
+
raise ConfigError(
|
|
1118
|
+
f"ignored diff column(s) not found: {', '.join(sorted(unknown_ignored))}"
|
|
1119
|
+
)
|
|
1120
|
+
rows: list[dict[str, Any]] = []
|
|
1121
|
+
if old is None:
|
|
1122
|
+
ids = new.ids
|
|
1123
|
+
else:
|
|
1124
|
+
ids = _ordered_diff_ids(old, new)
|
|
1125
|
+
counts = {"added": 0, "removed": 0, "changed": 0, "unchanged": 0}
|
|
1126
|
+
for requirement_id in ids:
|
|
1127
|
+
old_row = old.by_id.get(requirement_id) if old else None
|
|
1128
|
+
new_row = new.by_id.get(requirement_id)
|
|
1129
|
+
if old is None:
|
|
1130
|
+
status = "unchanged"
|
|
1131
|
+
elif old_row is None:
|
|
1132
|
+
status = "added"
|
|
1133
|
+
elif new_row is None:
|
|
1134
|
+
status = "removed"
|
|
1135
|
+
else:
|
|
1136
|
+
changed_columns = [
|
|
1137
|
+
column
|
|
1138
|
+
for column in columns
|
|
1139
|
+
if column not in ignored
|
|
1140
|
+
and _comparison_value(old_row.get(column, ""), config)
|
|
1141
|
+
!= _comparison_value(new_row.get(column, ""), config)
|
|
1142
|
+
]
|
|
1143
|
+
status = "changed" if changed_columns else "unchanged"
|
|
1144
|
+
counts[status] += 1
|
|
1145
|
+
cells: dict[str, dict[str, Any]] = {}
|
|
1146
|
+
for column in columns:
|
|
1147
|
+
old_value = (
|
|
1148
|
+
old_row.get(column, "")
|
|
1149
|
+
if old_row is not None and column in old_row
|
|
1150
|
+
else None
|
|
1151
|
+
)
|
|
1152
|
+
new_value = (
|
|
1153
|
+
new_row.get(column, "")
|
|
1154
|
+
if new_row is not None and column in new_row
|
|
1155
|
+
else None
|
|
1156
|
+
)
|
|
1157
|
+
changed = (
|
|
1158
|
+
old is not None
|
|
1159
|
+
and column not in ignored
|
|
1160
|
+
and _comparison_value("" if old_value is None else old_value, config)
|
|
1161
|
+
!= _comparison_value("" if new_value is None else new_value, config)
|
|
1162
|
+
)
|
|
1163
|
+
cells[column] = _cell(
|
|
1164
|
+
column, old_value, new_value, changed, column in markdown_set
|
|
1165
|
+
)
|
|
1166
|
+
rows.append(
|
|
1167
|
+
{
|
|
1168
|
+
"id": requirement_id,
|
|
1169
|
+
"status": status,
|
|
1170
|
+
"sourceOrder": len(rows),
|
|
1171
|
+
"cells": cells,
|
|
1172
|
+
}
|
|
1173
|
+
)
|
|
1174
|
+
if old is not None and not config["diff"]["include_unchanged"]:
|
|
1175
|
+
rows = [row for row in rows if row["status"] != "unchanged"]
|
|
1176
|
+
labels = config["columns"]["labels"]
|
|
1177
|
+
trace_widths = (
|
|
1178
|
+
{"Name": 150, "Definition": 320, "Aliases": 140, "Rationale": 260}
|
|
1179
|
+
if traceability
|
|
1180
|
+
else {}
|
|
1181
|
+
)
|
|
1182
|
+
column_data = [
|
|
1183
|
+
{
|
|
1184
|
+
"name": name,
|
|
1185
|
+
"label": labels.get(name, name),
|
|
1186
|
+
"hidden": name in config["columns"]["hidden"],
|
|
1187
|
+
"width": config["columns"]["widths"].get(name),
|
|
1188
|
+
"estimatedWidth": max(
|
|
1189
|
+
_estimated_column_width(name, rows), trace_widths.get(name, 0)
|
|
1190
|
+
),
|
|
1191
|
+
"oldOnly": bool(old and name in old.headers and name not in new.headers),
|
|
1192
|
+
"newOnly": bool(old and name in new.headers and name not in old.headers),
|
|
1193
|
+
"markdown": name in markdown_set,
|
|
1194
|
+
}
|
|
1195
|
+
for name in columns
|
|
1196
|
+
]
|
|
1197
|
+
warnings: list[str] = [*new.warnings, *(old.warnings if old else ())]
|
|
1198
|
+
optional_names = set(config["columns"]["hidden"])
|
|
1199
|
+
optional_names.update(config["columns"]["labels"])
|
|
1200
|
+
optional_names.update(config["columns"]["widths"])
|
|
1201
|
+
unknown_optional = sorted(optional_names.difference(columns))
|
|
1202
|
+
if unknown_optional:
|
|
1203
|
+
warnings.append(
|
|
1204
|
+
"configured optional column(s) not found: " + ", ".join(unknown_optional)
|
|
1205
|
+
)
|
|
1206
|
+
if config["id_column"] in config["columns"]["hidden"]:
|
|
1207
|
+
warnings.append("the requirement ID column is initially hidden")
|
|
1208
|
+
stem = Path(output_name).stem
|
|
1209
|
+
stem = re.sub(r"-reviewed-[A-Za-z0-9_-]+(?:-\d{8}T\d{6})?$", "", stem)
|
|
1210
|
+
return {
|
|
1211
|
+
"format": "reqtable-report",
|
|
1212
|
+
"formatVersion": 1,
|
|
1213
|
+
"meta": {
|
|
1214
|
+
"title": config["title"],
|
|
1215
|
+
"mode": "diff" if old else "snapshot",
|
|
1216
|
+
"documentMode": config["document_mode"],
|
|
1217
|
+
"generated": generation_time(),
|
|
1218
|
+
"newFilename": new.filename,
|
|
1219
|
+
"oldFilename": old.filename if old else None,
|
|
1220
|
+
"baseFilename": stem,
|
|
1221
|
+
"idColumn": config["id_column"],
|
|
1222
|
+
"counts": counts,
|
|
1223
|
+
"ignoredColumns": sorted(ignored),
|
|
1224
|
+
"warnings": warnings,
|
|
1225
|
+
},
|
|
1226
|
+
"options": {
|
|
1227
|
+
"initialView": config["diff"]["initial_view"],
|
|
1228
|
+
"initialChangesOnly": config["diff"]["initial_changes_only"],
|
|
1229
|
+
"commentsEnabled": config["comments"]["enabled"],
|
|
1230
|
+
"reviewerInitials": config["comments"]["reviewer_initials"] or "",
|
|
1231
|
+
"includeUnchanged": config["diff"]["include_unchanged"],
|
|
1232
|
+
"ignoreTrailingWhitespace": config["diff"]["ignore_trailing_whitespace"],
|
|
1233
|
+
"commentsColumn": config["comments"]["csv_column"],
|
|
1234
|
+
"commentsFormat": config["comments"]["csv_format"],
|
|
1235
|
+
"csvDelimiter": config["csv"]["delimiter"],
|
|
1236
|
+
"quality": quality,
|
|
1237
|
+
"traceability": traceability,
|
|
1238
|
+
},
|
|
1239
|
+
"columns": column_data,
|
|
1240
|
+
"rows": rows,
|
|
1241
|
+
"source": {
|
|
1242
|
+
"new": {"headers": new.headers, "rows": new.rows},
|
|
1243
|
+
"old": {"headers": old.headers, "rows": old.rows} if old else None,
|
|
1244
|
+
},
|
|
1245
|
+
"comments": comments,
|
|
1246
|
+
"baselines": list(baselines),
|
|
1247
|
+
"controls": controls,
|
|
1248
|
+
"gates": [
|
|
1249
|
+
resolve_gate_enforcement(gate, config["baselines"]["enforcement"])
|
|
1250
|
+
for gate in config["gates"]
|
|
1251
|
+
],
|
|
1252
|
+
}
|
|
1253
|
+
|
|
1254
|
+
|
|
1255
|
+
def atomic_write_text(path: str, content: str, encoding: str = "utf-8") -> None:
|
|
1256
|
+
destination = Path(path)
|
|
1257
|
+
try:
|
|
1258
|
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
1259
|
+
fd, temporary = tempfile.mkstemp(
|
|
1260
|
+
prefix=f".{destination.name}.", dir=destination.parent
|
|
1261
|
+
)
|
|
1262
|
+
try:
|
|
1263
|
+
with os.fdopen(fd, "w", encoding=encoding, newline="") as handle:
|
|
1264
|
+
handle.write(content)
|
|
1265
|
+
handle.flush()
|
|
1266
|
+
os.fsync(handle.fileno())
|
|
1267
|
+
os.replace(temporary, destination)
|
|
1268
|
+
except Exception:
|
|
1269
|
+
try:
|
|
1270
|
+
os.unlink(temporary)
|
|
1271
|
+
except OSError:
|
|
1272
|
+
pass
|
|
1273
|
+
raise
|
|
1274
|
+
except OSError as exc:
|
|
1275
|
+
raise OutputError(f"cannot write {destination}: {exc}") from exc
|
|
1276
|
+
|
|
1277
|
+
|
|
1278
|
+
def comments_as_text(comments: Iterable[dict[str, Any]]) -> str:
|
|
1279
|
+
blocks = []
|
|
1280
|
+
for comment in comments:
|
|
1281
|
+
timestamp = comment.get("modified") or comment.get("created", "")
|
|
1282
|
+
blocks.append(
|
|
1283
|
+
f"[{timestamp}] [{comment.get('initials', '')}] [{comment.get('column', '')}]\n"
|
|
1284
|
+
f"{comment.get('text', '')}"
|
|
1285
|
+
)
|
|
1286
|
+
return "\n\n---\n\n".join(blocks)
|
|
1287
|
+
|
|
1288
|
+
|
|
1289
|
+
def extract_report_json(report_path: str) -> dict[str, Any]:
|
|
1290
|
+
path = Path(report_path)
|
|
1291
|
+
try:
|
|
1292
|
+
source = path.read_text(encoding="utf-8")
|
|
1293
|
+
except (OSError, UnicodeError) as exc:
|
|
1294
|
+
raise InputError(f"cannot read report {path}: {exc}") from exc
|
|
1295
|
+
match = re.search(
|
|
1296
|
+
r'<script\s+id="reqtable-data"\s+type="application/json">(.*?)</script>',
|
|
1297
|
+
source,
|
|
1298
|
+
flags=re.DOTALL,
|
|
1299
|
+
)
|
|
1300
|
+
if not match:
|
|
1301
|
+
raise InputError(f"{path}: reqtable data block was not found")
|
|
1302
|
+
try:
|
|
1303
|
+
data = json.loads(match.group(1))
|
|
1304
|
+
except json.JSONDecodeError as exc:
|
|
1305
|
+
raise InputError(f"{path}: invalid embedded report data: {exc}") from exc
|
|
1306
|
+
if (
|
|
1307
|
+
not isinstance(data, dict)
|
|
1308
|
+
or data.get("format") != "reqtable-report"
|
|
1309
|
+
or data.get("formatVersion") not in (1, 2)
|
|
1310
|
+
):
|
|
1311
|
+
raise InputError(f"{path}: unsupported or invalid reqtable report format")
|
|
1312
|
+
if not isinstance(data.get("comments"), list) or not isinstance(
|
|
1313
|
+
data.get("source"), dict
|
|
1314
|
+
):
|
|
1315
|
+
raise InputError(f"{path}: malformed report data")
|
|
1316
|
+
for index, comment in enumerate(data["comments"]):
|
|
1317
|
+
required = {"id", "requirementId", "column", "initials", "created", "text"}
|
|
1318
|
+
if not isinstance(comment, dict) or not required.issubset(comment):
|
|
1319
|
+
raise InputError(f"{path}: malformed comment at index {index}")
|
|
1320
|
+
if not valid_initials(comment["initials"]):
|
|
1321
|
+
raise InputError(f"{path}: invalid reviewer initials in comment {index}")
|
|
1322
|
+
if not all(isinstance(comment[key], str) for key in required):
|
|
1323
|
+
raise InputError(f"{path}: invalid field type in comment {index}")
|
|
1324
|
+
if len(comment["text"]) > 1_000_000:
|
|
1325
|
+
raise InputError(f"{path}: comment {index} exceeds the size limit")
|
|
1326
|
+
return data
|
|
1327
|
+
|
|
1328
|
+
|
|
1329
|
+
def extract_comments_csv(
|
|
1330
|
+
report_path: str,
|
|
1331
|
+
output_path: str,
|
|
1332
|
+
comments_column: str,
|
|
1333
|
+
comments_format: str,
|
|
1334
|
+
csv_base: str,
|
|
1335
|
+
allow_comment_loss: bool,
|
|
1336
|
+
delimiter: str = ",",
|
|
1337
|
+
) -> None:
|
|
1338
|
+
data = extract_report_json(report_path)
|
|
1339
|
+
source = data["source"]
|
|
1340
|
+
mode = data["meta"].get("mode")
|
|
1341
|
+
if mode == "snapshot":
|
|
1342
|
+
selected = source["new"]
|
|
1343
|
+
allowed_ids = {
|
|
1344
|
+
str(row.get(data["meta"]["idColumn"], "")).strip()
|
|
1345
|
+
for row in selected["rows"]
|
|
1346
|
+
}
|
|
1347
|
+
add_status = False
|
|
1348
|
+
else:
|
|
1349
|
+
if csv_base not in {"new", "old", "union"}:
|
|
1350
|
+
raise ConfigError("--csv-base must be new, old, or union")
|
|
1351
|
+
if csv_base in {"new", "old"}:
|
|
1352
|
+
selected = source[csv_base]
|
|
1353
|
+
if selected is None:
|
|
1354
|
+
raise InputError(f"report does not contain a {csv_base} snapshot")
|
|
1355
|
+
allowed_ids = {
|
|
1356
|
+
str(row.get(data["meta"]["idColumn"], "")).strip()
|
|
1357
|
+
for row in selected["rows"]
|
|
1358
|
+
}
|
|
1359
|
+
add_status = False
|
|
1360
|
+
else:
|
|
1361
|
+
id_column = data["meta"]["idColumn"]
|
|
1362
|
+
old_rows = {
|
|
1363
|
+
str(row.get(id_column, "")).strip(): row
|
|
1364
|
+
for row in source["old"]["rows"]
|
|
1365
|
+
}
|
|
1366
|
+
new_rows = {
|
|
1367
|
+
str(row.get(id_column, "")).strip(): row
|
|
1368
|
+
for row in source["new"]["rows"]
|
|
1369
|
+
}
|
|
1370
|
+
order = [
|
|
1371
|
+
str(row.get(id_column, "")).strip() for row in source["new"]["rows"]
|
|
1372
|
+
]
|
|
1373
|
+
order += [
|
|
1374
|
+
str(row.get(id_column, "")).strip()
|
|
1375
|
+
for row in source["old"]["rows"]
|
|
1376
|
+
if str(row.get(id_column, "")).strip() not in new_rows
|
|
1377
|
+
]
|
|
1378
|
+
headers = list(source["new"]["headers"]) + [
|
|
1379
|
+
h for h in source["old"]["headers"] if h not in source["new"]["headers"]
|
|
1380
|
+
]
|
|
1381
|
+
if "_Diff Status" in headers:
|
|
1382
|
+
raise InputError(
|
|
1383
|
+
"cannot use union CSV base because '_Diff Status' already exists"
|
|
1384
|
+
)
|
|
1385
|
+
if data["formatVersion"] == 1:
|
|
1386
|
+
status_by_id = {row["id"]: row["status"] for row in data["rows"]}
|
|
1387
|
+
else:
|
|
1388
|
+
status_by_id = data.get("diffStatus")
|
|
1389
|
+
if not isinstance(status_by_id, dict):
|
|
1390
|
+
raise InputError(f"{report_path}: malformed diff status map")
|
|
1391
|
+
selected = {
|
|
1392
|
+
"headers": headers + ["_Diff Status"],
|
|
1393
|
+
"rows": [
|
|
1394
|
+
{
|
|
1395
|
+
**old_rows.get(rid, {}),
|
|
1396
|
+
**new_rows.get(rid, {}),
|
|
1397
|
+
"_Diff Status": status_by_id.get(rid, ""),
|
|
1398
|
+
}
|
|
1399
|
+
for rid in order
|
|
1400
|
+
],
|
|
1401
|
+
}
|
|
1402
|
+
allowed_ids = set(order)
|
|
1403
|
+
add_status = True
|
|
1404
|
+
excluded = [c for c in data["comments"] if c["requirementId"] not in allowed_ids]
|
|
1405
|
+
if excluded and not allow_comment_loss:
|
|
1406
|
+
excluded_ids = sorted({c["requirementId"] for c in excluded})
|
|
1407
|
+
raise InputError(
|
|
1408
|
+
"selected CSV base would omit comments for requirement(s): "
|
|
1409
|
+
+ ", ".join(excluded_ids)
|
|
1410
|
+
+ "; select another --csv-base or pass --allow-comment-loss"
|
|
1411
|
+
)
|
|
1412
|
+
headers = list(selected["headers"])
|
|
1413
|
+
if comments_column in headers:
|
|
1414
|
+
raise InputError(
|
|
1415
|
+
f"comments column {comments_column!r} already exists; choose another name"
|
|
1416
|
+
)
|
|
1417
|
+
headers.append(comments_column)
|
|
1418
|
+
grouped: dict[str, list[dict[str, Any]]] = {}
|
|
1419
|
+
for comment in data["comments"]:
|
|
1420
|
+
if comment["requirementId"] in allowed_ids:
|
|
1421
|
+
exported = {
|
|
1422
|
+
key: comment.get(key)
|
|
1423
|
+
for key in ("column", "initials", "created", "modified", "text")
|
|
1424
|
+
}
|
|
1425
|
+
grouped.setdefault(comment["requirementId"], []).append(exported)
|
|
1426
|
+
id_column = data["meta"]["idColumn"]
|
|
1427
|
+
buffer = io.StringIO(newline="")
|
|
1428
|
+
writer = csv.DictWriter(
|
|
1429
|
+
buffer,
|
|
1430
|
+
fieldnames=headers,
|
|
1431
|
+
delimiter=delimiter,
|
|
1432
|
+
extrasaction="ignore",
|
|
1433
|
+
lineterminator="\n",
|
|
1434
|
+
)
|
|
1435
|
+
writer.writeheader()
|
|
1436
|
+
for source_row in selected["rows"]:
|
|
1437
|
+
row = dict(source_row)
|
|
1438
|
+
rid = str(row.get(id_column, "")).strip()
|
|
1439
|
+
comments = sorted(
|
|
1440
|
+
grouped.get(rid, []), key=lambda item: item.get("created") or ""
|
|
1441
|
+
)
|
|
1442
|
+
row[comments_column] = (
|
|
1443
|
+
json.dumps(comments, ensure_ascii=False, separators=(",", ":"))
|
|
1444
|
+
if comments_format == "json"
|
|
1445
|
+
else comments_as_text(comments)
|
|
1446
|
+
)
|
|
1447
|
+
writer.writerow(row)
|
|
1448
|
+
atomic_write_text(output_path, buffer.getvalue(), encoding="utf-8-sig")
|