reqtable 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
reqtable/core.py ADDED
@@ -0,0 +1,1448 @@
1
+ from __future__ import annotations
2
+
3
+ import csv
4
+ import difflib
5
+ import html
6
+ import io
7
+ import json
8
+ import os
9
+ import re
10
+ import tempfile
11
+ from dataclasses import dataclass
12
+ from datetime import datetime, timezone
13
+ from pathlib import Path
14
+ from typing import Any, Iterable, Sequence
15
+
16
+ from .markdown import render_markdown
17
+ from .traceability import TRACE_COLUMNS
18
+
19
+
20
+ class ReqtableError(Exception):
21
+ """Base class for actionable application errors."""
22
+
23
+
24
+ class ConfigError(ReqtableError):
25
+ pass
26
+
27
+
28
+ class InputError(ReqtableError):
29
+ pass
30
+
31
+
32
+ class OutputError(ReqtableError):
33
+ pass
34
+
35
+
36
+ RECORD_TYPES = (
37
+ "System",
38
+ "Requirement",
39
+ "Need",
40
+ "Source",
41
+ "Term",
42
+ "Value",
43
+ "Field",
44
+ "Test",
45
+ "Test plan",
46
+ "Change request",
47
+ )
48
+
49
+
50
+ def _preset_gate(
51
+ id: str,
52
+ name: str,
53
+ produces: str,
54
+ checks: list[dict[str, Any]],
55
+ reviewers: int = 2,
56
+ **extra: Any,
57
+ ) -> dict[str, Any]:
58
+ return {
59
+ "id": id,
60
+ "name": name,
61
+ "produces": produces,
62
+ "scope": "table",
63
+ "includes": ["Requirement", "System"],
64
+ "captures_status": False,
65
+ "checks": checks,
66
+ "signoff": [
67
+ {"role": "Reviewer", "min": reviewers},
68
+ {"role": "Approver", "min": 1},
69
+ ],
70
+ **extra,
71
+ }
72
+
73
+
74
+ def _check(id: str, description: str, **condition: Any) -> dict[str, Any]:
75
+ return {"id": id, "description": description, **condition}
76
+
77
+
78
+ def _no_findings(prefix: str) -> dict[str, Any]:
79
+ return _check(
80
+ f"{prefix}-no-findings", "There are no trace findings", no_findings=True
81
+ )
82
+
83
+
84
+ def _change_control(prefix: str) -> dict[str, Any]:
85
+ return _check(
86
+ f"{prefix}-change-control",
87
+ "Every change since the last baseline has an approved change request",
88
+ unchanged_since=True,
89
+ )
90
+
91
+
92
+ _COMPLIANT = {"Compliance": ["Compliant", "Waived"]}
93
+ _TESTS = {"type": "Test"}
94
+
95
+ PRESET_GATES = [
96
+ _preset_gate(
97
+ "SRR",
98
+ "System Requirements Review",
99
+ "Functional baseline",
100
+ [
101
+ _check(
102
+ "srr-text",
103
+ "Every requirement has its requirement text",
104
+ filled="Requirement",
105
+ ),
106
+ _check(
107
+ "srr-method",
108
+ "Every requirement has a verification method",
109
+ filled="Verification Method",
110
+ ),
111
+ _check(
112
+ "srr-priority", "Every requirement has a priority", filled="Priority"
113
+ ),
114
+ _no_findings("srr"),
115
+ ],
116
+ ),
117
+ _preset_gate(
118
+ "SFR",
119
+ "System Functional Review",
120
+ "Functional baseline",
121
+ [
122
+ _check(
123
+ "sfr-allocated",
124
+ "Every requirement is allocated to a system",
125
+ linked="Allocated to",
126
+ ),
127
+ _no_findings("sfr"),
128
+ _change_control("sfr"),
129
+ ],
130
+ ),
131
+ _preset_gate(
132
+ "PDR",
133
+ "Preliminary Design Review",
134
+ "Allocated baseline",
135
+ [
136
+ _check(
137
+ "pdr-allocated-leaf",
138
+ "Every requirement is allocated to a leaf system",
139
+ linked="Allocated to",
140
+ leaf=True,
141
+ ),
142
+ _check(
143
+ "pdr-level",
144
+ "Every requirement has a verification level",
145
+ filled="Verification Level",
146
+ ),
147
+ _no_findings("pdr"),
148
+ _change_control("pdr"),
149
+ ],
150
+ ),
151
+ _preset_gate(
152
+ "DDR",
153
+ "Detailed Design Review",
154
+ "Subsystem design baseline",
155
+ [
156
+ _check(
157
+ "ddr-verified",
158
+ "Every requirement is linked to a verifying test",
159
+ linked="Verified by",
160
+ ),
161
+ _no_findings("ddr"),
162
+ _change_control("ddr"),
163
+ ],
164
+ reviewers=1,
165
+ scope="system",
166
+ ),
167
+ _preset_gate(
168
+ "CDR",
169
+ "Critical Design Review",
170
+ "Product baseline",
171
+ [
172
+ _check(
173
+ "cdr-verified",
174
+ "Every requirement is linked to a verifying test",
175
+ linked="Verified by",
176
+ ),
177
+ _no_findings("cdr"),
178
+ _change_control("cdr"),
179
+ ],
180
+ ),
181
+ _preset_gate(
182
+ "TRR",
183
+ "Test Readiness Review",
184
+ "Test baseline",
185
+ [
186
+ _check(
187
+ "trr-test-plan",
188
+ "Every test belongs to a test plan",
189
+ applies_to=_TESTS,
190
+ linked="Test plan",
191
+ ),
192
+ _check(
193
+ "trr-test-method",
194
+ "Every test has a verification method",
195
+ applies_to=_TESTS,
196
+ filled="Verification Method",
197
+ ),
198
+ _no_findings("trr"),
199
+ _change_control("trr"),
200
+ ],
201
+ includes=["Requirement", "System", "Test", "Test plan"],
202
+ ),
203
+ _preset_gate(
204
+ "SVR",
205
+ "System Verification Review",
206
+ "Verified baseline",
207
+ [
208
+ _check(
209
+ "svr-results",
210
+ "Every test has passed",
211
+ applies_to=_TESTS,
212
+ **{"in": {"Result": ["Passed"]}},
213
+ ),
214
+ _check(
215
+ "svr-compliance",
216
+ "Every requirement is compliant or waived",
217
+ **{"in": _COMPLIANT},
218
+ ),
219
+ _no_findings("svr"),
220
+ _change_control("svr"),
221
+ ],
222
+ captures_status=True,
223
+ ),
224
+ _preset_gate(
225
+ "FCA",
226
+ "Functional Configuration Audit",
227
+ "Functional audit baseline",
228
+ [
229
+ _check(
230
+ "fca-compliance",
231
+ "Every requirement is compliant or waived",
232
+ **{"in": _COMPLIANT},
233
+ ),
234
+ _no_findings("fca"),
235
+ _change_control("fca"),
236
+ ],
237
+ captures_status=True,
238
+ ),
239
+ _preset_gate(
240
+ "PCA",
241
+ "Physical Configuration Audit",
242
+ "As-built product baseline",
243
+ [
244
+ _check(
245
+ "pca-compliance",
246
+ "Every requirement is compliant or waived",
247
+ **{"in": _COMPLIANT},
248
+ ),
249
+ _no_findings("pca"),
250
+ _change_control("pca"),
251
+ ],
252
+ captures_status=True,
253
+ ),
254
+ ]
255
+
256
+ DEFAULT_CONFIG: dict[str, Any] = {
257
+ "id_column": None,
258
+ "title": "Requirements",
259
+ "document_mode": "review",
260
+ "traceability": False,
261
+ "csv": {"encoding": "utf-8-sig", "delimiter": ",", "strict_columns": True},
262
+ "columns": {
263
+ "hidden": [],
264
+ "order": [],
265
+ "labels": {},
266
+ "widths": {},
267
+ "markdown": None,
268
+ },
269
+ "diff": {
270
+ "initial_view": "unified",
271
+ "initial_changes_only": False,
272
+ "include_unchanged": True,
273
+ "ignore_columns": [],
274
+ "ignore_trailing_whitespace": False,
275
+ },
276
+ "comments": {
277
+ "enabled": True,
278
+ "reviewer_initials": None,
279
+ "csv_column": "Comments",
280
+ "csv_format": "json",
281
+ },
282
+ "quality": {
283
+ "requirement_column": None,
284
+ "required_columns": [],
285
+ "obligation": "shall",
286
+ },
287
+ "xlsx": {"sources": []},
288
+ "baselines": {
289
+ "store": "git",
290
+ "git": {"tag_prefix": "baseline/"},
291
+ "files": {"directory": "baselines"},
292
+ "enforcement": "advisory",
293
+ "status_columns": ["Compliance", "Result"],
294
+ "change_control": True,
295
+ },
296
+ "gates": PRESET_GATES,
297
+ }
298
+
299
+
300
+ @dataclass(frozen=True)
301
+ class CsvSource:
302
+ """A CSV to read: a file on disk (text is None) or already-read text with a display label."""
303
+
304
+ label: str
305
+ text: str | None
306
+ path: Path
307
+ warnings: tuple[str, ...] = ()
308
+ comments: tuple[dict[str, Any], ...] = ()
309
+
310
+ @property
311
+ def filename(self) -> str:
312
+ return self.path.name if self.text is None else self.label
313
+
314
+
315
+ @dataclass
316
+ class CsvTable:
317
+ label: str
318
+ filename: str
319
+ headers: list[str]
320
+ rows: list[dict[str, str]]
321
+ ids: list[str]
322
+ by_id: dict[str, dict[str, str]]
323
+ warnings: tuple[str, ...] = ()
324
+ comments: tuple[dict[str, Any], ...] = ()
325
+
326
+
327
+ def _merge_known(
328
+ base: dict[str, Any], override: dict[str, Any], prefix: str = ""
329
+ ) -> dict[str, Any]:
330
+ result = json.loads(json.dumps(base))
331
+ for key, value in override.items():
332
+ where = f"{prefix}.{key}" if prefix else key
333
+ if key not in base:
334
+ raise ConfigError(f"unknown configuration key: {where}")
335
+ if isinstance(base[key], dict):
336
+ if not isinstance(value, dict):
337
+ raise ConfigError(f"configuration value {where} must be an object")
338
+ result[key] = (
339
+ dict(value) if not base[key] else _merge_known(base[key], value, where)
340
+ )
341
+ else:
342
+ result[key] = value
343
+ return result
344
+
345
+
346
+ def load_config(
347
+ path: str | None, overrides: dict[str, Any] | None = None
348
+ ) -> dict[str, Any]:
349
+ supplied: dict[str, Any] = {}
350
+ if path:
351
+ try:
352
+ supplied = json.loads(Path(path).read_text(encoding="utf-8"))
353
+ except (OSError, UnicodeError) as exc:
354
+ raise ConfigError(f"cannot read configuration {path}: {exc}") from exc
355
+ except json.JSONDecodeError as exc:
356
+ raise ConfigError(
357
+ f"invalid JSON configuration {path}:{exc.lineno}:{exc.colno}: {exc.msg}"
358
+ ) from exc
359
+ if not isinstance(supplied, dict):
360
+ raise ConfigError("configuration root must be a JSON object")
361
+ config = _merge_known(DEFAULT_CONFIG, supplied)
362
+ for dotted, value in (overrides or {}).items():
363
+ if value is None:
364
+ continue
365
+ parts = dotted.split(".")
366
+ cursor = config
367
+ for part in parts[:-1]:
368
+ cursor = cursor[part]
369
+ cursor[parts[-1]] = value
370
+ validate_config(config)
371
+ return config
372
+
373
+
374
+ def validate_config(config: dict[str, Any]) -> None:
375
+ if not isinstance(config["traceability"], bool):
376
+ raise ConfigError("traceability must be true or false")
377
+ if config["traceability"] and config["id_column"] in TRACE_COLUMNS:
378
+ raise ConfigError("the ID column cannot be a reserved traceability column")
379
+ quality = config["quality"]
380
+ for field in ("requirement_column", "obligation"):
381
+ value = quality[field]
382
+ if value is not None and (not isinstance(value, str) or not value.strip()):
383
+ raise ConfigError(f"quality.{field} must be null or a non-empty string")
384
+ if not isinstance(quality["required_columns"], list) or not all(
385
+ isinstance(column, str) and column.strip()
386
+ for column in quality["required_columns"]
387
+ ):
388
+ raise ConfigError(
389
+ "quality.required_columns must be an array of non-empty column names"
390
+ )
391
+ if config["id_column"] is not None and not isinstance(config["id_column"], str):
392
+ raise ConfigError("id_column must be null or a string")
393
+ if not isinstance(config["title"], str):
394
+ raise ConfigError("title must be a string")
395
+ if config["document_mode"] not in {"review", "develop"}:
396
+ raise ConfigError("document_mode must be 'review' or 'develop'")
397
+ if not isinstance(config["csv"]["encoding"], str) or not config["csv"]["encoding"]:
398
+ raise ConfigError("csv.encoding must be a non-empty string")
399
+ delimiter = config["csv"]["delimiter"]
400
+ if (
401
+ not isinstance(delimiter, str)
402
+ or len(delimiter) != 1
403
+ or ord(delimiter) < 32
404
+ or ord(delimiter) == 127
405
+ ):
406
+ raise ConfigError("csv.delimiter must be one printable character")
407
+ if not isinstance(config["csv"]["strict_columns"], bool):
408
+ raise ConfigError("csv.strict_columns must be true or false")
409
+ for field in ("hidden", "order"):
410
+ value = config["columns"][field]
411
+ if not isinstance(value, list) or not all(isinstance(x, str) for x in value):
412
+ raise ConfigError(f"columns.{field} must be an array of strings")
413
+ markdown = config["columns"]["markdown"]
414
+ if markdown is not None and (
415
+ not isinstance(markdown, list) or not all(isinstance(x, str) for x in markdown)
416
+ ):
417
+ raise ConfigError("columns.markdown must be null or an array of strings")
418
+ for field in ("labels", "widths"):
419
+ value = config["columns"][field]
420
+ if not isinstance(value, dict) or not all(
421
+ isinstance(k, str) and isinstance(v, str) for k, v in value.items()
422
+ ):
423
+ raise ConfigError(f"columns.{field} must be an object of string values")
424
+ width_pattern = re.compile(r"^(?:[1-9]\d*(?:\.\d+)?)(?:px|rem|em|ch|%)$")
425
+ for column, width in config["columns"]["widths"].items():
426
+ if not width_pattern.fullmatch(width):
427
+ raise ConfigError(
428
+ f"unsafe or invalid width for column {column!r}: {width!r}"
429
+ )
430
+ if config["diff"]["initial_view"] not in {"unified", "side-by-side"}:
431
+ raise ConfigError("diff.initial_view must be 'unified' or 'side-by-side'")
432
+ for field in (
433
+ "initial_changes_only",
434
+ "include_unchanged",
435
+ "ignore_trailing_whitespace",
436
+ ):
437
+ if not isinstance(config["diff"][field], bool):
438
+ raise ConfigError(f"diff.{field} must be true or false")
439
+ ignored = config["diff"]["ignore_columns"]
440
+ if not isinstance(ignored, list) or not all(isinstance(x, str) for x in ignored):
441
+ raise ConfigError("diff.ignore_columns must be an array of strings")
442
+ if config["comments"]["csv_format"] not in {"json", "text"}:
443
+ raise ConfigError("comments.csv_format must be 'json' or 'text'")
444
+ if not isinstance(config["comments"]["enabled"], bool):
445
+ raise ConfigError("comments.enabled must be true or false")
446
+ if (
447
+ not isinstance(config["comments"]["csv_column"], str)
448
+ or not config["comments"]["csv_column"]
449
+ ):
450
+ raise ConfigError("comments.csv_column must be a non-empty string")
451
+ initials = config["comments"]["reviewer_initials"]
452
+ if initials is not None and not valid_initials(initials):
453
+ raise ConfigError(
454
+ "comments.reviewer_initials must contain 1-16 non-control characters"
455
+ )
456
+ validate_xlsx_sources(config["xlsx"]["sources"])
457
+ validate_baselines(config["baselines"])
458
+ validate_gates(config["gates"])
459
+
460
+
461
+ def validate_baselines(baselines: dict[str, Any]) -> None:
462
+ if baselines["store"] not in {"git", "files"}:
463
+ raise ConfigError("baselines.store must be 'git' or 'files'")
464
+ prefix = baselines["git"]["tag_prefix"]
465
+ if not isinstance(prefix, str) or not prefix.endswith("/") or prefix == "/":
466
+ raise ConfigError(
467
+ "baselines.git.tag_prefix must be a non-empty string that ends with '/'"
468
+ )
469
+ directory = baselines["files"]["directory"]
470
+ if not isinstance(directory, str) or not directory.strip():
471
+ raise ConfigError("baselines.files.directory must be a non-empty string")
472
+ validate_enforcement(baselines["enforcement"], "baselines.enforcement")
473
+ columns = baselines["status_columns"]
474
+ if not isinstance(columns, list) or not all(isinstance(x, str) for x in columns):
475
+ raise ConfigError("baselines.status_columns must be an array of strings")
476
+ if not isinstance(baselines["change_control"], bool):
477
+ raise ConfigError("baselines.change_control must be true or false")
478
+
479
+
480
+ ENFORCEMENT_LEVELS = ("advisory", "strict")
481
+
482
+
483
+ def validate_enforcement(value: Any, where: str) -> None:
484
+ if value not in ENFORCEMENT_LEVELS:
485
+ raise ConfigError(f"{where} must be 'advisory' or 'strict'")
486
+
487
+
488
+ def resolve_gate_enforcement(gate: dict[str, Any], default: str) -> dict[str, Any]:
489
+ """The gate with every check carrying its effective enforcement: the check's, else the gate's, else the default."""
490
+ level = gate.get("enforcement", default)
491
+ return {
492
+ **gate,
493
+ "checks": [
494
+ {**check, "enforcement": check.get("enforcement", level)}
495
+ for check in gate.get("checks", [])
496
+ ],
497
+ }
498
+
499
+
500
+ _GATE_KEYS = {
501
+ "id",
502
+ "name",
503
+ "produces",
504
+ "scope",
505
+ "includes",
506
+ "captures_status",
507
+ "checks",
508
+ "signoff",
509
+ "enforcement",
510
+ }
511
+
512
+
513
+ def validate_gates(gates: Any) -> None:
514
+ from .gates import validate_checks
515
+ from .signoff import validate_roles
516
+
517
+ if not isinstance(gates, list) or not all(isinstance(g, dict) for g in gates):
518
+ raise ConfigError("gates must be an array of objects")
519
+ seen: set[str] = set()
520
+ for index, gate in enumerate(gates):
521
+ where = f"gates[{index}]"
522
+ unknown = sorted(set(gate) - _GATE_KEYS)
523
+ if unknown:
524
+ raise ConfigError(f"unknown configuration key: {where}.{unknown[0]}")
525
+ gate_id = gate.get("id")
526
+ if not isinstance(gate_id, str) or not gate_id.strip():
527
+ raise ConfigError(f"{where}.id must be a non-empty string")
528
+ where = f"gates[{index}] ({gate_id})"
529
+ if "/" in gate_id:
530
+ raise ConfigError(f"{where}: gate ID cannot contain '/'")
531
+ if re.search(r"-\d+$", gate_id):
532
+ raise ConfigError(
533
+ f"{where}: gate ID cannot end in '-' and a number; numbers are reserved for re-baselines"
534
+ )
535
+ if gate_id in seen:
536
+ raise ConfigError(f"{where}: duplicate gate ID")
537
+ seen.add(gate_id)
538
+ for key in ("name", "produces"):
539
+ if key in gate and not isinstance(gate[key], str):
540
+ raise ConfigError(f"{where}.{key} must be a string")
541
+ if gate.get("scope", "table") not in {"table", "system"}:
542
+ raise ConfigError(f"{where}.scope must be 'table' or 'system'")
543
+ includes = gate.get("includes", ["Requirement", "System"])
544
+ if (
545
+ not isinstance(includes, list)
546
+ or not includes
547
+ or not all(x in RECORD_TYPES for x in includes)
548
+ ):
549
+ raise ConfigError(
550
+ f"{where}.includes must be a non-empty array of: {', '.join(RECORD_TYPES)}"
551
+ )
552
+ if not isinstance(gate.get("captures_status", False), bool):
553
+ raise ConfigError(f"{where}.captures_status must be true or false")
554
+ if "enforcement" in gate:
555
+ validate_enforcement(gate["enforcement"], f"{where}.enforcement")
556
+ validate_roles(gate.get("signoff", []), where)
557
+ validate_checks(gate.get("checks", []), where)
558
+
559
+
560
+ _XLSX_SOURCE_KEYS = {
561
+ "table",
562
+ "sheet",
563
+ "header_row",
564
+ "columns",
565
+ "exclude",
566
+ "type",
567
+ "id_prefix",
568
+ "set",
569
+ "id_from",
570
+ "lists",
571
+ "headings",
572
+ "merge",
573
+ }
574
+
575
+
576
+ def validate_xlsx_sources(sources: Any) -> None:
577
+ if not isinstance(sources, list) or not all(isinstance(s, dict) for s in sources):
578
+ raise ConfigError("xlsx.sources must be an array of objects")
579
+ for index, source in enumerate(sources):
580
+ where = f"xlsx.sources[{index}]"
581
+ unknown = sorted(set(source) - _XLSX_SOURCE_KEYS)
582
+ if unknown:
583
+ raise ConfigError(f"unknown configuration key: {where}.{unknown[0]}")
584
+ has_table, has_sheet = "table" in source, "sheet" in source
585
+ if has_table == has_sheet:
586
+ raise ConfigError(f"{where} needs exactly one of 'table' or 'sheet'")
587
+ for key in ("table", "sheet", "type", "id_prefix"):
588
+ if key in source and (
589
+ not isinstance(source[key], str) or not source[key].strip()
590
+ ):
591
+ raise ConfigError(f"{where}.{key} must be a non-empty string")
592
+ if "header_row" in source:
593
+ row = source["header_row"]
594
+ if not has_sheet:
595
+ raise ConfigError(f"{where}.header_row applies only with 'sheet'")
596
+ if isinstance(row, bool) or not isinstance(row, int) or row < 1:
597
+ raise ConfigError(
598
+ f"{where}.header_row must be a whole number of 1 or greater"
599
+ )
600
+ columns = source.get("columns", {})
601
+ if not isinstance(columns, dict) or not all(
602
+ isinstance(k, str) and isinstance(v, str) and v.strip()
603
+ for k, v in columns.items()
604
+ ):
605
+ raise ConfigError(
606
+ f"{where}.columns must be an object of source header to output header strings"
607
+ )
608
+ exclude = source.get("exclude", [])
609
+ if not isinstance(exclude, list) or not all(
610
+ isinstance(x, str) for x in exclude
611
+ ):
612
+ raise ConfigError(f"{where}.exclude must be an array of strings")
613
+ constants = source.get("set", {})
614
+ if not isinstance(constants, dict) or not all(
615
+ isinstance(k, str) and k.strip() and isinstance(v, str)
616
+ for k, v in constants.items()
617
+ ):
618
+ raise ConfigError(
619
+ f"{where}.set must be an object of output header to constant string"
620
+ )
621
+ if "type" in source and "Type" in constants:
622
+ raise ConfigError(f"{where} gives both 'type' and set.Type; use one")
623
+ if "merge" in source:
624
+ if not isinstance(source["merge"], bool):
625
+ raise ConfigError(f"{where}.merge must be true or false")
626
+ if source["merge"]:
627
+ given = [
628
+ key
629
+ for key in ("type", "id_prefix", "id_from", "headings")
630
+ if key in source
631
+ ]
632
+ given += ["set.Type"] if "Type" in constants else []
633
+ if given:
634
+ raise ConfigError(
635
+ f"{where} is a merge source, which adds no records, so it cannot give {', '.join(given)}"
636
+ )
637
+ if "id_from" in source:
638
+ id_from = source["id_from"]
639
+ if (
640
+ not isinstance(id_from, list)
641
+ or not id_from
642
+ or not all(isinstance(x, str) and x.strip() for x in id_from)
643
+ ):
644
+ raise ConfigError(
645
+ f"{where}.id_from must be a non-empty array of column names"
646
+ )
647
+ if "id_prefix" not in source:
648
+ raise ConfigError(f"{where}.id_from needs id_prefix")
649
+ if "headings" in source:
650
+ validate_headings(source["headings"], f"{where}.headings")
651
+ lists = source.get("lists", {})
652
+ if not isinstance(lists, dict) or not all(
653
+ isinstance(k, str) and isinstance(v, str) and v for k, v in lists.items()
654
+ ):
655
+ raise ConfigError(
656
+ f'{where}.lists must be an object of output header to separator string (or "newline")'
657
+ )
658
+ if sources and all(source.get("merge") is True for source in sources):
659
+ raise ConfigError(
660
+ "xlsx.sources: every source is a merge source; at least one must add the records they fill in"
661
+ )
662
+
663
+
664
+ def validate_headings(headings: Any, where: str) -> None:
665
+ if not isinstance(headings, dict):
666
+ raise ConfigError(f"{where} must be an object with column, values and into")
667
+ unknown = sorted(set(headings) - {"column", "values", "text", "into"})
668
+ if unknown:
669
+ raise ConfigError(f"unknown configuration key: {where}.{unknown[0]}")
670
+ for key in ("column", "into"):
671
+ if not isinstance(headings.get(key), str) or not headings[key].strip():
672
+ raise ConfigError(
673
+ f"{where}.{key} is required and must be a non-empty string"
674
+ )
675
+ if "text" in headings and (
676
+ not isinstance(headings["text"], str) or not headings["text"].strip()
677
+ ):
678
+ raise ConfigError(f"{where}.text must be a non-empty string")
679
+ values = headings.get("values")
680
+ if (
681
+ not isinstance(values, list)
682
+ or not values
683
+ or not all(isinstance(v, str) and v.strip() for v in values)
684
+ ):
685
+ raise ConfigError(
686
+ f"{where}.values must be a non-empty array of the type values that mark a heading row"
687
+ )
688
+
689
+
690
+ def valid_initials(value: Any) -> bool:
691
+ return (
692
+ isinstance(value, str)
693
+ and 1 <= len(value.strip()) <= 16
694
+ and not any(ord(c) < 32 or ord(c) == 127 for c in value)
695
+ )
696
+
697
+
698
+ def read_csv_table(path: str, config: dict[str, Any]) -> CsvTable:
699
+ return read_csv_source(CsvSource(str(Path(path)), None, Path(path)), config)
700
+
701
+
702
+ def read_csv_source(csv_source: CsvSource, config: dict[str, Any]) -> CsvTable:
703
+ source = csv_source.label
704
+ encoding = config["csv"]["encoding"]
705
+ delimiter = config["csv"]["delimiter"]
706
+ if csv_source.text is not None:
707
+ handle: Any = io.StringIO(csv_source.text, newline="")
708
+ else:
709
+ try:
710
+ handle = csv_source.path.open("r", encoding=encoding, newline="")
711
+ except (OSError, LookupError) as exc:
712
+ raise InputError(f"cannot open {source}: {exc}") from exc
713
+ with handle:
714
+ reader = csv.reader(handle, delimiter=delimiter, strict=True)
715
+ try:
716
+ headers = next(reader)
717
+ except StopIteration as exc:
718
+ raise InputError(f"{source}: CSV file is empty") from exc
719
+ except csv.Error as exc:
720
+ raise InputError(f"{source}: invalid CSV header: {exc}") from exc
721
+ if not headers:
722
+ raise InputError(f"{source}: CSV header is empty")
723
+ if any(h == "" for h in headers):
724
+ raise InputError(f"{source}: empty column name in header")
725
+ duplicates = sorted({h for h in headers if headers.count(h) > 1})
726
+ if duplicates:
727
+ raise InputError(
728
+ f"{source}: duplicate column name(s): {', '.join(duplicates)}"
729
+ )
730
+ id_column = config.get("id_column")
731
+ if not id_column:
732
+ raise ConfigError(
733
+ "an ID column is required; use --id-column or set id_column in the config"
734
+ )
735
+ if id_column not in headers:
736
+ raise InputError(f"{source}: ID column {id_column!r} is not present")
737
+ rows: list[dict[str, str]] = []
738
+ ids: list[str] = []
739
+ by_id: dict[str, dict[str, str]] = {}
740
+ record_by_id: dict[str, int] = {}
741
+ try:
742
+ for record_number, values in enumerate(reader, start=2):
743
+ if values == []:
744
+ continue
745
+ if config["csv"]["strict_columns"] and len(values) != len(headers):
746
+ raise InputError(
747
+ f"{source}: record {record_number} has {len(values)} fields; expected {len(headers)}"
748
+ )
749
+ if len(values) < len(headers):
750
+ values += [""] * (len(headers) - len(values))
751
+ elif len(values) > len(headers):
752
+ values = values[: len(headers)]
753
+ row = {
754
+ header: value.replace("\r\n", "\n").replace("\r", "\n")
755
+ for header, value in zip(headers, values)
756
+ }
757
+ normalized_id = row[id_column].strip()
758
+ if not normalized_id:
759
+ raise InputError(
760
+ f"{source}: record {record_number} has an empty ID in {id_column!r}"
761
+ )
762
+ if normalized_id in by_id:
763
+ raise InputError(
764
+ f"{source}: duplicate ID {normalized_id!r} at records "
765
+ f"{record_by_id[normalized_id]} and {record_number}"
766
+ )
767
+ rows.append(row)
768
+ ids.append(normalized_id)
769
+ by_id[normalized_id] = row
770
+ record_by_id[normalized_id] = record_number
771
+ except (csv.Error, UnicodeError) as exc:
772
+ raise InputError(
773
+ f"{source}: invalid CSV near record {reader.line_num}: {exc}"
774
+ ) from exc
775
+ return CsvTable(
776
+ source,
777
+ csv_source.filename,
778
+ headers,
779
+ rows,
780
+ ids,
781
+ by_id,
782
+ csv_source.warnings,
783
+ csv_source.comments,
784
+ )
785
+
786
+
787
+ def effective_columns(
788
+ old: CsvTable | None, new: CsvTable, config: dict[str, Any]
789
+ ) -> list[str]:
790
+ headers = list(new.headers)
791
+ if old:
792
+ headers.extend(h for h in old.headers if h not in headers)
793
+ configured = config["columns"]["order"]
794
+ missing = [h for h in configured if h not in headers]
795
+ if missing:
796
+ raise ConfigError(
797
+ f"configured ordered column(s) not found: {', '.join(missing)}"
798
+ )
799
+ return configured + [h for h in headers if h not in configured]
800
+
801
+
802
+ def _comparison_value(value: str, config: dict[str, Any]) -> str:
803
+ if config["diff"]["ignore_trailing_whitespace"]:
804
+ return "\n".join(line.rstrip() for line in value.split("\n"))
805
+ return value
806
+
807
+
808
+ def _tokens(line: str) -> list[str]:
809
+ return re.findall(r"\s+|\w+|[^\w\s]", line, flags=re.UNICODE)
810
+
811
+
812
+ def _word_pair(old: str, new: str) -> tuple[str, str]:
813
+ a, b = _tokens(old), _tokens(new)
814
+ matcher = difflib.SequenceMatcher(None, a, b, autojunk=False)
815
+ old_parts: list[str] = []
816
+ new_parts: list[str] = []
817
+ for tag, i1, i2, j1, j2 in matcher.get_opcodes():
818
+ left = html.escape("".join(a[i1:i2]))
819
+ right = html.escape("".join(b[j1:j2]))
820
+ if tag == "equal":
821
+ old_parts.append(left)
822
+ new_parts.append(right)
823
+ else:
824
+ if left:
825
+ old_parts.append(f'<span class="diff-word removed-word">{left}</span>')
826
+ if right:
827
+ new_parts.append(f'<span class="diff-word added-word">{right}</span>')
828
+ return "".join(old_parts), "".join(new_parts)
829
+
830
+
831
+ def render_diff(old: str, new: str) -> dict[str, str]:
832
+ # Do not manufacture a highlighted blank line for a genuinely empty side.
833
+ # This is especially important for added and removed rows in unified view.
834
+ if old == "" and new != "":
835
+ new_html = "".join(
836
+ f'<div class="diff-line added-line">{html.escape(line) or "&nbsp;"}</div>'
837
+ for line in new.splitlines()
838
+ )
839
+ return {"oldHtml": "", "newHtml": new_html, "unifiedHtml": new_html}
840
+ if new == "" and old != "":
841
+ old_html = "".join(
842
+ f'<div class="diff-line removed-line">{html.escape(line) or "&nbsp;"}</div>'
843
+ for line in old.splitlines()
844
+ )
845
+ return {"oldHtml": old_html, "newHtml": "", "unifiedHtml": old_html}
846
+ if max(len(old), len(new)) > 100_000:
847
+ old_html = html.escape(old)
848
+ new_html = html.escape(new)
849
+ return {
850
+ "oldHtml": f'<div class="diff-line removed-line">{old_html}</div>',
851
+ "newHtml": f'<div class="diff-line added-line">{new_html}</div>',
852
+ "unifiedHtml": (
853
+ f'<div class="diff-line removed-line">{old_html}</div>'
854
+ f'<div class="diff-line added-line">{new_html}</div>'
855
+ ),
856
+ }
857
+ a = old.splitlines() or ([""] if old == "" else [])
858
+ b = new.splitlines() or ([""] if new == "" else [])
859
+ matcher = difflib.SequenceMatcher(None, a, b, autojunk=False)
860
+ old_out: list[str] = []
861
+ new_out: list[str] = []
862
+ unified: list[str] = []
863
+ for tag, i1, i2, j1, j2 in matcher.get_opcodes():
864
+ if tag == "equal":
865
+ for line in a[i1:i2]:
866
+ escaped = html.escape(line) or "&nbsp;"
867
+ block = f'<div class="diff-line context-line">{escaped}</div>'
868
+ old_out.append(block)
869
+ new_out.append(block)
870
+ unified.append(block)
871
+ elif tag == "delete":
872
+ for line in a[i1:i2]:
873
+ block = f'<div class="diff-line removed-line">{html.escape(line) or "&nbsp;"}</div>'
874
+ old_out.append(block)
875
+ unified.append(block)
876
+ elif tag == "insert":
877
+ for line in b[j1:j2]:
878
+ block = f'<div class="diff-line added-line">{html.escape(line) or "&nbsp;"}</div>'
879
+ new_out.append(block)
880
+ unified.append(block)
881
+ else:
882
+ old_lines, new_lines = a[i1:i2], b[j1:j2]
883
+ count = max(len(old_lines), len(new_lines))
884
+ for index in range(count):
885
+ left = old_lines[index] if index < len(old_lines) else None
886
+ right = new_lines[index] if index < len(new_lines) else None
887
+ if left is not None and right is not None:
888
+ left_html, right_html = _word_pair(left, right)
889
+ old_block = f'<div class="diff-line removed-line">{left_html or "&nbsp;"}</div>'
890
+ new_block = f'<div class="diff-line added-line">{right_html or "&nbsp;"}</div>'
891
+ old_out.append(old_block)
892
+ new_out.append(new_block)
893
+ unified.extend((old_block, new_block))
894
+ elif left is not None:
895
+ block = f'<div class="diff-line removed-line">{html.escape(left) or "&nbsp;"}</div>'
896
+ old_out.append(block)
897
+ unified.append(block)
898
+ elif right is not None:
899
+ block = f'<div class="diff-line added-line">{html.escape(right) or "&nbsp;"}</div>'
900
+ new_out.append(block)
901
+ unified.append(block)
902
+ return {
903
+ "oldHtml": "".join(old_out),
904
+ "newHtml": "".join(new_out),
905
+ "unifiedHtml": "".join(unified),
906
+ }
907
+
908
+
909
+ def _estimated_column_width(name: str, rows: list[dict[str, Any]]) -> int:
910
+ """Estimate a compact initial pixel width without letting outliers dominate."""
911
+ lengths = [len(name)]
912
+ multiline = False
913
+ for row in rows[:1000]:
914
+ cell = row["cells"][name]
915
+ for value in (cell.get("old"), cell.get("new")):
916
+ if value is None:
917
+ continue
918
+ text = str(value)
919
+ multiline = multiline or "\n" in text
920
+ lengths.extend(len(line) for line in text.splitlines() or [""])
921
+ lengths.sort()
922
+ percentile = lengths[min(len(lengths) - 1, int((len(lengths) - 1) * 0.90))]
923
+ useful_chars = max(len(name) + 2, percentile)
924
+ cap = 44 if multiline or useful_chars > 36 else 32
925
+ return max(82, min(460, 30 + min(useful_chars, cap) * 7))
926
+
927
+
928
+ def _ordered_diff_ids(old: CsvTable, new: CsvTable) -> list[str]:
929
+ result = list(new.ids)
930
+ old_positions = {value: index for index, value in enumerate(old.ids)}
931
+ removed = [value for value in old.ids if value not in new.by_id]
932
+ for removed_id in removed:
933
+ following = next(
934
+ (
935
+ value
936
+ for value in old.ids[old_positions[removed_id] + 1 :]
937
+ if value in result
938
+ ),
939
+ None,
940
+ )
941
+ if following is None:
942
+ result.append(removed_id)
943
+ else:
944
+ result.insert(result.index(following), removed_id)
945
+ return result
946
+
947
+
948
+ def _cell(
949
+ column: str,
950
+ old_value: str | None,
951
+ new_value: str | None,
952
+ changed: bool,
953
+ markdown: bool,
954
+ ) -> dict[str, Any]:
955
+ old_text = "" if old_value is None else old_value
956
+ new_text = "" if new_value is None else new_value
957
+ result: dict[str, Any] = {
958
+ "column": column,
959
+ "old": old_value,
960
+ "new": new_value,
961
+ "changed": changed,
962
+ "search": f"{old_text}\n{new_text}".casefold(),
963
+ }
964
+ if changed:
965
+ result.update(render_diff(old_text, new_text))
966
+ else:
967
+ display = new_text if new_value is not None else old_text
968
+ rendered = (
969
+ render_markdown(display)
970
+ if markdown
971
+ else html.escape(display).replace("\n", "<br>")
972
+ )
973
+ result.update(
974
+ {"oldHtml": rendered, "newHtml": rendered, "unifiedHtml": rendered}
975
+ )
976
+ return result
977
+
978
+
979
+ def generation_time() -> str:
980
+ epoch = os.environ.get("SOURCE_DATE_EPOCH")
981
+ if epoch is not None:
982
+ try:
983
+ return (
984
+ datetime.fromtimestamp(int(epoch), timezone.utc)
985
+ .astimezone()
986
+ .isoformat(timespec="seconds")
987
+ )
988
+ except (ValueError, OverflowError, OSError) as exc:
989
+ raise ConfigError(
990
+ "SOURCE_DATE_EPOCH must be a valid integer timestamp"
991
+ ) from exc
992
+ return datetime.now().astimezone().isoformat(timespec="seconds")
993
+
994
+
995
+ def prepare_trace_table(
996
+ table: CsvTable, config: dict[str, Any]
997
+ ) -> tuple[CsvTable, list[dict[str, Any]]]:
998
+ comments_config = config["comments"]
999
+ comment_column = comments_config["csv_column"]
1000
+ if comments_config["enabled"] and comment_column in (
1001
+ *TRACE_COLUMNS,
1002
+ config["id_column"],
1003
+ ):
1004
+ raise ConfigError("the comments column cannot also store traceability data")
1005
+ managed_comments = (
1006
+ comments_config["enabled"]
1007
+ and comments_config["csv_format"] == "json"
1008
+ and comment_column in table.headers
1009
+ )
1010
+ headers = [
1011
+ name for name in table.headers if not managed_comments or name != comment_column
1012
+ ]
1013
+ headers.extend(name for name in TRACE_COLUMNS if name not in headers)
1014
+ rows = [{name: row.get(name, "") for name in headers} for row in table.rows]
1015
+ comments = []
1016
+ if managed_comments:
1017
+ for rid, row in zip(table.ids, table.rows):
1018
+ message = f"{table.label}: expected exported JSON comments in {comment_column!r} for {rid!r}"
1019
+ try:
1020
+ records = json.loads(row[comment_column] or "[]")
1021
+ except ValueError as exc:
1022
+ raise InputError(message) from exc
1023
+ if not isinstance(records, list):
1024
+ raise InputError(message)
1025
+ for record in records:
1026
+ if (
1027
+ not isinstance(record, dict)
1028
+ or record.get("column") not in headers
1029
+ or record["column"] == config["id_column"]
1030
+ or not valid_initials(record.get("initials"))
1031
+ or not all(
1032
+ isinstance(record.get(key), str) for key in ("created", "text")
1033
+ )
1034
+ or (
1035
+ record.get("modified") is not None
1036
+ and not isinstance(record["modified"], str)
1037
+ )
1038
+ ):
1039
+ raise InputError(message)
1040
+ comments.append(
1041
+ {
1042
+ "id": f"imported-{len(comments)}",
1043
+ "requirementId": rid,
1044
+ **{
1045
+ key: record.get(key)
1046
+ for key in (
1047
+ "column",
1048
+ "initials",
1049
+ "created",
1050
+ "modified",
1051
+ "text",
1052
+ )
1053
+ },
1054
+ }
1055
+ )
1056
+ prepared = CsvTable(
1057
+ table.label,
1058
+ table.filename,
1059
+ headers,
1060
+ rows,
1061
+ table.ids,
1062
+ dict(zip(table.ids, rows)),
1063
+ table.warnings,
1064
+ table.comments,
1065
+ )
1066
+ return prepared, comments
1067
+
1068
+
1069
+ def build_report_data(
1070
+ new: CsvTable,
1071
+ config: dict[str, Any],
1072
+ output_name: str,
1073
+ old: CsvTable | None = None,
1074
+ baselines: Sequence[dict[str, Any]] = (),
1075
+ controls: dict[str, Any] | None = None,
1076
+ ) -> dict[str, Any]:
1077
+ comments = []
1078
+ traceability = config["traceability"]
1079
+ if traceability:
1080
+ new, comments = prepare_trace_table(new, config)
1081
+ if old:
1082
+ old, old_comments = prepare_trace_table(old, config)
1083
+ comments.extend(
1084
+ {**comment, "id": f"old-{comment['id']}"}
1085
+ for comment in old_comments
1086
+ if comment["requirementId"] not in new.by_id
1087
+ )
1088
+ if config["comments"]["enabled"]:
1089
+ comments.extend(c for c in new.comments if c["column"] in new.headers)
1090
+ if old:
1091
+ comments.extend(
1092
+ {**c, "id": f"old-{c['id']}"}
1093
+ for c in old.comments
1094
+ if c["requirementId"] not in new.by_id and c["column"] in old.headers
1095
+ )
1096
+ columns = effective_columns(old, new, config)
1097
+ quality = config["quality"]
1098
+ quality_columns = set(quality["required_columns"])
1099
+ if quality["requirement_column"] is not None:
1100
+ quality_columns.add(quality["requirement_column"])
1101
+ invalid_quality_columns = quality_columns.difference(
1102
+ set(new.headers) - {config["id_column"]}
1103
+ )
1104
+ if invalid_quality_columns:
1105
+ names = ", ".join(sorted(invalid_quality_columns))
1106
+ raise ConfigError(
1107
+ f"quality columns must be non-ID columns in the current CSV: {names}"
1108
+ )
1109
+ markdown_columns = config["columns"]["markdown"]
1110
+ markdown_set = set(columns if markdown_columns is None else markdown_columns)
1111
+ for name in markdown_set:
1112
+ if name not in columns:
1113
+ raise ConfigError(f"configured Markdown column not found: {name}")
1114
+ ignored = set(config["diff"]["ignore_columns"])
1115
+ unknown_ignored = ignored.difference(columns)
1116
+ if unknown_ignored:
1117
+ raise ConfigError(
1118
+ f"ignored diff column(s) not found: {', '.join(sorted(unknown_ignored))}"
1119
+ )
1120
+ rows: list[dict[str, Any]] = []
1121
+ if old is None:
1122
+ ids = new.ids
1123
+ else:
1124
+ ids = _ordered_diff_ids(old, new)
1125
+ counts = {"added": 0, "removed": 0, "changed": 0, "unchanged": 0}
1126
+ for requirement_id in ids:
1127
+ old_row = old.by_id.get(requirement_id) if old else None
1128
+ new_row = new.by_id.get(requirement_id)
1129
+ if old is None:
1130
+ status = "unchanged"
1131
+ elif old_row is None:
1132
+ status = "added"
1133
+ elif new_row is None:
1134
+ status = "removed"
1135
+ else:
1136
+ changed_columns = [
1137
+ column
1138
+ for column in columns
1139
+ if column not in ignored
1140
+ and _comparison_value(old_row.get(column, ""), config)
1141
+ != _comparison_value(new_row.get(column, ""), config)
1142
+ ]
1143
+ status = "changed" if changed_columns else "unchanged"
1144
+ counts[status] += 1
1145
+ cells: dict[str, dict[str, Any]] = {}
1146
+ for column in columns:
1147
+ old_value = (
1148
+ old_row.get(column, "")
1149
+ if old_row is not None and column in old_row
1150
+ else None
1151
+ )
1152
+ new_value = (
1153
+ new_row.get(column, "")
1154
+ if new_row is not None and column in new_row
1155
+ else None
1156
+ )
1157
+ changed = (
1158
+ old is not None
1159
+ and column not in ignored
1160
+ and _comparison_value("" if old_value is None else old_value, config)
1161
+ != _comparison_value("" if new_value is None else new_value, config)
1162
+ )
1163
+ cells[column] = _cell(
1164
+ column, old_value, new_value, changed, column in markdown_set
1165
+ )
1166
+ rows.append(
1167
+ {
1168
+ "id": requirement_id,
1169
+ "status": status,
1170
+ "sourceOrder": len(rows),
1171
+ "cells": cells,
1172
+ }
1173
+ )
1174
+ if old is not None and not config["diff"]["include_unchanged"]:
1175
+ rows = [row for row in rows if row["status"] != "unchanged"]
1176
+ labels = config["columns"]["labels"]
1177
+ trace_widths = (
1178
+ {"Name": 150, "Definition": 320, "Aliases": 140, "Rationale": 260}
1179
+ if traceability
1180
+ else {}
1181
+ )
1182
+ column_data = [
1183
+ {
1184
+ "name": name,
1185
+ "label": labels.get(name, name),
1186
+ "hidden": name in config["columns"]["hidden"],
1187
+ "width": config["columns"]["widths"].get(name),
1188
+ "estimatedWidth": max(
1189
+ _estimated_column_width(name, rows), trace_widths.get(name, 0)
1190
+ ),
1191
+ "oldOnly": bool(old and name in old.headers and name not in new.headers),
1192
+ "newOnly": bool(old and name in new.headers and name not in old.headers),
1193
+ "markdown": name in markdown_set,
1194
+ }
1195
+ for name in columns
1196
+ ]
1197
+ warnings: list[str] = [*new.warnings, *(old.warnings if old else ())]
1198
+ optional_names = set(config["columns"]["hidden"])
1199
+ optional_names.update(config["columns"]["labels"])
1200
+ optional_names.update(config["columns"]["widths"])
1201
+ unknown_optional = sorted(optional_names.difference(columns))
1202
+ if unknown_optional:
1203
+ warnings.append(
1204
+ "configured optional column(s) not found: " + ", ".join(unknown_optional)
1205
+ )
1206
+ if config["id_column"] in config["columns"]["hidden"]:
1207
+ warnings.append("the requirement ID column is initially hidden")
1208
+ stem = Path(output_name).stem
1209
+ stem = re.sub(r"-reviewed-[A-Za-z0-9_-]+(?:-\d{8}T\d{6})?$", "", stem)
1210
+ return {
1211
+ "format": "reqtable-report",
1212
+ "formatVersion": 1,
1213
+ "meta": {
1214
+ "title": config["title"],
1215
+ "mode": "diff" if old else "snapshot",
1216
+ "documentMode": config["document_mode"],
1217
+ "generated": generation_time(),
1218
+ "newFilename": new.filename,
1219
+ "oldFilename": old.filename if old else None,
1220
+ "baseFilename": stem,
1221
+ "idColumn": config["id_column"],
1222
+ "counts": counts,
1223
+ "ignoredColumns": sorted(ignored),
1224
+ "warnings": warnings,
1225
+ },
1226
+ "options": {
1227
+ "initialView": config["diff"]["initial_view"],
1228
+ "initialChangesOnly": config["diff"]["initial_changes_only"],
1229
+ "commentsEnabled": config["comments"]["enabled"],
1230
+ "reviewerInitials": config["comments"]["reviewer_initials"] or "",
1231
+ "includeUnchanged": config["diff"]["include_unchanged"],
1232
+ "ignoreTrailingWhitespace": config["diff"]["ignore_trailing_whitespace"],
1233
+ "commentsColumn": config["comments"]["csv_column"],
1234
+ "commentsFormat": config["comments"]["csv_format"],
1235
+ "csvDelimiter": config["csv"]["delimiter"],
1236
+ "quality": quality,
1237
+ "traceability": traceability,
1238
+ },
1239
+ "columns": column_data,
1240
+ "rows": rows,
1241
+ "source": {
1242
+ "new": {"headers": new.headers, "rows": new.rows},
1243
+ "old": {"headers": old.headers, "rows": old.rows} if old else None,
1244
+ },
1245
+ "comments": comments,
1246
+ "baselines": list(baselines),
1247
+ "controls": controls,
1248
+ "gates": [
1249
+ resolve_gate_enforcement(gate, config["baselines"]["enforcement"])
1250
+ for gate in config["gates"]
1251
+ ],
1252
+ }
1253
+
1254
+
1255
+ def atomic_write_text(path: str, content: str, encoding: str = "utf-8") -> None:
1256
+ destination = Path(path)
1257
+ try:
1258
+ destination.parent.mkdir(parents=True, exist_ok=True)
1259
+ fd, temporary = tempfile.mkstemp(
1260
+ prefix=f".{destination.name}.", dir=destination.parent
1261
+ )
1262
+ try:
1263
+ with os.fdopen(fd, "w", encoding=encoding, newline="") as handle:
1264
+ handle.write(content)
1265
+ handle.flush()
1266
+ os.fsync(handle.fileno())
1267
+ os.replace(temporary, destination)
1268
+ except Exception:
1269
+ try:
1270
+ os.unlink(temporary)
1271
+ except OSError:
1272
+ pass
1273
+ raise
1274
+ except OSError as exc:
1275
+ raise OutputError(f"cannot write {destination}: {exc}") from exc
1276
+
1277
+
1278
+ def comments_as_text(comments: Iterable[dict[str, Any]]) -> str:
1279
+ blocks = []
1280
+ for comment in comments:
1281
+ timestamp = comment.get("modified") or comment.get("created", "")
1282
+ blocks.append(
1283
+ f"[{timestamp}] [{comment.get('initials', '')}] [{comment.get('column', '')}]\n"
1284
+ f"{comment.get('text', '')}"
1285
+ )
1286
+ return "\n\n---\n\n".join(blocks)
1287
+
1288
+
1289
+ def extract_report_json(report_path: str) -> dict[str, Any]:
1290
+ path = Path(report_path)
1291
+ try:
1292
+ source = path.read_text(encoding="utf-8")
1293
+ except (OSError, UnicodeError) as exc:
1294
+ raise InputError(f"cannot read report {path}: {exc}") from exc
1295
+ match = re.search(
1296
+ r'<script\s+id="reqtable-data"\s+type="application/json">(.*?)</script>',
1297
+ source,
1298
+ flags=re.DOTALL,
1299
+ )
1300
+ if not match:
1301
+ raise InputError(f"{path}: reqtable data block was not found")
1302
+ try:
1303
+ data = json.loads(match.group(1))
1304
+ except json.JSONDecodeError as exc:
1305
+ raise InputError(f"{path}: invalid embedded report data: {exc}") from exc
1306
+ if (
1307
+ not isinstance(data, dict)
1308
+ or data.get("format") != "reqtable-report"
1309
+ or data.get("formatVersion") not in (1, 2)
1310
+ ):
1311
+ raise InputError(f"{path}: unsupported or invalid reqtable report format")
1312
+ if not isinstance(data.get("comments"), list) or not isinstance(
1313
+ data.get("source"), dict
1314
+ ):
1315
+ raise InputError(f"{path}: malformed report data")
1316
+ for index, comment in enumerate(data["comments"]):
1317
+ required = {"id", "requirementId", "column", "initials", "created", "text"}
1318
+ if not isinstance(comment, dict) or not required.issubset(comment):
1319
+ raise InputError(f"{path}: malformed comment at index {index}")
1320
+ if not valid_initials(comment["initials"]):
1321
+ raise InputError(f"{path}: invalid reviewer initials in comment {index}")
1322
+ if not all(isinstance(comment[key], str) for key in required):
1323
+ raise InputError(f"{path}: invalid field type in comment {index}")
1324
+ if len(comment["text"]) > 1_000_000:
1325
+ raise InputError(f"{path}: comment {index} exceeds the size limit")
1326
+ return data
1327
+
1328
+
1329
+ def extract_comments_csv(
1330
+ report_path: str,
1331
+ output_path: str,
1332
+ comments_column: str,
1333
+ comments_format: str,
1334
+ csv_base: str,
1335
+ allow_comment_loss: bool,
1336
+ delimiter: str = ",",
1337
+ ) -> None:
1338
+ data = extract_report_json(report_path)
1339
+ source = data["source"]
1340
+ mode = data["meta"].get("mode")
1341
+ if mode == "snapshot":
1342
+ selected = source["new"]
1343
+ allowed_ids = {
1344
+ str(row.get(data["meta"]["idColumn"], "")).strip()
1345
+ for row in selected["rows"]
1346
+ }
1347
+ add_status = False
1348
+ else:
1349
+ if csv_base not in {"new", "old", "union"}:
1350
+ raise ConfigError("--csv-base must be new, old, or union")
1351
+ if csv_base in {"new", "old"}:
1352
+ selected = source[csv_base]
1353
+ if selected is None:
1354
+ raise InputError(f"report does not contain a {csv_base} snapshot")
1355
+ allowed_ids = {
1356
+ str(row.get(data["meta"]["idColumn"], "")).strip()
1357
+ for row in selected["rows"]
1358
+ }
1359
+ add_status = False
1360
+ else:
1361
+ id_column = data["meta"]["idColumn"]
1362
+ old_rows = {
1363
+ str(row.get(id_column, "")).strip(): row
1364
+ for row in source["old"]["rows"]
1365
+ }
1366
+ new_rows = {
1367
+ str(row.get(id_column, "")).strip(): row
1368
+ for row in source["new"]["rows"]
1369
+ }
1370
+ order = [
1371
+ str(row.get(id_column, "")).strip() for row in source["new"]["rows"]
1372
+ ]
1373
+ order += [
1374
+ str(row.get(id_column, "")).strip()
1375
+ for row in source["old"]["rows"]
1376
+ if str(row.get(id_column, "")).strip() not in new_rows
1377
+ ]
1378
+ headers = list(source["new"]["headers"]) + [
1379
+ h for h in source["old"]["headers"] if h not in source["new"]["headers"]
1380
+ ]
1381
+ if "_Diff Status" in headers:
1382
+ raise InputError(
1383
+ "cannot use union CSV base because '_Diff Status' already exists"
1384
+ )
1385
+ if data["formatVersion"] == 1:
1386
+ status_by_id = {row["id"]: row["status"] for row in data["rows"]}
1387
+ else:
1388
+ status_by_id = data.get("diffStatus")
1389
+ if not isinstance(status_by_id, dict):
1390
+ raise InputError(f"{report_path}: malformed diff status map")
1391
+ selected = {
1392
+ "headers": headers + ["_Diff Status"],
1393
+ "rows": [
1394
+ {
1395
+ **old_rows.get(rid, {}),
1396
+ **new_rows.get(rid, {}),
1397
+ "_Diff Status": status_by_id.get(rid, ""),
1398
+ }
1399
+ for rid in order
1400
+ ],
1401
+ }
1402
+ allowed_ids = set(order)
1403
+ add_status = True
1404
+ excluded = [c for c in data["comments"] if c["requirementId"] not in allowed_ids]
1405
+ if excluded and not allow_comment_loss:
1406
+ excluded_ids = sorted({c["requirementId"] for c in excluded})
1407
+ raise InputError(
1408
+ "selected CSV base would omit comments for requirement(s): "
1409
+ + ", ".join(excluded_ids)
1410
+ + "; select another --csv-base or pass --allow-comment-loss"
1411
+ )
1412
+ headers = list(selected["headers"])
1413
+ if comments_column in headers:
1414
+ raise InputError(
1415
+ f"comments column {comments_column!r} already exists; choose another name"
1416
+ )
1417
+ headers.append(comments_column)
1418
+ grouped: dict[str, list[dict[str, Any]]] = {}
1419
+ for comment in data["comments"]:
1420
+ if comment["requirementId"] in allowed_ids:
1421
+ exported = {
1422
+ key: comment.get(key)
1423
+ for key in ("column", "initials", "created", "modified", "text")
1424
+ }
1425
+ grouped.setdefault(comment["requirementId"], []).append(exported)
1426
+ id_column = data["meta"]["idColumn"]
1427
+ buffer = io.StringIO(newline="")
1428
+ writer = csv.DictWriter(
1429
+ buffer,
1430
+ fieldnames=headers,
1431
+ delimiter=delimiter,
1432
+ extrasaction="ignore",
1433
+ lineterminator="\n",
1434
+ )
1435
+ writer.writeheader()
1436
+ for source_row in selected["rows"]:
1437
+ row = dict(source_row)
1438
+ rid = str(row.get(id_column, "")).strip()
1439
+ comments = sorted(
1440
+ grouped.get(rid, []), key=lambda item: item.get("created") or ""
1441
+ )
1442
+ row[comments_column] = (
1443
+ json.dumps(comments, ensure_ascii=False, separators=(",", ":"))
1444
+ if comments_format == "json"
1445
+ else comments_as_text(comments)
1446
+ )
1447
+ writer.writerow(row)
1448
+ atomic_write_text(output_path, buffer.getvalue(), encoding="utf-8-sig")