reqtable 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
reqtable/xlsx.py ADDED
@@ -0,0 +1,1108 @@
1
+ """Read requirements from an .xlsx workbook with the standard library only.
2
+
3
+ Reading only: the workbook is never written. Cell values come out as the text
4
+ Excel displays for the stored value, so a block of cells can be fed to the CSV
5
+ pipeline unchanged. Merged cells keep their value in the top-left cell only.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import csv
11
+ import datetime as dt
12
+ import io
13
+ import json
14
+ import posixpath
15
+ import re
16
+ import zipfile
17
+ from dataclasses import dataclass, field
18
+ from pathlib import Path
19
+ from typing import Any, Iterator
20
+ from xml.etree import ElementTree as ET
21
+
22
+ from .core import DEFAULT_CONFIG, ConfigError, InputError, valid_initials
23
+
24
+ _MAIN = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"
25
+ _REL = "http://schemas.openxmlformats.org/package/2006/relationships"
26
+ _R_ID = "{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id"
27
+ _DC_MODIFIED = "{http://purl.org/dc/terms/}modified"
28
+ NS = {"m": _MAIN, "rel": _REL}
29
+
30
+ # Built-in number formats that Excel displays as dates or times (including the CJK locale ones).
31
+ _BUILTIN_DATE_FORMATS = frozenset(
32
+ {
33
+ 14,
34
+ 15,
35
+ 16,
36
+ 17,
37
+ 18,
38
+ 19,
39
+ 20,
40
+ 21,
41
+ 22,
42
+ 27,
43
+ 28,
44
+ 29,
45
+ 30,
46
+ 31,
47
+ 32,
48
+ 33,
49
+ 34,
50
+ 35,
51
+ 36,
52
+ 45,
53
+ 46,
54
+ 47,
55
+ 50,
56
+ 51,
57
+ 52,
58
+ 53,
59
+ 54,
60
+ 55,
61
+ 56,
62
+ 57,
63
+ 58,
64
+ }
65
+ )
66
+
67
+
68
+ @dataclass(frozen=True)
69
+ class Sheet:
70
+ name: str
71
+ part: str
72
+ state: str
73
+
74
+
75
+ @dataclass(frozen=True)
76
+ class Table:
77
+ name: str
78
+ sheet: str
79
+ ref: str
80
+ header_rows: int = 1
81
+ totals_rows: int = 0
82
+
83
+
84
+ @dataclass(frozen=True)
85
+ class XlsxSelection:
86
+ """Which block of a workbook to read, shared by every .xlsx input of one run."""
87
+
88
+ table: str | None = None
89
+ sheet: str | None = None
90
+ header_row: int | None = None
91
+
92
+ def __post_init__(self) -> None:
93
+ if self.table and self.sheet:
94
+ raise ConfigError(
95
+ "--table and --sheet cannot be combined; a table already belongs to one sheet"
96
+ )
97
+ if self.header_row is not None:
98
+ if not self.sheet:
99
+ raise ConfigError("--header-row needs --sheet")
100
+ if self.header_row < 1:
101
+ raise ConfigError("--header-row must be 1 or greater")
102
+
103
+ @property
104
+ def given(self) -> bool:
105
+ return bool(self.table or self.sheet or self.header_row is not None)
106
+
107
+
108
+ @dataclass(frozen=True)
109
+ class Block:
110
+ """The selected cells: `name` is the table or sheet they came from, row 0 is the header.
111
+
112
+ `rows[i]` is worksheet row `first_row + i` of `sheet`, and its first column is worksheet column `first_col`.
113
+ """
114
+
115
+ name: str
116
+ rows: list[list[str]]
117
+ first_row: int = 1
118
+ first_col: int = 1
119
+ sheet: str = ""
120
+
121
+
122
+ @dataclass(frozen=True)
123
+ class SourceStats:
124
+ """What one configured source gave: the rows it kept (None for a merge source, which adds no rows), the heading
125
+ rows it removed, and, when it has heading rows, each kept row's category in order."""
126
+
127
+ name: str
128
+ rows: int | None
129
+ headings_removed: int = 0
130
+ categories: tuple[str, ...] = ()
131
+
132
+
133
+ @dataclass(frozen=True)
134
+ class Imported:
135
+ """What an .xlsx input yields: the combined table as a block, repair warnings, Excel comments as review comments,
136
+ and what each configured source contributed."""
137
+
138
+ block: Block
139
+ warnings: tuple[str, ...] = ()
140
+ comments: tuple[dict[str, Any], ...] = ()
141
+ sources: tuple[SourceStats, ...] = ()
142
+
143
+
144
+ @dataclass(frozen=True)
145
+ class RawComment:
146
+ row: int
147
+ col: int
148
+ author: str
149
+ created: str
150
+ text: str
151
+
152
+
153
+ @dataclass
154
+ class Part:
155
+ """One source's rows on their way to the combined table, with the worksheet position of each."""
156
+
157
+ name: str
158
+ sheet: str
159
+ headers: list[str]
160
+ rows: list[list[str]]
161
+ numbers: list[int]
162
+ columns: dict[int, str | None]
163
+ # A merge source's rows add no records: keys[i] are the IDs of the records row i fills in.
164
+ keys: list[list[str]] | None = None
165
+ lists: frozenset[str] = frozenset()
166
+ headings_removed: int = 0
167
+ categories: tuple[str, ...] = ()
168
+
169
+
170
+ def is_xlsx(path: str) -> bool:
171
+ return path.lower().endswith(".xlsx")
172
+
173
+
174
+ def split_ref(ref: str) -> tuple[int, int]:
175
+ match = re.fullmatch(r"\$?([A-Za-z]+)\$?(\d+)", ref)
176
+ if not match:
177
+ raise InputError(f"invalid cell reference {ref!r}")
178
+ col = 0
179
+ for ch in match.group(1).upper():
180
+ col = col * 26 + ord(ch) - 64
181
+ return int(match.group(2)), col
182
+
183
+
184
+ def column_name(col: int) -> str:
185
+ out = ""
186
+ while col:
187
+ col, rem = divmod(col - 1, 26)
188
+ out = chr(65 + rem) + out
189
+ return out
190
+
191
+
192
+ def _range(ref: str) -> tuple[int, int, int, int]:
193
+ start, _, end = ref.partition(":")
194
+ (r1, c1), (r2, c2) = split_ref(start), split_ref(end or start)
195
+ return r1, c1, r2, c2
196
+
197
+
198
+ def _text_of(node: ET.Element) -> str:
199
+ """Text of an <si> or <is>: plain <t> or rich-text <r> runs; phonetic <rPh> hints are not content."""
200
+ parts = []
201
+ for child in node:
202
+ if child.tag == f"{{{_MAIN}}}t":
203
+ parts.append(child.text or "")
204
+ elif child.tag == f"{{{_MAIN}}}r":
205
+ parts.extend(t.text or "" for t in child.findall("m:t", NS))
206
+ # Excel writes control characters as _xHHHH_, such as the CR in a line break typed on Windows.
207
+ return re.sub(
208
+ r"_x([0-9A-Fa-f]{4})_", lambda m: chr(int(m.group(1), 16)), "".join(parts)
209
+ )
210
+
211
+
212
+ def _excel_date(serial: float, date1904: bool) -> str:
213
+ if date1904:
214
+ base = dt.datetime(1904, 1, 1)
215
+ else:
216
+ # Excel treats 1900 as a leap year, so serials before its fake Feb 29 count from a day later.
217
+ base = dt.datetime(1899, 12, 31) if serial < 60 else dt.datetime(1899, 12, 30)
218
+ stamp = base + dt.timedelta(seconds=round(serial * 86400))
219
+ if serial < 1:
220
+ return stamp.time().isoformat(timespec="seconds")
221
+ if stamp.time() == dt.time(0):
222
+ return stamp.date().isoformat()
223
+ return stamp.isoformat(timespec="seconds")
224
+
225
+
226
+ def _number(raw: str) -> str:
227
+ number = float(raw)
228
+ if number.is_integer() and abs(number) < 1e16:
229
+ return str(int(number))
230
+ return repr(number)
231
+
232
+
233
+ def _is_date_format(format_id: int, code: str) -> bool:
234
+ if format_id in _BUILTIN_DATE_FORMATS:
235
+ return True
236
+ visible = re.sub(r'"[^"]*"|\[[^\]]*\]|\\.|_.|\*.', "", code)
237
+ return bool(re.search(r"[dmyhs]", visible, re.I))
238
+
239
+
240
+ @dataclass
241
+ class Workbook:
242
+ path: Path
243
+ sheets: list[Sheet]
244
+ tables: list[Table]
245
+ shared: list[str]
246
+ date_styles: frozenset[int]
247
+ date1904: bool
248
+ _zip: zipfile.ZipFile = field(repr=False)
249
+ _grids: dict[str, dict[tuple[int, int], str]] = field(
250
+ default_factory=dict, repr=False
251
+ )
252
+
253
+ def __enter__(self) -> "Workbook":
254
+ return self
255
+
256
+ def __exit__(self, *exc: object) -> None:
257
+ self._zip.close()
258
+
259
+ @classmethod
260
+ def open(cls, path: str | Path) -> "Workbook":
261
+ path = Path(path)
262
+ try:
263
+ archive = zipfile.ZipFile(path)
264
+ except (OSError, zipfile.BadZipFile) as exc:
265
+ raise InputError(f"cannot open {path} as an .xlsx workbook: {exc}") from exc
266
+ try:
267
+ return cls._load(path, archive)
268
+ except (KeyError, ET.ParseError, ValueError, zipfile.BadZipFile) as exc:
269
+ archive.close()
270
+ raise InputError(f"{path} is not a readable .xlsx workbook: {exc}") from exc
271
+ except Exception:
272
+ archive.close()
273
+ raise
274
+
275
+ @classmethod
276
+ def _load(cls, path: Path, archive: zipfile.ZipFile) -> "Workbook":
277
+ names = set(archive.namelist())
278
+ root = ET.fromstring(archive.read("xl/workbook.xml"))
279
+ relations = _relationships(archive, "xl/workbook.xml")
280
+ properties = root.find("m:workbookPr", NS)
281
+ sheets = [
282
+ Sheet(s.get("name", ""), relations[s.get(_R_ID)], s.get("state", "visible"))
283
+ for s in root.findall("m:sheets/m:sheet", NS)
284
+ ]
285
+ shared: list[str] = []
286
+ if "xl/sharedStrings.xml" in names:
287
+ shared = [
288
+ _text_of(si)
289
+ for si in ET.fromstring(archive.read("xl/sharedStrings.xml")).findall(
290
+ "m:si", NS
291
+ )
292
+ ]
293
+ tables = []
294
+ for sheet in sheets:
295
+ for target in _relationships(archive, sheet.part).values():
296
+ if "/tables/" not in target or target not in names:
297
+ continue
298
+ table = ET.fromstring(archive.read(target))
299
+ tables.append(
300
+ Table(
301
+ table.get("displayName") or table.get("name") or "",
302
+ sheet.name,
303
+ table.get("ref", ""),
304
+ int(table.get("headerRowCount", "1")),
305
+ int(table.get("totalsRowCount", "0")),
306
+ )
307
+ )
308
+ return cls(
309
+ path,
310
+ sheets,
311
+ tables,
312
+ shared,
313
+ _date_styles(archive, names),
314
+ properties is not None and properties.get("date1904") in ("1", "true"),
315
+ archive,
316
+ )
317
+
318
+ def sheet_named(self, name: str) -> Sheet:
319
+ found = next(
320
+ (s for s in self.sheets if s.name.casefold() == name.casefold()), None
321
+ )
322
+ if found is None:
323
+ raise InputError(
324
+ f"{self.path.name} has no sheet {name!r}; sheets: {_names(s.name for s in self.sheets)}"
325
+ )
326
+ return found
327
+
328
+ def table_named(self, name: str) -> Table:
329
+ found = next(
330
+ (t for t in self.tables if t.name.casefold() == name.casefold()), None
331
+ )
332
+ if found is None:
333
+ raise InputError(
334
+ f"{self.path.name} has no table {name!r}; tables: {_names(t.name for t in self.tables)}"
335
+ )
336
+ return found
337
+
338
+ def cells(self, sheet: Sheet) -> dict[tuple[int, int], str]:
339
+ """Non-empty cells of a sheet keyed by (row, column), both 1-based."""
340
+ if sheet.name not in self._grids:
341
+ try:
342
+ self._grids[sheet.name] = dict(self._parse_cells(sheet))
343
+ except (KeyError, ET.ParseError, ValueError, zipfile.BadZipFile) as exc:
344
+ raise InputError(
345
+ f"{self.path.name}: cannot read sheet {sheet.name!r}: {exc}"
346
+ ) from exc
347
+ return self._grids[sheet.name]
348
+
349
+ def read_block(
350
+ self, sheet: Sheet, r1: int, c1: int, r2: int, c2: int
351
+ ) -> list[list[str]]:
352
+ grid = self.cells(sheet)
353
+ return [
354
+ [grid.get((r, c), "") for c in range(c1, c2 + 1)] for r in range(r1, r2 + 1)
355
+ ]
356
+
357
+ def raw_comments(self, sheet: Sheet) -> list[RawComment]:
358
+ """Comments of a sheet: threaded ones with each reply on its own, else the legacy note on that cell."""
359
+ names = set(self._zip.namelist())
360
+ links = [
361
+ (kind, target)
362
+ for _, kind, target in _typed_relationships(self._zip, sheet.part)
363
+ if target in names
364
+ ]
365
+ threaded: list[RawComment] = []
366
+ for _, target in (l for l in links if l[0].endswith("/threadedComment")):
367
+ people = self._people()
368
+ for node in ET.fromstring(self._zip.read(target)).findall(
369
+ "{*}threadedComment"
370
+ ):
371
+ text = node.find("{*}text")
372
+ row, col = split_ref(node.get("ref", ""))
373
+ threaded.append(
374
+ RawComment(
375
+ row,
376
+ col,
377
+ people.get(node.get("personId", ""), ""),
378
+ node.get("dT", ""),
379
+ (text.text or "").strip() if text is not None else "",
380
+ )
381
+ )
382
+ threaded.sort(key=lambda c: (c.row, c.col, c.created))
383
+ threaded_cells = {(c.row, c.col) for c in threaded}
384
+ notes: list[RawComment] = []
385
+ for _, target in (l for l in links if l[0].endswith("/comments")):
386
+ root = ET.fromstring(self._zip.read(target))
387
+ authors = [
388
+ (a.text or "").strip() for a in root.findall("m:authors/m:author", NS)
389
+ ]
390
+ for node in root.findall("m:commentList/m:comment", NS):
391
+ row, col = split_ref(node.get("ref", ""))
392
+ text_node = node.find("m:text", NS)
393
+ text = _text_of(text_node) if text_node is not None else ""
394
+ if (row, col) in threaded_cells and text.startswith(
395
+ "[Threaded comment]"
396
+ ):
397
+ continue
398
+ index = int(node.get("authorId") or -1)
399
+ author = authors[index] if 0 <= index < len(authors) else ""
400
+ if author and text.startswith(author + ":\n"):
401
+ text = text[len(author) + 2 :]
402
+ notes.append(
403
+ RawComment(row, col, author, self._modified(), text.strip())
404
+ )
405
+ notes.sort(key=lambda c: (c.row, c.col))
406
+ return [*notes, *threaded]
407
+
408
+ def _people(self) -> dict[str, str]:
409
+ result: dict[str, str] = {}
410
+ for _, kind, target in _typed_relationships(self._zip, "xl/workbook.xml"):
411
+ if kind.endswith("/person") and target in self._zip.namelist():
412
+ for person in ET.fromstring(self._zip.read(target)).findall(
413
+ "{*}person"
414
+ ):
415
+ result[person.get("id", "")] = (
416
+ person.get("displayName") or ""
417
+ ).strip()
418
+ return result
419
+
420
+ def _modified(self) -> str:
421
+ if "docProps/core.xml" not in self._zip.namelist():
422
+ return ""
423
+ node = ET.fromstring(self._zip.read("docProps/core.xml")).find(_DC_MODIFIED)
424
+ return (node.text or "").strip() if node is not None else ""
425
+
426
+ def _parse_cells(self, sheet: Sheet) -> Iterator[tuple[tuple[int, int], str]]:
427
+ row_number = col_number = 0
428
+ row_tag, cell_tag = f"{{{_MAIN}}}row", f"{{{_MAIN}}}c"
429
+ with self._zip.open(sheet.part) as handle:
430
+ for event, element in ET.iterparse(handle, events=("start", "end")):
431
+ if element.tag == row_tag and event == "start":
432
+ row_number = int(element.get("r") or row_number + 1)
433
+ col_number = 0
434
+ elif element.tag == cell_tag and event == "end":
435
+ if element.get("r"):
436
+ row_number, col_number = split_ref(element.get("r", ""))
437
+ else:
438
+ col_number += 1
439
+ value = self._value(element)
440
+ if value != "":
441
+ yield (row_number, col_number), value
442
+ element.clear()
443
+
444
+ def _value(self, cell: ET.Element) -> str:
445
+ kind = cell.get("t", "n")
446
+ v = cell.find("m:v", NS)
447
+ raw = v.text if v is not None and v.text is not None else ""
448
+ if kind == "s":
449
+ return self.shared[int(raw)] if raw else ""
450
+ if kind == "inlineStr":
451
+ node = cell.find("m:is", NS)
452
+ return _text_of(node) if node is not None else ""
453
+ if kind in ("str", "e", "d"):
454
+ return raw
455
+ if kind == "b":
456
+ return "TRUE" if raw == "1" else "FALSE"
457
+ if raw == "":
458
+ return ""
459
+ if int(cell.get("s", "0")) in self.date_styles:
460
+ try:
461
+ return _excel_date(float(raw), self.date1904)
462
+ except (OverflowError, ValueError):
463
+ pass
464
+ return _number(raw)
465
+
466
+
467
+ def _names(names: Iterator[str]) -> str:
468
+ return ", ".join(names) or "(none)"
469
+
470
+
471
+ def _typed_relationships(
472
+ archive: zipfile.ZipFile, part: str
473
+ ) -> list[tuple[str, str, str]]:
474
+ """(id, type, target part) for each relationship of a part."""
475
+ folder, name = posixpath.split(part)
476
+ path = posixpath.join(folder, "_rels", name + ".rels")
477
+ if path not in archive.namelist():
478
+ return []
479
+ result = []
480
+ for rel in ET.fromstring(archive.read(path)).findall("rel:Relationship", NS):
481
+ target = rel.get("Target", "")
482
+ result.append(
483
+ (
484
+ rel.get("Id", ""),
485
+ rel.get("Type", ""),
486
+ target.lstrip("/")
487
+ if target.startswith("/")
488
+ else posixpath.normpath(posixpath.join(folder, target)),
489
+ )
490
+ )
491
+ return result
492
+
493
+
494
+ def _relationships(archive: zipfile.ZipFile, part: str) -> dict[str, str]:
495
+ return {rid: target for rid, _, target in _typed_relationships(archive, part)}
496
+
497
+
498
+ def _date_styles(archive: zipfile.ZipFile, names: set[str]) -> frozenset[int]:
499
+ if "xl/styles.xml" not in names:
500
+ return frozenset()
501
+ root = ET.fromstring(archive.read("xl/styles.xml"))
502
+ custom = {
503
+ int(f.get("numFmtId", "0")): f.get("formatCode", "")
504
+ for f in root.findall("m:numFmts/m:numFmt", NS)
505
+ }
506
+ result = set()
507
+ for index, xf in enumerate(root.findall("m:cellXfs/m:xf", NS)):
508
+ format_id = int(xf.get("numFmtId", "0"))
509
+ if _is_date_format(format_id, custom.get(format_id, "")):
510
+ result.add(index)
511
+ return frozenset(result)
512
+
513
+
514
+ def _trim_empty_tail(rows: list[list[str]]) -> list[list[str]]:
515
+ while rows and not any(rows[-1]):
516
+ rows.pop()
517
+ return rows
518
+
519
+
520
+ def _table_block(wb: Workbook, table: Table) -> Block:
521
+ if table.header_rows < 1:
522
+ raise InputError(f"{wb.path.name}: table {table.name!r} has no header row")
523
+ r1, c1, r2, c2 = _range(table.ref)
524
+ rows = wb.read_block(
525
+ wb.sheet_named(table.sheet), r1, c1, r2 - table.totals_rows, c2
526
+ )
527
+ return Block(table.name, _trim_empty_tail(rows), r1, c1, table.sheet)
528
+
529
+
530
+ def _sheet_block(wb: Workbook, sheet: Sheet, header_row: int | None) -> Block:
531
+ grid = wb.cells(sheet)
532
+ if header_row is None:
533
+ if not grid:
534
+ raise InputError(f"{wb.path.name}: sheet {sheet.name!r} is empty")
535
+ header_row = min(r for r, _ in grid)
536
+ header_cols = [c for r, c in grid if r == header_row]
537
+ if not header_cols:
538
+ raise InputError(
539
+ f"{wb.path.name}: sheet {sheet.name!r} has no header cells in row {header_row}"
540
+ )
541
+ c1, c2 = min(header_cols), max(header_cols)
542
+ last = max((r for r, c in grid if c1 <= c <= c2), default=header_row)
543
+ return Block(
544
+ sheet.name,
545
+ wb.read_block(sheet, header_row, c1, last, c2),
546
+ header_row,
547
+ c1,
548
+ sheet.name,
549
+ )
550
+
551
+
552
+ def select_block(wb: Workbook, selection: XlsxSelection) -> Block:
553
+ if selection.table:
554
+ return _table_block(wb, wb.table_named(selection.table))
555
+ if selection.sheet:
556
+ return _sheet_block(wb, wb.sheet_named(selection.sheet), selection.header_row)
557
+ if len(wb.tables) == 1:
558
+ return _table_block(wb, wb.tables[0])
559
+ raise InputError(
560
+ f"{wb.path.name} has {len(wb.tables)} tables; choose one with --table or a sheet with --sheet "
561
+ f"(tables: {_names(t.name for t in wb.tables)}; sheets: {_names(s.name for s in wb.sheets)})"
562
+ )
563
+
564
+
565
+ def read_selection(path: str | Path, selection: XlsxSelection) -> Block:
566
+ with Workbook.open(path) as wb:
567
+ return select_block(wb, selection)
568
+
569
+
570
+ def _source_name(source: dict[str, Any]) -> str:
571
+ return source.get("table") or source["sheet"]
572
+
573
+
574
+ def _slug(text: str) -> str:
575
+ return re.sub(r"[^A-Z0-9]+", "-", text.upper()).strip("-")
576
+
577
+
578
+ def _map_source(
579
+ block: Block, source: dict[str, Any], id_column: str | None, traceability: bool
580
+ ) -> Part:
581
+ """Apply one configured source to its block: headings, exclude, rename, set, lists, then Type and generated IDs."""
582
+ where = f"xlsx source {_source_name(source)!r}"
583
+ headers = list(block.rows[0])
584
+ if "" in headers:
585
+ raise InputError(f"{where}: empty column name in header")
586
+ duplicates = sorted({h for h in headers if headers.count(h) > 1})
587
+ if duplicates:
588
+ raise InputError(f"{where}: duplicate column name(s): {', '.join(duplicates)}")
589
+ columns, exclude = source.get("columns", {}), source.get("exclude", [])
590
+ headings = source.get("headings")
591
+ heading_columns = (
592
+ [headings[k] for k in ("column", "text") if k in headings] if headings else []
593
+ )
594
+ for name in (*columns, *exclude, *heading_columns):
595
+ if name not in headers:
596
+ raise ConfigError(
597
+ f"{where} has no column {name!r}; columns: {_names(iter(headers))}"
598
+ )
599
+ live = [
600
+ (block.first_row + 1 + i, row)
601
+ for i, row in enumerate(block.rows[1:])
602
+ if any(cell.strip() for cell in row)
603
+ ]
604
+ under: list[str] = [""] * len(live)
605
+ removed = 0
606
+ if headings:
607
+ type_at = headers.index(headings["column"])
608
+ text_at = headers.index(headings["text"]) if "text" in headings else None
609
+ values = {v.strip().casefold() for v in headings["values"]}
610
+ kept, under, current = [], [], ""
611
+ for number, row in live:
612
+ if row[type_at].strip().casefold() in values:
613
+ if text_at is not None:
614
+ current = row[text_at].strip()
615
+ else:
616
+ current = next(
617
+ (
618
+ c.strip()
619
+ for i, c in enumerate(row)
620
+ if i != type_at and c.strip()
621
+ ),
622
+ "",
623
+ )
624
+ else:
625
+ kept.append((number, row))
626
+ under.append(current)
627
+ removed = len(live) - len(kept)
628
+ live = kept
629
+ keep = [i for i, h in enumerate(headers) if h not in exclude]
630
+ out_headers = [columns.get(headers[i], headers[i]) for i in keep]
631
+ clashes = sorted({h for h in out_headers if out_headers.count(h) > 1})
632
+ if clashes:
633
+ raise ConfigError(
634
+ f"{where}: columns map to the same output header: {', '.join(clashes)}"
635
+ )
636
+ numbers = [number for number, _ in live]
637
+ rows = [[row[i] for i in keep] for _, row in live]
638
+
639
+ def column(name: str) -> int:
640
+ if name not in out_headers:
641
+ out_headers.append(name)
642
+ for row in rows:
643
+ row.append("")
644
+ return out_headers.index(name)
645
+
646
+ def need(name: str, key: str) -> int:
647
+ if name not in out_headers:
648
+ raise ConfigError(
649
+ f"{where}: {key} names {name!r}, which is not a mapped column; columns: {_names(iter(out_headers))}"
650
+ )
651
+ return out_headers.index(name)
652
+
653
+ constants = dict(source.get("set", {}))
654
+ if "type" in source:
655
+ constants["Type"] = source["type"]
656
+ if "Type" in constants and not traceability:
657
+ raise ConfigError(
658
+ f"{where} sets 'type', which needs traceability: use --traceability or set \"traceability\": true"
659
+ )
660
+ for name, constant in constants.items():
661
+ at = column(name)
662
+ for row in rows:
663
+ row[at] = constant
664
+ categories: tuple[str, ...] = ()
665
+ if headings:
666
+ at = column(headings["into"])
667
+ for row, heading in zip(rows, under):
668
+ if heading and not row[at].strip():
669
+ row[at] = heading
670
+ categories = tuple(row[at].strip() for row in rows)
671
+ for name, separator in source.get("lists", {}).items():
672
+ at = need(name, "lists")
673
+ pattern = r"\r?\n" if separator == "newline" else re.escape(separator)
674
+ for row in rows:
675
+ row[at] = "; ".join(
676
+ p for p in (part.strip() for part in re.split(pattern, row[at])) if p
677
+ )
678
+ if "id_prefix" in source:
679
+ if not id_column:
680
+ raise ConfigError(
681
+ f"{where} sets 'id_prefix', which needs an ID column: use --id-column or set id_column"
682
+ )
683
+ at = column(id_column)
684
+ id_from = source.get("id_from") or (
685
+ ["Field", "Name"] if constants.get("Type") == "Value" else None
686
+ )
687
+ parts_at = [need(name, "id_from") for name in id_from] if id_from else []
688
+ names = out_headers.index("Name") if "Name" in out_headers else None
689
+ aliases = out_headers.index("Aliases") if "Aliases" in out_headers else None
690
+ for row in rows:
691
+ if row[at].strip():
692
+ continue
693
+ if id_from:
694
+ slug = "-".join(filter(None, (_slug(row[i]) for i in parts_at)))
695
+ else:
696
+ alias = (
697
+ next((a.strip() for a in row[aliases].split(";") if a.strip()), "")
698
+ if aliases is not None
699
+ else ""
700
+ )
701
+ slug = _slug(alias) or (_slug(row[names]) if names is not None else "")
702
+ if slug:
703
+ row[at] = source["id_prefix"] + slug
704
+ elif source.get("merge"):
705
+ if not id_column:
706
+ raise ConfigError(
707
+ f"{where} is a merge source, which needs an ID column: use --id-column or set id_column"
708
+ )
709
+ if id_column not in out_headers:
710
+ raise ConfigError(
711
+ f"{where} is a merge source with no {id_column!r} column to match records by; "
712
+ f"columns: {_names(iter(out_headers))}"
713
+ )
714
+ elif id_column and id_column not in out_headers:
715
+ raise ConfigError(
716
+ f"{where} has no {id_column!r} column: map one of its columns to {id_column!r}, "
717
+ f"set id_prefix to generate IDs, or remove the source"
718
+ )
719
+ sheet_columns = {
720
+ block.first_col + i: None if h in exclude else columns.get(h, h)
721
+ for i, h in enumerate(headers)
722
+ }
723
+ lists = frozenset(source.get("lists", {}))
724
+ keys = None
725
+ if source.get("merge"):
726
+ at = out_headers.index(id_column)
727
+ keys = [
728
+ _split(row[at])
729
+ if id_column in lists
730
+ else [row[at].strip()]
731
+ if row[at].strip()
732
+ else []
733
+ for row in rows
734
+ ]
735
+ return Part(
736
+ block.name,
737
+ block.sheet,
738
+ out_headers,
739
+ rows,
740
+ numbers,
741
+ sheet_columns,
742
+ keys,
743
+ lists,
744
+ removed,
745
+ categories,
746
+ )
747
+
748
+
749
+ def _plain_part(block: Block) -> Part:
750
+ headers = list(block.rows[0])
751
+ body = [
752
+ (block.first_row + 1 + i, row)
753
+ for i, row in enumerate(block.rows[1:])
754
+ if any(c.strip() for c in row)
755
+ ]
756
+ return Part(
757
+ block.name,
758
+ block.sheet,
759
+ headers,
760
+ [row for _, row in body],
761
+ [n for n, _ in body],
762
+ {block.first_col + i: h for i, h in enumerate(headers)},
763
+ )
764
+
765
+
766
+ def _split(cell: str) -> list[str]:
767
+ return [p for p in (part.strip() for part in cell.split(";")) if p]
768
+
769
+
770
+ def _listed(items: list[str], limit: int = 12) -> str:
771
+ return ", ".join(items[:limit]) + (
772
+ f" and {len(items) - limit} more" if len(items) > limit else ""
773
+ )
774
+
775
+
776
+ def _repair_ids(parts: list[Part], id_column: str | None, file: str) -> list[str]:
777
+ """Name rows that have no ID ROW-<worksheet row> and suffix repeated IDs with ~2, ~3, ...; one warning per case."""
778
+ warnings: list[str] = []
779
+ if not id_column:
780
+ return warnings
781
+ usable = [
782
+ (part, part.headers.index(id_column))
783
+ for part in parts
784
+ if id_column in part.headers
785
+ ]
786
+ for part, at in usable:
787
+ named = []
788
+ for number, row in zip(part.numbers, part.rows):
789
+ if not row[at].strip():
790
+ row[at] = f"ROW-{number}"
791
+ named.append(row[at])
792
+ if named:
793
+ noun = "row has" if len(named) == 1 else "rows have"
794
+ warnings.append(
795
+ f"{file} [{part.name}]: {len(named)} {noun} no ID in {id_column!r}; "
796
+ f"named {'it' if len(named) == 1 else 'them'} {_listed(named)}"
797
+ )
798
+ seen: dict[str, list[tuple[int, list[str], int]]] = {}
799
+ for part, at in usable:
800
+ for number, row in zip(part.numbers, part.rows):
801
+ seen.setdefault(row[at].strip(), []).append((number, row, at))
802
+ taken = set(seen)
803
+ for rid, found in seen.items():
804
+ if len(found) < 2:
805
+ continue
806
+ renamed = []
807
+ for _, row, at in found[1:]:
808
+ n = 2
809
+ while f"{rid}~{n}" in taken:
810
+ n += 1
811
+ taken.add(f"{rid}~{n}")
812
+ row[at] = f"{rid}~{n}"
813
+ renamed.append(row[at])
814
+ warnings.append(
815
+ f"{file} [{' + '.join(p.name for p in parts)}]: duplicate ID {rid!r} at rows {', '.join(str(n) for n, _, _ in found)}; "
816
+ f"renamed the later {'one' if len(renamed) == 1 else 'ones'} {', '.join(renamed)}"
817
+ )
818
+ return warnings
819
+
820
+
821
+ def _combine(parts: list[Part], id_column: str | None) -> Block:
822
+ """One block from several parts: shared headers line up, missing cells are empty."""
823
+ headers: list[str] = (
824
+ [id_column] if id_column and any(id_column in p.headers for p in parts) else []
825
+ )
826
+ for part in parts:
827
+ headers += [h for h in part.headers if h not in headers]
828
+ rows = [headers]
829
+ for part in parts:
830
+ rows += [
831
+ [row[part.headers.index(h)] if h in part.headers else "" for h in headers]
832
+ for row in part.rows
833
+ ]
834
+ return Block(" + ".join(p.name for p in parts), rows)
835
+
836
+
837
+ def _merge(
838
+ block: Block, part: Part, id_column: str, list_columns: frozenset[str], file: str
839
+ ) -> list[str]:
840
+ """Fill the combined table's records from one merge source; one warning for unknown IDs, one for conflicts."""
841
+ headers = block.rows[0]
842
+ for name in part.headers:
843
+ if name not in headers:
844
+ headers.append(name)
845
+ for row in block.rows[1:]:
846
+ row.append("")
847
+ by_id = {row[headers.index(id_column)].strip(): row for row in block.rows[1:]}
848
+ fills = [
849
+ (
850
+ part.headers.index(h),
851
+ headers.index(h),
852
+ h in list_columns or h in part.lists,
853
+ h,
854
+ )
855
+ for h in part.headers
856
+ if h != id_column
857
+ ]
858
+ unknown: dict[str, None] = {}
859
+ conflicts: dict[str, None] = {}
860
+ for row, keys in zip(part.rows, part.keys or []):
861
+ for key in keys:
862
+ target = by_id.get(key)
863
+ if target is None:
864
+ unknown[key] = None
865
+ continue
866
+ for source_at, target_at, is_list, name in fills:
867
+ value = row[source_at]
868
+ if is_list:
869
+ # The record's own cell may not have been split by a lists entry, so read it either way.
870
+ have = {
871
+ part.strip() for part in re.split(r"[;,]", target[target_at])
872
+ }
873
+ new = [v for v in dict.fromkeys(_split(value)) if v not in have]
874
+ if new:
875
+ target[target_at] = "; ".join(
876
+ filter(None, [target[target_at].strip(), *new])
877
+ )
878
+ elif value.strip() and not target[target_at].strip():
879
+ target[target_at] = value
880
+ elif value.strip() and target[target_at].strip() != value.strip():
881
+ conflicts[
882
+ f"{key} {name}: kept {target[target_at].strip()!r}, ignored {value.strip()!r}"
883
+ ] = None
884
+ warnings = []
885
+ where = f"{file} [{part.name}]"
886
+ if unknown:
887
+ noun = "ID matches" if len(unknown) == 1 else "IDs match"
888
+ warnings.append(
889
+ f"{where}: {len(unknown)} {noun} no record in {id_column!r}; ignored {_listed(list(unknown))}"
890
+ )
891
+ if conflicts:
892
+ noun = "value differs" if len(conflicts) == 1 else "values differ"
893
+ warnings.append(
894
+ f"{where}: {len(conflicts)} {noun} from the record; {_listed(list(conflicts))}"
895
+ )
896
+ return warnings
897
+
898
+
899
+ def _initials(author: str) -> str:
900
+ words = re.sub(r"[\x00-\x1f\x7f]", " ", author).split()
901
+ if not words:
902
+ return "XL"
903
+ value = (words[0][:2] if len(words) == 1 else "".join(w[0] for w in words)).upper()[
904
+ :16
905
+ ]
906
+ return value if valid_initials(value) else "XL"
907
+
908
+
909
+ def _place(
910
+ part: Part, id_column: str | None, comment: RawComment, ids: set[str]
911
+ ) -> tuple[str, str, str] | None:
912
+ """The (requirement ID, output column, text) a comment lands on in this part, or None if it falls outside it.
913
+
914
+ A comment on a merge source's row lands on the first record that row fills in.
915
+ """
916
+ column = part.columns.get(comment.col)
917
+ if (
918
+ not id_column
919
+ or id_column not in part.headers
920
+ or column is None
921
+ or comment.row not in part.numbers
922
+ ):
923
+ return None
924
+ at = part.numbers.index(comment.row)
925
+ if part.keys is None:
926
+ rid = part.rows[at][part.headers.index(id_column)].strip()
927
+ else:
928
+ rid = next((key for key in part.keys[at] if key in ids), None)
929
+ if rid is None:
930
+ return None
931
+ if column != id_column:
932
+ return rid, column, comment.text
933
+ others = [h for h in part.headers if h != id_column]
934
+ return (rid, others[0], f"[on ID] {comment.text}") if others else None
935
+
936
+
937
+ def _review_comments(
938
+ wb: Workbook,
939
+ parts: list[Part],
940
+ id_column: str | None,
941
+ ids: set[str],
942
+ label: str,
943
+ ) -> tuple[list[dict[str, Any]], list[str]]:
944
+ comments: list[dict[str, Any]] = []
945
+ dropped = 0
946
+ for sheet_name in dict.fromkeys(part.sheet for part in parts):
947
+ try:
948
+ found = wb.raw_comments(wb.sheet_named(sheet_name))
949
+ except (KeyError, ET.ParseError, ValueError, zipfile.BadZipFile) as exc:
950
+ raise InputError(
951
+ f"{wb.path.name}: cannot read comments of sheet {sheet_name!r}: {exc}"
952
+ ) from exc
953
+ for raw in found:
954
+ placed = next(
955
+ (
956
+ hit
957
+ for part in parts
958
+ if part.sheet == sheet_name
959
+ and (hit := _place(part, id_column, raw, ids))
960
+ ),
961
+ None,
962
+ )
963
+ if placed is None:
964
+ dropped += 1
965
+ continue
966
+ rid, column, text = placed
967
+ comments.append(
968
+ {
969
+ "id": f"xlsx-{len(comments) + 1}",
970
+ "requirementId": rid,
971
+ "column": column,
972
+ "initials": _initials(raw.author),
973
+ "created": raw.created,
974
+ "modified": None,
975
+ "text": text,
976
+ }
977
+ )
978
+ warnings = []
979
+ if dropped:
980
+ noun = "comment" if dropped == 1 else "comments"
981
+ warnings.append(
982
+ f"{label}: dropped {dropped} Excel {noun} on heading rows, excluded columns or cells outside the imported block"
983
+ )
984
+ return comments, warnings
985
+
986
+
987
+ def import_workbook(
988
+ path: str | Path,
989
+ selection: XlsxSelection,
990
+ sources: list[dict[str, Any]],
991
+ id_column: str | None,
992
+ traceability: bool,
993
+ comments: bool,
994
+ ) -> Imported:
995
+ """Read one workbook the lenient way: configured sources or the selected block, IDs repaired, comments mapped."""
996
+ with Workbook.open(path) as wb:
997
+ if sources:
998
+ parts = [
999
+ _map_source(
1000
+ select_block(
1001
+ wb,
1002
+ XlsxSelection(
1003
+ s.get("table"), s.get("sheet"), s.get("header_row")
1004
+ ),
1005
+ ),
1006
+ s,
1007
+ id_column,
1008
+ traceability,
1009
+ )
1010
+ for s in sources
1011
+ ]
1012
+ else:
1013
+ parts = [_plain_part(select_block(wb, selection))]
1014
+ file = Path(path).name
1015
+ records = [part for part in parts if part.keys is None]
1016
+ warnings = _repair_ids(records, id_column, file)
1017
+ block = _combine(records, id_column)
1018
+ if len(records) < len(parts):
1019
+ # findings imports baselines, which imports this module.
1020
+ from .findings import TRACE_FIELDS
1021
+
1022
+ list_columns = frozenset(
1023
+ [
1024
+ *(f for f, (_, _, single) in TRACE_FIELDS.items() if not single),
1025
+ "Aliases",
1026
+ ]
1027
+ )
1028
+ for part in parts:
1029
+ if part.keys is not None:
1030
+ warnings += _merge(block, part, id_column, list_columns, file)
1031
+ block = Block(" + ".join(p.name for p in parts), block.rows)
1032
+ label = f"{file} [{block.name}]"
1033
+ found: list[dict[str, Any]] = []
1034
+ if comments:
1035
+ at = block.rows[0].index(id_column) if id_column in block.rows[0] else None
1036
+ ids = (
1037
+ {row[at].strip() for row in block.rows[1:]} if at is not None else set()
1038
+ )
1039
+ found, extra = _review_comments(wb, parts, id_column, ids, label)
1040
+ warnings += extra
1041
+ stats = tuple(
1042
+ SourceStats(
1043
+ _source_name(s),
1044
+ None if part.keys is not None else len(part.rows),
1045
+ part.headings_removed,
1046
+ part.categories,
1047
+ )
1048
+ for s, part in zip(sources, parts)
1049
+ )
1050
+ return Imported(block, tuple(warnings), tuple(found), stats)
1051
+
1052
+
1053
+ def mapping_template(path: str | Path) -> dict[str, Any]:
1054
+ """A complete config for a workbook: every table, and every sheet without one, mapped to itself."""
1055
+ sources: list[dict[str, Any]] = []
1056
+ with Workbook.open(path) as wb:
1057
+ for table in wb.tables:
1058
+ if table.header_rows >= 1:
1059
+ sources.append({"table": table.name, "block": _table_block(wb, table)})
1060
+ for sheet in wb.sheets:
1061
+ if any(t.sheet == sheet.name for t in wb.tables) or not wb.cells(sheet):
1062
+ continue
1063
+ first = min(r for r, _ in wb.cells(sheet))
1064
+ sources.append(
1065
+ {
1066
+ "sheet": sheet.name,
1067
+ "header_row": first,
1068
+ "block": _sheet_block(wb, sheet, first),
1069
+ }
1070
+ )
1071
+ config = json.loads(json.dumps(DEFAULT_CONFIG))
1072
+ id_headers = []
1073
+ for source in sources:
1074
+ headers = source.pop("block").rows[0]
1075
+ source["columns"] = {h: h for h in headers}
1076
+ id_headers += [h for h in headers if h.casefold() == "id"]
1077
+ config["id_column"] = id_headers[0] if id_headers else None
1078
+ config["xlsx"]["sources"] = sources
1079
+ return config
1080
+
1081
+
1082
+ def block_to_csv(rows: list[list[str]], delimiter: str) -> str:
1083
+ """CRLF records, like the browser's Save CSV, without its byte-order mark."""
1084
+ buffer = io.StringIO(newline="")
1085
+ csv.writer(buffer, delimiter=delimiter, lineterminator="\r\n").writerows(rows)
1086
+ return buffer.getvalue()
1087
+
1088
+
1089
+ def describe(path: str | Path) -> str:
1090
+ with Workbook.open(path) as wb:
1091
+ lines = ["Sheets:"]
1092
+ lines += [
1093
+ f" {s.name}" + ("" if s.state == "visible" else f" ({s.state})")
1094
+ for s in wb.sheets
1095
+ ]
1096
+ lines.append("Tables:")
1097
+ if not wb.tables:
1098
+ lines.append(" (none)")
1099
+ for table in wb.tables:
1100
+ headers = ""
1101
+ if table.header_rows >= 1:
1102
+ r1, c1, _, c2 = _range(table.ref)
1103
+ row = wb.read_block(wb.sheet_named(table.sheet), r1, c1, r1, c2)[0]
1104
+ headers = " headers: " + ", ".join(row)
1105
+ lines.append(
1106
+ f" {table.name} sheet: {table.sheet} ref: {table.ref}{headers}"
1107
+ )
1108
+ return "\n".join(lines)