reqtable 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- reqtable/__init__.py +3 -0
- reqtable/__main__.py +3 -0
- reqtable/baselines.py +920 -0
- reqtable/cli.py +631 -0
- reqtable/comparison.py +63 -0
- reqtable/core.py +1448 -0
- reqtable/drafts.py +166 -0
- reqtable/findings.py +605 -0
- reqtable/gate_checks.py +84 -0
- reqtable/gates.py +348 -0
- reqtable/glossary.py +162 -0
- reqtable/gui.py +952 -0
- reqtable/gui_lookups.py +192 -0
- reqtable/gui_mapping.py +789 -0
- reqtable/gui_page.py +691 -0
- reqtable/gui_page_gates.py +173 -0
- reqtable/gui_page_mapping.py +390 -0
- reqtable/gui_server.py +467 -0
- reqtable/html_report.py +2595 -0
- reqtable/markdown.py +127 -0
- reqtable/package.py +163 -0
- reqtable/quality.py +422 -0
- reqtable/review.py +318 -0
- reqtable/signoff.py +179 -0
- reqtable/traceability.py +1670 -0
- reqtable/values.py +192 -0
- reqtable/waivers.py +109 -0
- reqtable/xlsx.py +1108 -0
- reqtable-0.1.0.dist-info/METADATA +106 -0
- reqtable-0.1.0.dist-info/RECORD +33 -0
- reqtable-0.1.0.dist-info/WHEEL +4 -0
- reqtable-0.1.0.dist-info/entry_points.txt +2 -0
- reqtable-0.1.0.dist-info/licenses/LICENSE +21 -0
reqtable/xlsx.py
ADDED
|
@@ -0,0 +1,1108 @@
|
|
|
1
|
+
"""Read requirements from an .xlsx workbook with the standard library only.
|
|
2
|
+
|
|
3
|
+
Reading only: the workbook is never written. Cell values come out as the text
|
|
4
|
+
Excel displays for the stored value, so a block of cells can be fed to the CSV
|
|
5
|
+
pipeline unchanged. Merged cells keep their value in the top-left cell only.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import csv
|
|
11
|
+
import datetime as dt
|
|
12
|
+
import io
|
|
13
|
+
import json
|
|
14
|
+
import posixpath
|
|
15
|
+
import re
|
|
16
|
+
import zipfile
|
|
17
|
+
from dataclasses import dataclass, field
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Any, Iterator
|
|
20
|
+
from xml.etree import ElementTree as ET
|
|
21
|
+
|
|
22
|
+
from .core import DEFAULT_CONFIG, ConfigError, InputError, valid_initials
|
|
23
|
+
|
|
24
|
+
_MAIN = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"
|
|
25
|
+
_REL = "http://schemas.openxmlformats.org/package/2006/relationships"
|
|
26
|
+
_R_ID = "{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id"
|
|
27
|
+
_DC_MODIFIED = "{http://purl.org/dc/terms/}modified"
|
|
28
|
+
NS = {"m": _MAIN, "rel": _REL}
|
|
29
|
+
|
|
30
|
+
# Built-in number formats that Excel displays as dates or times (including the CJK locale ones).
|
|
31
|
+
_BUILTIN_DATE_FORMATS = frozenset(
|
|
32
|
+
{
|
|
33
|
+
14,
|
|
34
|
+
15,
|
|
35
|
+
16,
|
|
36
|
+
17,
|
|
37
|
+
18,
|
|
38
|
+
19,
|
|
39
|
+
20,
|
|
40
|
+
21,
|
|
41
|
+
22,
|
|
42
|
+
27,
|
|
43
|
+
28,
|
|
44
|
+
29,
|
|
45
|
+
30,
|
|
46
|
+
31,
|
|
47
|
+
32,
|
|
48
|
+
33,
|
|
49
|
+
34,
|
|
50
|
+
35,
|
|
51
|
+
36,
|
|
52
|
+
45,
|
|
53
|
+
46,
|
|
54
|
+
47,
|
|
55
|
+
50,
|
|
56
|
+
51,
|
|
57
|
+
52,
|
|
58
|
+
53,
|
|
59
|
+
54,
|
|
60
|
+
55,
|
|
61
|
+
56,
|
|
62
|
+
57,
|
|
63
|
+
58,
|
|
64
|
+
}
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@dataclass(frozen=True)
|
|
69
|
+
class Sheet:
|
|
70
|
+
name: str
|
|
71
|
+
part: str
|
|
72
|
+
state: str
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@dataclass(frozen=True)
|
|
76
|
+
class Table:
|
|
77
|
+
name: str
|
|
78
|
+
sheet: str
|
|
79
|
+
ref: str
|
|
80
|
+
header_rows: int = 1
|
|
81
|
+
totals_rows: int = 0
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@dataclass(frozen=True)
|
|
85
|
+
class XlsxSelection:
|
|
86
|
+
"""Which block of a workbook to read, shared by every .xlsx input of one run."""
|
|
87
|
+
|
|
88
|
+
table: str | None = None
|
|
89
|
+
sheet: str | None = None
|
|
90
|
+
header_row: int | None = None
|
|
91
|
+
|
|
92
|
+
def __post_init__(self) -> None:
|
|
93
|
+
if self.table and self.sheet:
|
|
94
|
+
raise ConfigError(
|
|
95
|
+
"--table and --sheet cannot be combined; a table already belongs to one sheet"
|
|
96
|
+
)
|
|
97
|
+
if self.header_row is not None:
|
|
98
|
+
if not self.sheet:
|
|
99
|
+
raise ConfigError("--header-row needs --sheet")
|
|
100
|
+
if self.header_row < 1:
|
|
101
|
+
raise ConfigError("--header-row must be 1 or greater")
|
|
102
|
+
|
|
103
|
+
@property
|
|
104
|
+
def given(self) -> bool:
|
|
105
|
+
return bool(self.table or self.sheet or self.header_row is not None)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
@dataclass(frozen=True)
|
|
109
|
+
class Block:
|
|
110
|
+
"""The selected cells: `name` is the table or sheet they came from, row 0 is the header.
|
|
111
|
+
|
|
112
|
+
`rows[i]` is worksheet row `first_row + i` of `sheet`, and its first column is worksheet column `first_col`.
|
|
113
|
+
"""
|
|
114
|
+
|
|
115
|
+
name: str
|
|
116
|
+
rows: list[list[str]]
|
|
117
|
+
first_row: int = 1
|
|
118
|
+
first_col: int = 1
|
|
119
|
+
sheet: str = ""
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@dataclass(frozen=True)
|
|
123
|
+
class SourceStats:
|
|
124
|
+
"""What one configured source gave: the rows it kept (None for a merge source, which adds no rows), the heading
|
|
125
|
+
rows it removed, and, when it has heading rows, each kept row's category in order."""
|
|
126
|
+
|
|
127
|
+
name: str
|
|
128
|
+
rows: int | None
|
|
129
|
+
headings_removed: int = 0
|
|
130
|
+
categories: tuple[str, ...] = ()
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
@dataclass(frozen=True)
|
|
134
|
+
class Imported:
|
|
135
|
+
"""What an .xlsx input yields: the combined table as a block, repair warnings, Excel comments as review comments,
|
|
136
|
+
and what each configured source contributed."""
|
|
137
|
+
|
|
138
|
+
block: Block
|
|
139
|
+
warnings: tuple[str, ...] = ()
|
|
140
|
+
comments: tuple[dict[str, Any], ...] = ()
|
|
141
|
+
sources: tuple[SourceStats, ...] = ()
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
@dataclass(frozen=True)
|
|
145
|
+
class RawComment:
|
|
146
|
+
row: int
|
|
147
|
+
col: int
|
|
148
|
+
author: str
|
|
149
|
+
created: str
|
|
150
|
+
text: str
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
@dataclass
|
|
154
|
+
class Part:
|
|
155
|
+
"""One source's rows on their way to the combined table, with the worksheet position of each."""
|
|
156
|
+
|
|
157
|
+
name: str
|
|
158
|
+
sheet: str
|
|
159
|
+
headers: list[str]
|
|
160
|
+
rows: list[list[str]]
|
|
161
|
+
numbers: list[int]
|
|
162
|
+
columns: dict[int, str | None]
|
|
163
|
+
# A merge source's rows add no records: keys[i] are the IDs of the records row i fills in.
|
|
164
|
+
keys: list[list[str]] | None = None
|
|
165
|
+
lists: frozenset[str] = frozenset()
|
|
166
|
+
headings_removed: int = 0
|
|
167
|
+
categories: tuple[str, ...] = ()
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def is_xlsx(path: str) -> bool:
|
|
171
|
+
return path.lower().endswith(".xlsx")
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def split_ref(ref: str) -> tuple[int, int]:
|
|
175
|
+
match = re.fullmatch(r"\$?([A-Za-z]+)\$?(\d+)", ref)
|
|
176
|
+
if not match:
|
|
177
|
+
raise InputError(f"invalid cell reference {ref!r}")
|
|
178
|
+
col = 0
|
|
179
|
+
for ch in match.group(1).upper():
|
|
180
|
+
col = col * 26 + ord(ch) - 64
|
|
181
|
+
return int(match.group(2)), col
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def column_name(col: int) -> str:
|
|
185
|
+
out = ""
|
|
186
|
+
while col:
|
|
187
|
+
col, rem = divmod(col - 1, 26)
|
|
188
|
+
out = chr(65 + rem) + out
|
|
189
|
+
return out
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _range(ref: str) -> tuple[int, int, int, int]:
|
|
193
|
+
start, _, end = ref.partition(":")
|
|
194
|
+
(r1, c1), (r2, c2) = split_ref(start), split_ref(end or start)
|
|
195
|
+
return r1, c1, r2, c2
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _text_of(node: ET.Element) -> str:
|
|
199
|
+
"""Text of an <si> or <is>: plain <t> or rich-text <r> runs; phonetic <rPh> hints are not content."""
|
|
200
|
+
parts = []
|
|
201
|
+
for child in node:
|
|
202
|
+
if child.tag == f"{{{_MAIN}}}t":
|
|
203
|
+
parts.append(child.text or "")
|
|
204
|
+
elif child.tag == f"{{{_MAIN}}}r":
|
|
205
|
+
parts.extend(t.text or "" for t in child.findall("m:t", NS))
|
|
206
|
+
# Excel writes control characters as _xHHHH_, such as the CR in a line break typed on Windows.
|
|
207
|
+
return re.sub(
|
|
208
|
+
r"_x([0-9A-Fa-f]{4})_", lambda m: chr(int(m.group(1), 16)), "".join(parts)
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def _excel_date(serial: float, date1904: bool) -> str:
|
|
213
|
+
if date1904:
|
|
214
|
+
base = dt.datetime(1904, 1, 1)
|
|
215
|
+
else:
|
|
216
|
+
# Excel treats 1900 as a leap year, so serials before its fake Feb 29 count from a day later.
|
|
217
|
+
base = dt.datetime(1899, 12, 31) if serial < 60 else dt.datetime(1899, 12, 30)
|
|
218
|
+
stamp = base + dt.timedelta(seconds=round(serial * 86400))
|
|
219
|
+
if serial < 1:
|
|
220
|
+
return stamp.time().isoformat(timespec="seconds")
|
|
221
|
+
if stamp.time() == dt.time(0):
|
|
222
|
+
return stamp.date().isoformat()
|
|
223
|
+
return stamp.isoformat(timespec="seconds")
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _number(raw: str) -> str:
|
|
227
|
+
number = float(raw)
|
|
228
|
+
if number.is_integer() and abs(number) < 1e16:
|
|
229
|
+
return str(int(number))
|
|
230
|
+
return repr(number)
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _is_date_format(format_id: int, code: str) -> bool:
|
|
234
|
+
if format_id in _BUILTIN_DATE_FORMATS:
|
|
235
|
+
return True
|
|
236
|
+
visible = re.sub(r'"[^"]*"|\[[^\]]*\]|\\.|_.|\*.', "", code)
|
|
237
|
+
return bool(re.search(r"[dmyhs]", visible, re.I))
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
@dataclass
|
|
241
|
+
class Workbook:
|
|
242
|
+
path: Path
|
|
243
|
+
sheets: list[Sheet]
|
|
244
|
+
tables: list[Table]
|
|
245
|
+
shared: list[str]
|
|
246
|
+
date_styles: frozenset[int]
|
|
247
|
+
date1904: bool
|
|
248
|
+
_zip: zipfile.ZipFile = field(repr=False)
|
|
249
|
+
_grids: dict[str, dict[tuple[int, int], str]] = field(
|
|
250
|
+
default_factory=dict, repr=False
|
|
251
|
+
)
|
|
252
|
+
|
|
253
|
+
def __enter__(self) -> "Workbook":
|
|
254
|
+
return self
|
|
255
|
+
|
|
256
|
+
def __exit__(self, *exc: object) -> None:
|
|
257
|
+
self._zip.close()
|
|
258
|
+
|
|
259
|
+
@classmethod
|
|
260
|
+
def open(cls, path: str | Path) -> "Workbook":
|
|
261
|
+
path = Path(path)
|
|
262
|
+
try:
|
|
263
|
+
archive = zipfile.ZipFile(path)
|
|
264
|
+
except (OSError, zipfile.BadZipFile) as exc:
|
|
265
|
+
raise InputError(f"cannot open {path} as an .xlsx workbook: {exc}") from exc
|
|
266
|
+
try:
|
|
267
|
+
return cls._load(path, archive)
|
|
268
|
+
except (KeyError, ET.ParseError, ValueError, zipfile.BadZipFile) as exc:
|
|
269
|
+
archive.close()
|
|
270
|
+
raise InputError(f"{path} is not a readable .xlsx workbook: {exc}") from exc
|
|
271
|
+
except Exception:
|
|
272
|
+
archive.close()
|
|
273
|
+
raise
|
|
274
|
+
|
|
275
|
+
@classmethod
|
|
276
|
+
def _load(cls, path: Path, archive: zipfile.ZipFile) -> "Workbook":
|
|
277
|
+
names = set(archive.namelist())
|
|
278
|
+
root = ET.fromstring(archive.read("xl/workbook.xml"))
|
|
279
|
+
relations = _relationships(archive, "xl/workbook.xml")
|
|
280
|
+
properties = root.find("m:workbookPr", NS)
|
|
281
|
+
sheets = [
|
|
282
|
+
Sheet(s.get("name", ""), relations[s.get(_R_ID)], s.get("state", "visible"))
|
|
283
|
+
for s in root.findall("m:sheets/m:sheet", NS)
|
|
284
|
+
]
|
|
285
|
+
shared: list[str] = []
|
|
286
|
+
if "xl/sharedStrings.xml" in names:
|
|
287
|
+
shared = [
|
|
288
|
+
_text_of(si)
|
|
289
|
+
for si in ET.fromstring(archive.read("xl/sharedStrings.xml")).findall(
|
|
290
|
+
"m:si", NS
|
|
291
|
+
)
|
|
292
|
+
]
|
|
293
|
+
tables = []
|
|
294
|
+
for sheet in sheets:
|
|
295
|
+
for target in _relationships(archive, sheet.part).values():
|
|
296
|
+
if "/tables/" not in target or target not in names:
|
|
297
|
+
continue
|
|
298
|
+
table = ET.fromstring(archive.read(target))
|
|
299
|
+
tables.append(
|
|
300
|
+
Table(
|
|
301
|
+
table.get("displayName") or table.get("name") or "",
|
|
302
|
+
sheet.name,
|
|
303
|
+
table.get("ref", ""),
|
|
304
|
+
int(table.get("headerRowCount", "1")),
|
|
305
|
+
int(table.get("totalsRowCount", "0")),
|
|
306
|
+
)
|
|
307
|
+
)
|
|
308
|
+
return cls(
|
|
309
|
+
path,
|
|
310
|
+
sheets,
|
|
311
|
+
tables,
|
|
312
|
+
shared,
|
|
313
|
+
_date_styles(archive, names),
|
|
314
|
+
properties is not None and properties.get("date1904") in ("1", "true"),
|
|
315
|
+
archive,
|
|
316
|
+
)
|
|
317
|
+
|
|
318
|
+
def sheet_named(self, name: str) -> Sheet:
|
|
319
|
+
found = next(
|
|
320
|
+
(s for s in self.sheets if s.name.casefold() == name.casefold()), None
|
|
321
|
+
)
|
|
322
|
+
if found is None:
|
|
323
|
+
raise InputError(
|
|
324
|
+
f"{self.path.name} has no sheet {name!r}; sheets: {_names(s.name for s in self.sheets)}"
|
|
325
|
+
)
|
|
326
|
+
return found
|
|
327
|
+
|
|
328
|
+
def table_named(self, name: str) -> Table:
|
|
329
|
+
found = next(
|
|
330
|
+
(t for t in self.tables if t.name.casefold() == name.casefold()), None
|
|
331
|
+
)
|
|
332
|
+
if found is None:
|
|
333
|
+
raise InputError(
|
|
334
|
+
f"{self.path.name} has no table {name!r}; tables: {_names(t.name for t in self.tables)}"
|
|
335
|
+
)
|
|
336
|
+
return found
|
|
337
|
+
|
|
338
|
+
def cells(self, sheet: Sheet) -> dict[tuple[int, int], str]:
|
|
339
|
+
"""Non-empty cells of a sheet keyed by (row, column), both 1-based."""
|
|
340
|
+
if sheet.name not in self._grids:
|
|
341
|
+
try:
|
|
342
|
+
self._grids[sheet.name] = dict(self._parse_cells(sheet))
|
|
343
|
+
except (KeyError, ET.ParseError, ValueError, zipfile.BadZipFile) as exc:
|
|
344
|
+
raise InputError(
|
|
345
|
+
f"{self.path.name}: cannot read sheet {sheet.name!r}: {exc}"
|
|
346
|
+
) from exc
|
|
347
|
+
return self._grids[sheet.name]
|
|
348
|
+
|
|
349
|
+
def read_block(
|
|
350
|
+
self, sheet: Sheet, r1: int, c1: int, r2: int, c2: int
|
|
351
|
+
) -> list[list[str]]:
|
|
352
|
+
grid = self.cells(sheet)
|
|
353
|
+
return [
|
|
354
|
+
[grid.get((r, c), "") for c in range(c1, c2 + 1)] for r in range(r1, r2 + 1)
|
|
355
|
+
]
|
|
356
|
+
|
|
357
|
+
def raw_comments(self, sheet: Sheet) -> list[RawComment]:
|
|
358
|
+
"""Comments of a sheet: threaded ones with each reply on its own, else the legacy note on that cell."""
|
|
359
|
+
names = set(self._zip.namelist())
|
|
360
|
+
links = [
|
|
361
|
+
(kind, target)
|
|
362
|
+
for _, kind, target in _typed_relationships(self._zip, sheet.part)
|
|
363
|
+
if target in names
|
|
364
|
+
]
|
|
365
|
+
threaded: list[RawComment] = []
|
|
366
|
+
for _, target in (l for l in links if l[0].endswith("/threadedComment")):
|
|
367
|
+
people = self._people()
|
|
368
|
+
for node in ET.fromstring(self._zip.read(target)).findall(
|
|
369
|
+
"{*}threadedComment"
|
|
370
|
+
):
|
|
371
|
+
text = node.find("{*}text")
|
|
372
|
+
row, col = split_ref(node.get("ref", ""))
|
|
373
|
+
threaded.append(
|
|
374
|
+
RawComment(
|
|
375
|
+
row,
|
|
376
|
+
col,
|
|
377
|
+
people.get(node.get("personId", ""), ""),
|
|
378
|
+
node.get("dT", ""),
|
|
379
|
+
(text.text or "").strip() if text is not None else "",
|
|
380
|
+
)
|
|
381
|
+
)
|
|
382
|
+
threaded.sort(key=lambda c: (c.row, c.col, c.created))
|
|
383
|
+
threaded_cells = {(c.row, c.col) for c in threaded}
|
|
384
|
+
notes: list[RawComment] = []
|
|
385
|
+
for _, target in (l for l in links if l[0].endswith("/comments")):
|
|
386
|
+
root = ET.fromstring(self._zip.read(target))
|
|
387
|
+
authors = [
|
|
388
|
+
(a.text or "").strip() for a in root.findall("m:authors/m:author", NS)
|
|
389
|
+
]
|
|
390
|
+
for node in root.findall("m:commentList/m:comment", NS):
|
|
391
|
+
row, col = split_ref(node.get("ref", ""))
|
|
392
|
+
text_node = node.find("m:text", NS)
|
|
393
|
+
text = _text_of(text_node) if text_node is not None else ""
|
|
394
|
+
if (row, col) in threaded_cells and text.startswith(
|
|
395
|
+
"[Threaded comment]"
|
|
396
|
+
):
|
|
397
|
+
continue
|
|
398
|
+
index = int(node.get("authorId") or -1)
|
|
399
|
+
author = authors[index] if 0 <= index < len(authors) else ""
|
|
400
|
+
if author and text.startswith(author + ":\n"):
|
|
401
|
+
text = text[len(author) + 2 :]
|
|
402
|
+
notes.append(
|
|
403
|
+
RawComment(row, col, author, self._modified(), text.strip())
|
|
404
|
+
)
|
|
405
|
+
notes.sort(key=lambda c: (c.row, c.col))
|
|
406
|
+
return [*notes, *threaded]
|
|
407
|
+
|
|
408
|
+
def _people(self) -> dict[str, str]:
|
|
409
|
+
result: dict[str, str] = {}
|
|
410
|
+
for _, kind, target in _typed_relationships(self._zip, "xl/workbook.xml"):
|
|
411
|
+
if kind.endswith("/person") and target in self._zip.namelist():
|
|
412
|
+
for person in ET.fromstring(self._zip.read(target)).findall(
|
|
413
|
+
"{*}person"
|
|
414
|
+
):
|
|
415
|
+
result[person.get("id", "")] = (
|
|
416
|
+
person.get("displayName") or ""
|
|
417
|
+
).strip()
|
|
418
|
+
return result
|
|
419
|
+
|
|
420
|
+
def _modified(self) -> str:
|
|
421
|
+
if "docProps/core.xml" not in self._zip.namelist():
|
|
422
|
+
return ""
|
|
423
|
+
node = ET.fromstring(self._zip.read("docProps/core.xml")).find(_DC_MODIFIED)
|
|
424
|
+
return (node.text or "").strip() if node is not None else ""
|
|
425
|
+
|
|
426
|
+
def _parse_cells(self, sheet: Sheet) -> Iterator[tuple[tuple[int, int], str]]:
|
|
427
|
+
row_number = col_number = 0
|
|
428
|
+
row_tag, cell_tag = f"{{{_MAIN}}}row", f"{{{_MAIN}}}c"
|
|
429
|
+
with self._zip.open(sheet.part) as handle:
|
|
430
|
+
for event, element in ET.iterparse(handle, events=("start", "end")):
|
|
431
|
+
if element.tag == row_tag and event == "start":
|
|
432
|
+
row_number = int(element.get("r") or row_number + 1)
|
|
433
|
+
col_number = 0
|
|
434
|
+
elif element.tag == cell_tag and event == "end":
|
|
435
|
+
if element.get("r"):
|
|
436
|
+
row_number, col_number = split_ref(element.get("r", ""))
|
|
437
|
+
else:
|
|
438
|
+
col_number += 1
|
|
439
|
+
value = self._value(element)
|
|
440
|
+
if value != "":
|
|
441
|
+
yield (row_number, col_number), value
|
|
442
|
+
element.clear()
|
|
443
|
+
|
|
444
|
+
def _value(self, cell: ET.Element) -> str:
|
|
445
|
+
kind = cell.get("t", "n")
|
|
446
|
+
v = cell.find("m:v", NS)
|
|
447
|
+
raw = v.text if v is not None and v.text is not None else ""
|
|
448
|
+
if kind == "s":
|
|
449
|
+
return self.shared[int(raw)] if raw else ""
|
|
450
|
+
if kind == "inlineStr":
|
|
451
|
+
node = cell.find("m:is", NS)
|
|
452
|
+
return _text_of(node) if node is not None else ""
|
|
453
|
+
if kind in ("str", "e", "d"):
|
|
454
|
+
return raw
|
|
455
|
+
if kind == "b":
|
|
456
|
+
return "TRUE" if raw == "1" else "FALSE"
|
|
457
|
+
if raw == "":
|
|
458
|
+
return ""
|
|
459
|
+
if int(cell.get("s", "0")) in self.date_styles:
|
|
460
|
+
try:
|
|
461
|
+
return _excel_date(float(raw), self.date1904)
|
|
462
|
+
except (OverflowError, ValueError):
|
|
463
|
+
pass
|
|
464
|
+
return _number(raw)
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def _names(names: Iterator[str]) -> str:
|
|
468
|
+
return ", ".join(names) or "(none)"
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def _typed_relationships(
|
|
472
|
+
archive: zipfile.ZipFile, part: str
|
|
473
|
+
) -> list[tuple[str, str, str]]:
|
|
474
|
+
"""(id, type, target part) for each relationship of a part."""
|
|
475
|
+
folder, name = posixpath.split(part)
|
|
476
|
+
path = posixpath.join(folder, "_rels", name + ".rels")
|
|
477
|
+
if path not in archive.namelist():
|
|
478
|
+
return []
|
|
479
|
+
result = []
|
|
480
|
+
for rel in ET.fromstring(archive.read(path)).findall("rel:Relationship", NS):
|
|
481
|
+
target = rel.get("Target", "")
|
|
482
|
+
result.append(
|
|
483
|
+
(
|
|
484
|
+
rel.get("Id", ""),
|
|
485
|
+
rel.get("Type", ""),
|
|
486
|
+
target.lstrip("/")
|
|
487
|
+
if target.startswith("/")
|
|
488
|
+
else posixpath.normpath(posixpath.join(folder, target)),
|
|
489
|
+
)
|
|
490
|
+
)
|
|
491
|
+
return result
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
def _relationships(archive: zipfile.ZipFile, part: str) -> dict[str, str]:
|
|
495
|
+
return {rid: target for rid, _, target in _typed_relationships(archive, part)}
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
def _date_styles(archive: zipfile.ZipFile, names: set[str]) -> frozenset[int]:
|
|
499
|
+
if "xl/styles.xml" not in names:
|
|
500
|
+
return frozenset()
|
|
501
|
+
root = ET.fromstring(archive.read("xl/styles.xml"))
|
|
502
|
+
custom = {
|
|
503
|
+
int(f.get("numFmtId", "0")): f.get("formatCode", "")
|
|
504
|
+
for f in root.findall("m:numFmts/m:numFmt", NS)
|
|
505
|
+
}
|
|
506
|
+
result = set()
|
|
507
|
+
for index, xf in enumerate(root.findall("m:cellXfs/m:xf", NS)):
|
|
508
|
+
format_id = int(xf.get("numFmtId", "0"))
|
|
509
|
+
if _is_date_format(format_id, custom.get(format_id, "")):
|
|
510
|
+
result.add(index)
|
|
511
|
+
return frozenset(result)
|
|
512
|
+
|
|
513
|
+
|
|
514
|
+
def _trim_empty_tail(rows: list[list[str]]) -> list[list[str]]:
|
|
515
|
+
while rows and not any(rows[-1]):
|
|
516
|
+
rows.pop()
|
|
517
|
+
return rows
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
def _table_block(wb: Workbook, table: Table) -> Block:
|
|
521
|
+
if table.header_rows < 1:
|
|
522
|
+
raise InputError(f"{wb.path.name}: table {table.name!r} has no header row")
|
|
523
|
+
r1, c1, r2, c2 = _range(table.ref)
|
|
524
|
+
rows = wb.read_block(
|
|
525
|
+
wb.sheet_named(table.sheet), r1, c1, r2 - table.totals_rows, c2
|
|
526
|
+
)
|
|
527
|
+
return Block(table.name, _trim_empty_tail(rows), r1, c1, table.sheet)
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
def _sheet_block(wb: Workbook, sheet: Sheet, header_row: int | None) -> Block:
|
|
531
|
+
grid = wb.cells(sheet)
|
|
532
|
+
if header_row is None:
|
|
533
|
+
if not grid:
|
|
534
|
+
raise InputError(f"{wb.path.name}: sheet {sheet.name!r} is empty")
|
|
535
|
+
header_row = min(r for r, _ in grid)
|
|
536
|
+
header_cols = [c for r, c in grid if r == header_row]
|
|
537
|
+
if not header_cols:
|
|
538
|
+
raise InputError(
|
|
539
|
+
f"{wb.path.name}: sheet {sheet.name!r} has no header cells in row {header_row}"
|
|
540
|
+
)
|
|
541
|
+
c1, c2 = min(header_cols), max(header_cols)
|
|
542
|
+
last = max((r for r, c in grid if c1 <= c <= c2), default=header_row)
|
|
543
|
+
return Block(
|
|
544
|
+
sheet.name,
|
|
545
|
+
wb.read_block(sheet, header_row, c1, last, c2),
|
|
546
|
+
header_row,
|
|
547
|
+
c1,
|
|
548
|
+
sheet.name,
|
|
549
|
+
)
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
def select_block(wb: Workbook, selection: XlsxSelection) -> Block:
|
|
553
|
+
if selection.table:
|
|
554
|
+
return _table_block(wb, wb.table_named(selection.table))
|
|
555
|
+
if selection.sheet:
|
|
556
|
+
return _sheet_block(wb, wb.sheet_named(selection.sheet), selection.header_row)
|
|
557
|
+
if len(wb.tables) == 1:
|
|
558
|
+
return _table_block(wb, wb.tables[0])
|
|
559
|
+
raise InputError(
|
|
560
|
+
f"{wb.path.name} has {len(wb.tables)} tables; choose one with --table or a sheet with --sheet "
|
|
561
|
+
f"(tables: {_names(t.name for t in wb.tables)}; sheets: {_names(s.name for s in wb.sheets)})"
|
|
562
|
+
)
|
|
563
|
+
|
|
564
|
+
|
|
565
|
+
def read_selection(path: str | Path, selection: XlsxSelection) -> Block:
|
|
566
|
+
with Workbook.open(path) as wb:
|
|
567
|
+
return select_block(wb, selection)
|
|
568
|
+
|
|
569
|
+
|
|
570
|
+
def _source_name(source: dict[str, Any]) -> str:
|
|
571
|
+
return source.get("table") or source["sheet"]
|
|
572
|
+
|
|
573
|
+
|
|
574
|
+
def _slug(text: str) -> str:
|
|
575
|
+
return re.sub(r"[^A-Z0-9]+", "-", text.upper()).strip("-")
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
def _map_source(
|
|
579
|
+
block: Block, source: dict[str, Any], id_column: str | None, traceability: bool
|
|
580
|
+
) -> Part:
|
|
581
|
+
"""Apply one configured source to its block: headings, exclude, rename, set, lists, then Type and generated IDs."""
|
|
582
|
+
where = f"xlsx source {_source_name(source)!r}"
|
|
583
|
+
headers = list(block.rows[0])
|
|
584
|
+
if "" in headers:
|
|
585
|
+
raise InputError(f"{where}: empty column name in header")
|
|
586
|
+
duplicates = sorted({h for h in headers if headers.count(h) > 1})
|
|
587
|
+
if duplicates:
|
|
588
|
+
raise InputError(f"{where}: duplicate column name(s): {', '.join(duplicates)}")
|
|
589
|
+
columns, exclude = source.get("columns", {}), source.get("exclude", [])
|
|
590
|
+
headings = source.get("headings")
|
|
591
|
+
heading_columns = (
|
|
592
|
+
[headings[k] for k in ("column", "text") if k in headings] if headings else []
|
|
593
|
+
)
|
|
594
|
+
for name in (*columns, *exclude, *heading_columns):
|
|
595
|
+
if name not in headers:
|
|
596
|
+
raise ConfigError(
|
|
597
|
+
f"{where} has no column {name!r}; columns: {_names(iter(headers))}"
|
|
598
|
+
)
|
|
599
|
+
live = [
|
|
600
|
+
(block.first_row + 1 + i, row)
|
|
601
|
+
for i, row in enumerate(block.rows[1:])
|
|
602
|
+
if any(cell.strip() for cell in row)
|
|
603
|
+
]
|
|
604
|
+
under: list[str] = [""] * len(live)
|
|
605
|
+
removed = 0
|
|
606
|
+
if headings:
|
|
607
|
+
type_at = headers.index(headings["column"])
|
|
608
|
+
text_at = headers.index(headings["text"]) if "text" in headings else None
|
|
609
|
+
values = {v.strip().casefold() for v in headings["values"]}
|
|
610
|
+
kept, under, current = [], [], ""
|
|
611
|
+
for number, row in live:
|
|
612
|
+
if row[type_at].strip().casefold() in values:
|
|
613
|
+
if text_at is not None:
|
|
614
|
+
current = row[text_at].strip()
|
|
615
|
+
else:
|
|
616
|
+
current = next(
|
|
617
|
+
(
|
|
618
|
+
c.strip()
|
|
619
|
+
for i, c in enumerate(row)
|
|
620
|
+
if i != type_at and c.strip()
|
|
621
|
+
),
|
|
622
|
+
"",
|
|
623
|
+
)
|
|
624
|
+
else:
|
|
625
|
+
kept.append((number, row))
|
|
626
|
+
under.append(current)
|
|
627
|
+
removed = len(live) - len(kept)
|
|
628
|
+
live = kept
|
|
629
|
+
keep = [i for i, h in enumerate(headers) if h not in exclude]
|
|
630
|
+
out_headers = [columns.get(headers[i], headers[i]) for i in keep]
|
|
631
|
+
clashes = sorted({h for h in out_headers if out_headers.count(h) > 1})
|
|
632
|
+
if clashes:
|
|
633
|
+
raise ConfigError(
|
|
634
|
+
f"{where}: columns map to the same output header: {', '.join(clashes)}"
|
|
635
|
+
)
|
|
636
|
+
numbers = [number for number, _ in live]
|
|
637
|
+
rows = [[row[i] for i in keep] for _, row in live]
|
|
638
|
+
|
|
639
|
+
def column(name: str) -> int:
|
|
640
|
+
if name not in out_headers:
|
|
641
|
+
out_headers.append(name)
|
|
642
|
+
for row in rows:
|
|
643
|
+
row.append("")
|
|
644
|
+
return out_headers.index(name)
|
|
645
|
+
|
|
646
|
+
def need(name: str, key: str) -> int:
|
|
647
|
+
if name not in out_headers:
|
|
648
|
+
raise ConfigError(
|
|
649
|
+
f"{where}: {key} names {name!r}, which is not a mapped column; columns: {_names(iter(out_headers))}"
|
|
650
|
+
)
|
|
651
|
+
return out_headers.index(name)
|
|
652
|
+
|
|
653
|
+
constants = dict(source.get("set", {}))
|
|
654
|
+
if "type" in source:
|
|
655
|
+
constants["Type"] = source["type"]
|
|
656
|
+
if "Type" in constants and not traceability:
|
|
657
|
+
raise ConfigError(
|
|
658
|
+
f"{where} sets 'type', which needs traceability: use --traceability or set \"traceability\": true"
|
|
659
|
+
)
|
|
660
|
+
for name, constant in constants.items():
|
|
661
|
+
at = column(name)
|
|
662
|
+
for row in rows:
|
|
663
|
+
row[at] = constant
|
|
664
|
+
categories: tuple[str, ...] = ()
|
|
665
|
+
if headings:
|
|
666
|
+
at = column(headings["into"])
|
|
667
|
+
for row, heading in zip(rows, under):
|
|
668
|
+
if heading and not row[at].strip():
|
|
669
|
+
row[at] = heading
|
|
670
|
+
categories = tuple(row[at].strip() for row in rows)
|
|
671
|
+
for name, separator in source.get("lists", {}).items():
|
|
672
|
+
at = need(name, "lists")
|
|
673
|
+
pattern = r"\r?\n" if separator == "newline" else re.escape(separator)
|
|
674
|
+
for row in rows:
|
|
675
|
+
row[at] = "; ".join(
|
|
676
|
+
p for p in (part.strip() for part in re.split(pattern, row[at])) if p
|
|
677
|
+
)
|
|
678
|
+
if "id_prefix" in source:
|
|
679
|
+
if not id_column:
|
|
680
|
+
raise ConfigError(
|
|
681
|
+
f"{where} sets 'id_prefix', which needs an ID column: use --id-column or set id_column"
|
|
682
|
+
)
|
|
683
|
+
at = column(id_column)
|
|
684
|
+
id_from = source.get("id_from") or (
|
|
685
|
+
["Field", "Name"] if constants.get("Type") == "Value" else None
|
|
686
|
+
)
|
|
687
|
+
parts_at = [need(name, "id_from") for name in id_from] if id_from else []
|
|
688
|
+
names = out_headers.index("Name") if "Name" in out_headers else None
|
|
689
|
+
aliases = out_headers.index("Aliases") if "Aliases" in out_headers else None
|
|
690
|
+
for row in rows:
|
|
691
|
+
if row[at].strip():
|
|
692
|
+
continue
|
|
693
|
+
if id_from:
|
|
694
|
+
slug = "-".join(filter(None, (_slug(row[i]) for i in parts_at)))
|
|
695
|
+
else:
|
|
696
|
+
alias = (
|
|
697
|
+
next((a.strip() for a in row[aliases].split(";") if a.strip()), "")
|
|
698
|
+
if aliases is not None
|
|
699
|
+
else ""
|
|
700
|
+
)
|
|
701
|
+
slug = _slug(alias) or (_slug(row[names]) if names is not None else "")
|
|
702
|
+
if slug:
|
|
703
|
+
row[at] = source["id_prefix"] + slug
|
|
704
|
+
elif source.get("merge"):
|
|
705
|
+
if not id_column:
|
|
706
|
+
raise ConfigError(
|
|
707
|
+
f"{where} is a merge source, which needs an ID column: use --id-column or set id_column"
|
|
708
|
+
)
|
|
709
|
+
if id_column not in out_headers:
|
|
710
|
+
raise ConfigError(
|
|
711
|
+
f"{where} is a merge source with no {id_column!r} column to match records by; "
|
|
712
|
+
f"columns: {_names(iter(out_headers))}"
|
|
713
|
+
)
|
|
714
|
+
elif id_column and id_column not in out_headers:
|
|
715
|
+
raise ConfigError(
|
|
716
|
+
f"{where} has no {id_column!r} column: map one of its columns to {id_column!r}, "
|
|
717
|
+
f"set id_prefix to generate IDs, or remove the source"
|
|
718
|
+
)
|
|
719
|
+
sheet_columns = {
|
|
720
|
+
block.first_col + i: None if h in exclude else columns.get(h, h)
|
|
721
|
+
for i, h in enumerate(headers)
|
|
722
|
+
}
|
|
723
|
+
lists = frozenset(source.get("lists", {}))
|
|
724
|
+
keys = None
|
|
725
|
+
if source.get("merge"):
|
|
726
|
+
at = out_headers.index(id_column)
|
|
727
|
+
keys = [
|
|
728
|
+
_split(row[at])
|
|
729
|
+
if id_column in lists
|
|
730
|
+
else [row[at].strip()]
|
|
731
|
+
if row[at].strip()
|
|
732
|
+
else []
|
|
733
|
+
for row in rows
|
|
734
|
+
]
|
|
735
|
+
return Part(
|
|
736
|
+
block.name,
|
|
737
|
+
block.sheet,
|
|
738
|
+
out_headers,
|
|
739
|
+
rows,
|
|
740
|
+
numbers,
|
|
741
|
+
sheet_columns,
|
|
742
|
+
keys,
|
|
743
|
+
lists,
|
|
744
|
+
removed,
|
|
745
|
+
categories,
|
|
746
|
+
)
|
|
747
|
+
|
|
748
|
+
|
|
749
|
+
def _plain_part(block: Block) -> Part:
|
|
750
|
+
headers = list(block.rows[0])
|
|
751
|
+
body = [
|
|
752
|
+
(block.first_row + 1 + i, row)
|
|
753
|
+
for i, row in enumerate(block.rows[1:])
|
|
754
|
+
if any(c.strip() for c in row)
|
|
755
|
+
]
|
|
756
|
+
return Part(
|
|
757
|
+
block.name,
|
|
758
|
+
block.sheet,
|
|
759
|
+
headers,
|
|
760
|
+
[row for _, row in body],
|
|
761
|
+
[n for n, _ in body],
|
|
762
|
+
{block.first_col + i: h for i, h in enumerate(headers)},
|
|
763
|
+
)
|
|
764
|
+
|
|
765
|
+
|
|
766
|
+
def _split(cell: str) -> list[str]:
|
|
767
|
+
return [p for p in (part.strip() for part in cell.split(";")) if p]
|
|
768
|
+
|
|
769
|
+
|
|
770
|
+
def _listed(items: list[str], limit: int = 12) -> str:
|
|
771
|
+
return ", ".join(items[:limit]) + (
|
|
772
|
+
f" and {len(items) - limit} more" if len(items) > limit else ""
|
|
773
|
+
)
|
|
774
|
+
|
|
775
|
+
|
|
776
|
+
def _repair_ids(parts: list[Part], id_column: str | None, file: str) -> list[str]:
|
|
777
|
+
"""Name rows that have no ID ROW-<worksheet row> and suffix repeated IDs with ~2, ~3, ...; one warning per case."""
|
|
778
|
+
warnings: list[str] = []
|
|
779
|
+
if not id_column:
|
|
780
|
+
return warnings
|
|
781
|
+
usable = [
|
|
782
|
+
(part, part.headers.index(id_column))
|
|
783
|
+
for part in parts
|
|
784
|
+
if id_column in part.headers
|
|
785
|
+
]
|
|
786
|
+
for part, at in usable:
|
|
787
|
+
named = []
|
|
788
|
+
for number, row in zip(part.numbers, part.rows):
|
|
789
|
+
if not row[at].strip():
|
|
790
|
+
row[at] = f"ROW-{number}"
|
|
791
|
+
named.append(row[at])
|
|
792
|
+
if named:
|
|
793
|
+
noun = "row has" if len(named) == 1 else "rows have"
|
|
794
|
+
warnings.append(
|
|
795
|
+
f"{file} [{part.name}]: {len(named)} {noun} no ID in {id_column!r}; "
|
|
796
|
+
f"named {'it' if len(named) == 1 else 'them'} {_listed(named)}"
|
|
797
|
+
)
|
|
798
|
+
seen: dict[str, list[tuple[int, list[str], int]]] = {}
|
|
799
|
+
for part, at in usable:
|
|
800
|
+
for number, row in zip(part.numbers, part.rows):
|
|
801
|
+
seen.setdefault(row[at].strip(), []).append((number, row, at))
|
|
802
|
+
taken = set(seen)
|
|
803
|
+
for rid, found in seen.items():
|
|
804
|
+
if len(found) < 2:
|
|
805
|
+
continue
|
|
806
|
+
renamed = []
|
|
807
|
+
for _, row, at in found[1:]:
|
|
808
|
+
n = 2
|
|
809
|
+
while f"{rid}~{n}" in taken:
|
|
810
|
+
n += 1
|
|
811
|
+
taken.add(f"{rid}~{n}")
|
|
812
|
+
row[at] = f"{rid}~{n}"
|
|
813
|
+
renamed.append(row[at])
|
|
814
|
+
warnings.append(
|
|
815
|
+
f"{file} [{' + '.join(p.name for p in parts)}]: duplicate ID {rid!r} at rows {', '.join(str(n) for n, _, _ in found)}; "
|
|
816
|
+
f"renamed the later {'one' if len(renamed) == 1 else 'ones'} {', '.join(renamed)}"
|
|
817
|
+
)
|
|
818
|
+
return warnings
|
|
819
|
+
|
|
820
|
+
|
|
821
|
+
def _combine(parts: list[Part], id_column: str | None) -> Block:
|
|
822
|
+
"""One block from several parts: shared headers line up, missing cells are empty."""
|
|
823
|
+
headers: list[str] = (
|
|
824
|
+
[id_column] if id_column and any(id_column in p.headers for p in parts) else []
|
|
825
|
+
)
|
|
826
|
+
for part in parts:
|
|
827
|
+
headers += [h for h in part.headers if h not in headers]
|
|
828
|
+
rows = [headers]
|
|
829
|
+
for part in parts:
|
|
830
|
+
rows += [
|
|
831
|
+
[row[part.headers.index(h)] if h in part.headers else "" for h in headers]
|
|
832
|
+
for row in part.rows
|
|
833
|
+
]
|
|
834
|
+
return Block(" + ".join(p.name for p in parts), rows)
|
|
835
|
+
|
|
836
|
+
|
|
837
|
+
def _merge(
|
|
838
|
+
block: Block, part: Part, id_column: str, list_columns: frozenset[str], file: str
|
|
839
|
+
) -> list[str]:
|
|
840
|
+
"""Fill the combined table's records from one merge source; one warning for unknown IDs, one for conflicts."""
|
|
841
|
+
headers = block.rows[0]
|
|
842
|
+
for name in part.headers:
|
|
843
|
+
if name not in headers:
|
|
844
|
+
headers.append(name)
|
|
845
|
+
for row in block.rows[1:]:
|
|
846
|
+
row.append("")
|
|
847
|
+
by_id = {row[headers.index(id_column)].strip(): row for row in block.rows[1:]}
|
|
848
|
+
fills = [
|
|
849
|
+
(
|
|
850
|
+
part.headers.index(h),
|
|
851
|
+
headers.index(h),
|
|
852
|
+
h in list_columns or h in part.lists,
|
|
853
|
+
h,
|
|
854
|
+
)
|
|
855
|
+
for h in part.headers
|
|
856
|
+
if h != id_column
|
|
857
|
+
]
|
|
858
|
+
unknown: dict[str, None] = {}
|
|
859
|
+
conflicts: dict[str, None] = {}
|
|
860
|
+
for row, keys in zip(part.rows, part.keys or []):
|
|
861
|
+
for key in keys:
|
|
862
|
+
target = by_id.get(key)
|
|
863
|
+
if target is None:
|
|
864
|
+
unknown[key] = None
|
|
865
|
+
continue
|
|
866
|
+
for source_at, target_at, is_list, name in fills:
|
|
867
|
+
value = row[source_at]
|
|
868
|
+
if is_list:
|
|
869
|
+
# The record's own cell may not have been split by a lists entry, so read it either way.
|
|
870
|
+
have = {
|
|
871
|
+
part.strip() for part in re.split(r"[;,]", target[target_at])
|
|
872
|
+
}
|
|
873
|
+
new = [v for v in dict.fromkeys(_split(value)) if v not in have]
|
|
874
|
+
if new:
|
|
875
|
+
target[target_at] = "; ".join(
|
|
876
|
+
filter(None, [target[target_at].strip(), *new])
|
|
877
|
+
)
|
|
878
|
+
elif value.strip() and not target[target_at].strip():
|
|
879
|
+
target[target_at] = value
|
|
880
|
+
elif value.strip() and target[target_at].strip() != value.strip():
|
|
881
|
+
conflicts[
|
|
882
|
+
f"{key} {name}: kept {target[target_at].strip()!r}, ignored {value.strip()!r}"
|
|
883
|
+
] = None
|
|
884
|
+
warnings = []
|
|
885
|
+
where = f"{file} [{part.name}]"
|
|
886
|
+
if unknown:
|
|
887
|
+
noun = "ID matches" if len(unknown) == 1 else "IDs match"
|
|
888
|
+
warnings.append(
|
|
889
|
+
f"{where}: {len(unknown)} {noun} no record in {id_column!r}; ignored {_listed(list(unknown))}"
|
|
890
|
+
)
|
|
891
|
+
if conflicts:
|
|
892
|
+
noun = "value differs" if len(conflicts) == 1 else "values differ"
|
|
893
|
+
warnings.append(
|
|
894
|
+
f"{where}: {len(conflicts)} {noun} from the record; {_listed(list(conflicts))}"
|
|
895
|
+
)
|
|
896
|
+
return warnings
|
|
897
|
+
|
|
898
|
+
|
|
899
|
+
def _initials(author: str) -> str:
|
|
900
|
+
words = re.sub(r"[\x00-\x1f\x7f]", " ", author).split()
|
|
901
|
+
if not words:
|
|
902
|
+
return "XL"
|
|
903
|
+
value = (words[0][:2] if len(words) == 1 else "".join(w[0] for w in words)).upper()[
|
|
904
|
+
:16
|
|
905
|
+
]
|
|
906
|
+
return value if valid_initials(value) else "XL"
|
|
907
|
+
|
|
908
|
+
|
|
909
|
+
def _place(
|
|
910
|
+
part: Part, id_column: str | None, comment: RawComment, ids: set[str]
|
|
911
|
+
) -> tuple[str, str, str] | None:
|
|
912
|
+
"""The (requirement ID, output column, text) a comment lands on in this part, or None if it falls outside it.
|
|
913
|
+
|
|
914
|
+
A comment on a merge source's row lands on the first record that row fills in.
|
|
915
|
+
"""
|
|
916
|
+
column = part.columns.get(comment.col)
|
|
917
|
+
if (
|
|
918
|
+
not id_column
|
|
919
|
+
or id_column not in part.headers
|
|
920
|
+
or column is None
|
|
921
|
+
or comment.row not in part.numbers
|
|
922
|
+
):
|
|
923
|
+
return None
|
|
924
|
+
at = part.numbers.index(comment.row)
|
|
925
|
+
if part.keys is None:
|
|
926
|
+
rid = part.rows[at][part.headers.index(id_column)].strip()
|
|
927
|
+
else:
|
|
928
|
+
rid = next((key for key in part.keys[at] if key in ids), None)
|
|
929
|
+
if rid is None:
|
|
930
|
+
return None
|
|
931
|
+
if column != id_column:
|
|
932
|
+
return rid, column, comment.text
|
|
933
|
+
others = [h for h in part.headers if h != id_column]
|
|
934
|
+
return (rid, others[0], f"[on ID] {comment.text}") if others else None
|
|
935
|
+
|
|
936
|
+
|
|
937
|
+
def _review_comments(
|
|
938
|
+
wb: Workbook,
|
|
939
|
+
parts: list[Part],
|
|
940
|
+
id_column: str | None,
|
|
941
|
+
ids: set[str],
|
|
942
|
+
label: str,
|
|
943
|
+
) -> tuple[list[dict[str, Any]], list[str]]:
|
|
944
|
+
comments: list[dict[str, Any]] = []
|
|
945
|
+
dropped = 0
|
|
946
|
+
for sheet_name in dict.fromkeys(part.sheet for part in parts):
|
|
947
|
+
try:
|
|
948
|
+
found = wb.raw_comments(wb.sheet_named(sheet_name))
|
|
949
|
+
except (KeyError, ET.ParseError, ValueError, zipfile.BadZipFile) as exc:
|
|
950
|
+
raise InputError(
|
|
951
|
+
f"{wb.path.name}: cannot read comments of sheet {sheet_name!r}: {exc}"
|
|
952
|
+
) from exc
|
|
953
|
+
for raw in found:
|
|
954
|
+
placed = next(
|
|
955
|
+
(
|
|
956
|
+
hit
|
|
957
|
+
for part in parts
|
|
958
|
+
if part.sheet == sheet_name
|
|
959
|
+
and (hit := _place(part, id_column, raw, ids))
|
|
960
|
+
),
|
|
961
|
+
None,
|
|
962
|
+
)
|
|
963
|
+
if placed is None:
|
|
964
|
+
dropped += 1
|
|
965
|
+
continue
|
|
966
|
+
rid, column, text = placed
|
|
967
|
+
comments.append(
|
|
968
|
+
{
|
|
969
|
+
"id": f"xlsx-{len(comments) + 1}",
|
|
970
|
+
"requirementId": rid,
|
|
971
|
+
"column": column,
|
|
972
|
+
"initials": _initials(raw.author),
|
|
973
|
+
"created": raw.created,
|
|
974
|
+
"modified": None,
|
|
975
|
+
"text": text,
|
|
976
|
+
}
|
|
977
|
+
)
|
|
978
|
+
warnings = []
|
|
979
|
+
if dropped:
|
|
980
|
+
noun = "comment" if dropped == 1 else "comments"
|
|
981
|
+
warnings.append(
|
|
982
|
+
f"{label}: dropped {dropped} Excel {noun} on heading rows, excluded columns or cells outside the imported block"
|
|
983
|
+
)
|
|
984
|
+
return comments, warnings
|
|
985
|
+
|
|
986
|
+
|
|
987
|
+
def import_workbook(
|
|
988
|
+
path: str | Path,
|
|
989
|
+
selection: XlsxSelection,
|
|
990
|
+
sources: list[dict[str, Any]],
|
|
991
|
+
id_column: str | None,
|
|
992
|
+
traceability: bool,
|
|
993
|
+
comments: bool,
|
|
994
|
+
) -> Imported:
|
|
995
|
+
"""Read one workbook the lenient way: configured sources or the selected block, IDs repaired, comments mapped."""
|
|
996
|
+
with Workbook.open(path) as wb:
|
|
997
|
+
if sources:
|
|
998
|
+
parts = [
|
|
999
|
+
_map_source(
|
|
1000
|
+
select_block(
|
|
1001
|
+
wb,
|
|
1002
|
+
XlsxSelection(
|
|
1003
|
+
s.get("table"), s.get("sheet"), s.get("header_row")
|
|
1004
|
+
),
|
|
1005
|
+
),
|
|
1006
|
+
s,
|
|
1007
|
+
id_column,
|
|
1008
|
+
traceability,
|
|
1009
|
+
)
|
|
1010
|
+
for s in sources
|
|
1011
|
+
]
|
|
1012
|
+
else:
|
|
1013
|
+
parts = [_plain_part(select_block(wb, selection))]
|
|
1014
|
+
file = Path(path).name
|
|
1015
|
+
records = [part for part in parts if part.keys is None]
|
|
1016
|
+
warnings = _repair_ids(records, id_column, file)
|
|
1017
|
+
block = _combine(records, id_column)
|
|
1018
|
+
if len(records) < len(parts):
|
|
1019
|
+
# findings imports baselines, which imports this module.
|
|
1020
|
+
from .findings import TRACE_FIELDS
|
|
1021
|
+
|
|
1022
|
+
list_columns = frozenset(
|
|
1023
|
+
[
|
|
1024
|
+
*(f for f, (_, _, single) in TRACE_FIELDS.items() if not single),
|
|
1025
|
+
"Aliases",
|
|
1026
|
+
]
|
|
1027
|
+
)
|
|
1028
|
+
for part in parts:
|
|
1029
|
+
if part.keys is not None:
|
|
1030
|
+
warnings += _merge(block, part, id_column, list_columns, file)
|
|
1031
|
+
block = Block(" + ".join(p.name for p in parts), block.rows)
|
|
1032
|
+
label = f"{file} [{block.name}]"
|
|
1033
|
+
found: list[dict[str, Any]] = []
|
|
1034
|
+
if comments:
|
|
1035
|
+
at = block.rows[0].index(id_column) if id_column in block.rows[0] else None
|
|
1036
|
+
ids = (
|
|
1037
|
+
{row[at].strip() for row in block.rows[1:]} if at is not None else set()
|
|
1038
|
+
)
|
|
1039
|
+
found, extra = _review_comments(wb, parts, id_column, ids, label)
|
|
1040
|
+
warnings += extra
|
|
1041
|
+
stats = tuple(
|
|
1042
|
+
SourceStats(
|
|
1043
|
+
_source_name(s),
|
|
1044
|
+
None if part.keys is not None else len(part.rows),
|
|
1045
|
+
part.headings_removed,
|
|
1046
|
+
part.categories,
|
|
1047
|
+
)
|
|
1048
|
+
for s, part in zip(sources, parts)
|
|
1049
|
+
)
|
|
1050
|
+
return Imported(block, tuple(warnings), tuple(found), stats)
|
|
1051
|
+
|
|
1052
|
+
|
|
1053
|
+
def mapping_template(path: str | Path) -> dict[str, Any]:
|
|
1054
|
+
"""A complete config for a workbook: every table, and every sheet without one, mapped to itself."""
|
|
1055
|
+
sources: list[dict[str, Any]] = []
|
|
1056
|
+
with Workbook.open(path) as wb:
|
|
1057
|
+
for table in wb.tables:
|
|
1058
|
+
if table.header_rows >= 1:
|
|
1059
|
+
sources.append({"table": table.name, "block": _table_block(wb, table)})
|
|
1060
|
+
for sheet in wb.sheets:
|
|
1061
|
+
if any(t.sheet == sheet.name for t in wb.tables) or not wb.cells(sheet):
|
|
1062
|
+
continue
|
|
1063
|
+
first = min(r for r, _ in wb.cells(sheet))
|
|
1064
|
+
sources.append(
|
|
1065
|
+
{
|
|
1066
|
+
"sheet": sheet.name,
|
|
1067
|
+
"header_row": first,
|
|
1068
|
+
"block": _sheet_block(wb, sheet, first),
|
|
1069
|
+
}
|
|
1070
|
+
)
|
|
1071
|
+
config = json.loads(json.dumps(DEFAULT_CONFIG))
|
|
1072
|
+
id_headers = []
|
|
1073
|
+
for source in sources:
|
|
1074
|
+
headers = source.pop("block").rows[0]
|
|
1075
|
+
source["columns"] = {h: h for h in headers}
|
|
1076
|
+
id_headers += [h for h in headers if h.casefold() == "id"]
|
|
1077
|
+
config["id_column"] = id_headers[0] if id_headers else None
|
|
1078
|
+
config["xlsx"]["sources"] = sources
|
|
1079
|
+
return config
|
|
1080
|
+
|
|
1081
|
+
|
|
1082
|
+
def block_to_csv(rows: list[list[str]], delimiter: str) -> str:
|
|
1083
|
+
"""CRLF records, like the browser's Save CSV, without its byte-order mark."""
|
|
1084
|
+
buffer = io.StringIO(newline="")
|
|
1085
|
+
csv.writer(buffer, delimiter=delimiter, lineterminator="\r\n").writerows(rows)
|
|
1086
|
+
return buffer.getvalue()
|
|
1087
|
+
|
|
1088
|
+
|
|
1089
|
+
def describe(path: str | Path) -> str:
|
|
1090
|
+
with Workbook.open(path) as wb:
|
|
1091
|
+
lines = ["Sheets:"]
|
|
1092
|
+
lines += [
|
|
1093
|
+
f" {s.name}" + ("" if s.state == "visible" else f" ({s.state})")
|
|
1094
|
+
for s in wb.sheets
|
|
1095
|
+
]
|
|
1096
|
+
lines.append("Tables:")
|
|
1097
|
+
if not wb.tables:
|
|
1098
|
+
lines.append(" (none)")
|
|
1099
|
+
for table in wb.tables:
|
|
1100
|
+
headers = ""
|
|
1101
|
+
if table.header_rows >= 1:
|
|
1102
|
+
r1, c1, _, c2 = _range(table.ref)
|
|
1103
|
+
row = wb.read_block(wb.sheet_named(table.sheet), r1, c1, r1, c2)[0]
|
|
1104
|
+
headers = " headers: " + ", ".join(row)
|
|
1105
|
+
lines.append(
|
|
1106
|
+
f" {table.name} sheet: {table.sheet} ref: {table.ref}{headers}"
|
|
1107
|
+
)
|
|
1108
|
+
return "\n".join(lines)
|