cif-explorer 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,3 @@
1
+ """cif_explorer: trace IHMCIF/mmCIF rows up and down the category hierarchy."""
2
+
3
+ __version__ = "0.1.0"
@@ -0,0 +1,3 @@
1
+ from .cli import main
2
+
3
+ main()
cif_explorer/checks.py ADDED
@@ -0,0 +1,665 @@
1
+ """Dictionary conformance checks.
2
+
3
+ Two kinds of checks:
4
+
5
+ * :class:`Checker` - fast checks over the data already in memory, using the
6
+ python-ihm dictionary objects (type regular expressions, enumerations,
7
+ mandatory flags) plus ranges, duplicate keys and parent/child integrity.
8
+ Every distinct value is checked once, so this is cheap even for atom_site,
9
+ and every problem points at concrete rows.
10
+ * :func:`run_ihm_validator` - python-ihm's own
11
+ :meth:`ihm.dictionary.Dictionary.validate`, run on demand over the file.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import gzip
16
+ import io
17
+ import re
18
+ import threading
19
+ import weakref
20
+ from dataclasses import dataclass, field
21
+ from typing import Callable, Dict, List, Optional, Sequence, Tuple
22
+
23
+ import numpy as np
24
+
25
+ import ihm.dictionary
26
+
27
+ from .dictionary import DictInfo, Link
28
+ from .store import NULLS, CatColumn, CifData, Column, DenseColumn, ListColumn, Table
29
+
30
+ MAX_EXAMPLES = 25
31
+
32
+ ERROR, WARNING, INFO = "error", "warning", "info"
33
+
34
+
35
+ @dataclass
36
+ class Issue:
37
+ severity: str
38
+ kind: str
39
+ category: str
40
+ message: str
41
+ items: List[str] = field(default_factory=list)
42
+ count: int = 0 # affected rows
43
+ rows: List[int] = field(default_factory=list) # example rows
44
+ values: List[str] = field(default_factory=list) # example values
45
+ link: Optional[str] = None
46
+
47
+ def to_json(self) -> dict:
48
+ return {"severity": self.severity, "kind": self.kind,
49
+ "category": self.category, "items": self.items,
50
+ "message": self.message, "count": self.count,
51
+ "rows": self.rows, "values": self.values, "link": self.link}
52
+
53
+
54
+ def _num(s: Optional[str]) -> Optional[float]:
55
+ if s is None or s in NULLS:
56
+ return None
57
+ try:
58
+ return float(s)
59
+ except ValueError:
60
+ return None
61
+
62
+
63
+ def _in_ranges(x: float, ranges) -> bool:
64
+ """DDL2 semantics: each (min, max) is an open interval, or the single
65
+ value min when min == max; '.' means unbounded."""
66
+ for lo, hi in ranges:
67
+ if lo is not None and hi is not None and lo == hi:
68
+ if x == lo:
69
+ return True
70
+ continue
71
+ if (lo is None or x > lo) and (hi is None or x < hi):
72
+ return True
73
+ return False
74
+
75
+
76
+ # bytes allowed in plain decimal numbers (0 = numpy padding)
77
+ _DIGIT_LUT = np.zeros(256, dtype=bool)
78
+ for _c in b"0123456789":
79
+ _DIGIT_LUT[_c] = True
80
+ _FLOAT_LUT = np.zeros(256, dtype=bool)
81
+ for _c in b"0123456789.-+eE":
82
+ _FLOAT_LUT[_c] = True
83
+ _FLOAT_LUT[0] = True
84
+
85
+
86
+ _CHARCLASS_STAR = re.compile(r"\[\]?[^\]]*\]\*")
87
+
88
+
89
+ def _charset_lut(item_type) -> Optional[np.ndarray]:
90
+ """For types whose construct is one character class repeated
91
+ ('[...]*', e.g. code, line, text) return a byte -> allowed table, so a
92
+ whole numpy bytes column can be checked at once."""
93
+ if item_type is None or not _CHARCLASS_STAR.fullmatch(
94
+ item_type.construct):
95
+ return None
96
+ try:
97
+ exact = re.compile(item_type.construct + r"\Z")
98
+ except re.error:
99
+ return None
100
+ lut = np.zeros(256, dtype=bool)
101
+ for b in range(1, 128):
102
+ lut[b] = bool(exact.match(chr(b)))
103
+ lut[0] = True # numpy padding
104
+ return lut
105
+
106
+
107
+ def _find_bad_floats(arr: np.ndarray, cand_idx: np.ndarray) -> np.ndarray:
108
+ """Locate indices in ``cand_idx`` whose bytes fail float64 conversion."""
109
+ bad: List[np.ndarray] = []
110
+ for start in range(0, len(cand_idx), 4096):
111
+ chunk = cand_idx[start:start + 4096]
112
+ try:
113
+ arr[chunk].astype(np.float64)
114
+ except ValueError:
115
+ for s2 in range(0, len(chunk), 64):
116
+ sub = chunk[s2:s2 + 64]
117
+ try:
118
+ arr[sub].astype(np.float64)
119
+ except ValueError:
120
+ bad.append(sub)
121
+ if not bad:
122
+ return np.empty(0, dtype=np.int64)
123
+ return np.concatenate(bad)
124
+
125
+
126
+ class _DenseKeySet:
127
+ """Compact membership container for a single-column bytes DenseColumn."""
128
+
129
+ def __init__(self, arr: np.ndarray):
130
+ self._uniq = np.unique(arr)
131
+ self._max_len = arr.dtype.itemsize
132
+
133
+ def __contains__(self, key) -> bool:
134
+ if not isinstance(key, tuple) or len(key) != 1:
135
+ return False
136
+ try:
137
+ b = key[0].encode("ascii")
138
+ except (AttributeError, UnicodeEncodeError):
139
+ return False
140
+ if len(b) > self._max_len or not len(self._uniq):
141
+ return False
142
+ i = int(np.searchsorted(self._uniq, b))
143
+ return i < len(self._uniq) and self._uniq[i] == b
144
+
145
+
146
+ class Checker:
147
+ def __init__(self, dic: Optional[DictInfo]):
148
+ self.dic = dic
149
+ self._lock = threading.RLock()
150
+ self._luts: Dict[str, Optional[np.ndarray]] = {}
151
+ self._cell_cache: Dict[Tuple[str, str, str], Optional[str]] = {}
152
+ self._ranges: Dict[Tuple[str, str], list] = {}
153
+ self._data_ref: Optional[weakref.ref] = None
154
+ self._keysets: Dict[Tuple[str, Tuple[str, ...]], object] = {}
155
+ self._dup_rows: Dict[Tuple[str, Tuple[str, ...]], frozenset] = {}
156
+
157
+ def _bind_data(self, data: CifData) -> None:
158
+ if self._data_ref is None or self._data_ref() is not data:
159
+ self._keysets.clear()
160
+ self._dup_rows.clear()
161
+ self._data_ref = weakref.ref(data)
162
+
163
+ # -- single values ------------------------------------------------------
164
+ def _parsed_ranges(self, cat: str, item: str):
165
+ key = (cat, item)
166
+ with self._lock:
167
+ rs = self._ranges.get(key)
168
+ if rs is None:
169
+ im = self.dic.item(cat, item) if self.dic else None
170
+ rs = []
171
+ if im is not None:
172
+ for lo, hi in im.ranges:
173
+ rs.append((_num(lo), _num(hi)))
174
+ with self._lock:
175
+ self._ranges[key] = rs
176
+ return rs
177
+
178
+ def check_value(self, cat: str, item: str, value: str) -> Optional[str]:
179
+ """Problem with one value, or None. Results are cached."""
180
+ if self.dic is None:
181
+ return None
182
+ key = (cat, item, value)
183
+ with self._lock:
184
+ if key in self._cell_cache:
185
+ return self._cell_cache[key]
186
+ msg = self._check_value(cat, item, value)
187
+ with self._lock:
188
+ if len(self._cell_cache) > 500000:
189
+ self._cell_cache.clear()
190
+ self._cell_cache[key] = msg
191
+ return msg
192
+
193
+ def _check_value(self, cat, item, value):
194
+ kw = self.dic.keyword(cat, item)
195
+ if kw is None:
196
+ return None
197
+ if value in NULLS:
198
+ if value == "?" and kw.mandatory:
199
+ return "mandatory item has value '?'"
200
+ return None
201
+ if kw.enumeration and value not in kw.enumeration:
202
+ opts = sorted(kw.enumeration)
203
+ shown = ", ".join(opts[:12]) + (", ..." if len(opts) > 12
204
+ else "")
205
+ return "not an allowed value (%s)" % shown
206
+ it = kw.item_type
207
+ if it is not None and not it.regex.match(value):
208
+ return "does not match type '%s'" % it.name
209
+ rs = self._parsed_ranges(cat, item)
210
+ if rs:
211
+ x = _num(value)
212
+ if x is not None and not _in_ranges(x, rs):
213
+ return "out of allowed range (%s)" % self.range_text(cat,
214
+ item)
215
+ return None
216
+
217
+ def range_text(self, cat, item) -> str:
218
+ im = self.dic.item(cat, item) if self.dic else None
219
+ if im is None:
220
+ return ""
221
+ parts = []
222
+ for lo, hi in im.ranges:
223
+ if lo == hi and lo not in (None, "."):
224
+ parts.append("= %s" % lo)
225
+ else:
226
+ parts.append("%s < x < %s" % (
227
+ lo if lo not in (None, ".") else "-inf",
228
+ hi if hi not in (None, ".") else "inf"))
229
+ return " or ".join(parts)
230
+
231
+ # -- whole columns ------------------------------------------------------
232
+ def _numeric_fast_ok(self, col: Column, kw, has_ranges: bool
233
+ ) -> Optional[Tuple[np.ndarray, np.ndarray]]:
234
+ """For a dense bytes column, return (maybe_bad, nulls) masks if a fast
235
+ vectorized path applies, or None to fall back to factorize()."""
236
+ if not isinstance(col, DenseColumn) or col.arr.dtype.kind != "S":
237
+ return None
238
+ it = kw.item_type
239
+ if it is None or kw.enumeration:
240
+ return None
241
+ arr = col.arr
242
+ n = len(arr)
243
+ if n == 0:
244
+ return None
245
+ width = arr.dtype.itemsize
246
+ u8 = arr.view(np.uint8).reshape(n, width)
247
+ nulls = (arr == b"?") | (arr == b".")
248
+ embedded_nul = (
249
+ ((u8[:, :-1] == 0) & (u8[:, 1:] != 0)).any(axis=1)
250
+ if width > 1 else np.zeros(n, dtype=bool))
251
+ if it.primitive_code == "numb":
252
+ if it.name == "int" and it.construct == "[+-]?[0-9]+":
253
+ digits = _DIGIT_LUT[u8]
254
+ pad = u8 == 0
255
+ first = u8[:, 0]
256
+ has_sign = (first == ord("+")) | (first == ord("-"))
257
+ if width == 1:
258
+ ok = digits[:, 0]
259
+ else:
260
+ ok = ((digits[:, 0] | (has_sign & digits[:, 1]))
261
+ & (digits[:, 1:] | pad[:, 1:]).all(axis=1)
262
+ & ~embedded_nul)
263
+ maybe_bad = ~nulls & ~ok
264
+ elif it.name == "positive_int" and it.construct == "[1-9][0-9]*":
265
+ digits = _DIGIT_LUT[u8]
266
+ pad = u8 == 0
267
+ first = u8[:, 0]
268
+ ok = (first >= ord("1")) & (first <= ord("9"))
269
+ if width > 1:
270
+ ok &= ((digits[:, 1:] | pad[:, 1:]).all(axis=1)
271
+ & ~embedded_nul)
272
+ maybe_bad = ~nulls & ~ok
273
+ elif it.name == "float":
274
+ chars_ok = (_FLOAT_LUT[u8].all(axis=1)
275
+ & (u8[:, 0] != 0) & ~embedded_nul)
276
+ if not it.regex.match("+1"):
277
+ chars_ok &= u8[:, 0] != ord("+")
278
+ maybe_bad = ~nulls & ~chars_ok
279
+ cand = ~nulls & chars_ok
280
+ try:
281
+ arr[cand].astype(np.float64)
282
+ except ValueError:
283
+ bad_idx = _find_bad_floats(arr, np.flatnonzero(cand))
284
+ maybe_bad[bad_idx] = True
285
+ else:
286
+ return None
287
+ else:
288
+ if has_ranges:
289
+ return None
290
+ with self._lock:
291
+ if it.name not in self._luts:
292
+ self._luts[it.name] = _charset_lut(it)
293
+ lut = self._luts[it.name]
294
+ if lut is None:
295
+ return None
296
+ chars_ok = lut[u8].all(axis=1) & ~embedded_nul
297
+ maybe_bad = ~nulls & ~chars_ok
298
+ if int(maybe_bad.sum()) > 50000:
299
+ return None
300
+ return maybe_bad, nulls
301
+
302
+ def _column_problems(self, table: Table, item: str
303
+ ) -> Dict[str, Tuple[np.ndarray, List[str]]]:
304
+ """message -> (rows, example values) for one column."""
305
+ col = table.columns[item]
306
+ cat = table.name
307
+ kw = self.dic.keyword(cat, item)
308
+ if kw is None:
309
+ return {}
310
+ rs = self._parsed_ranges(cat, item)
311
+ fast = self._numeric_fast_ok(col, kw, bool(rs))
312
+ if fast is not None:
313
+ mask, nulls = fast
314
+ problems: Dict[str, Tuple[List[np.ndarray], List[str]]] = {}
315
+ rows = np.flatnonzero(mask)
316
+ by_msg: Dict[str, Tuple[List[int], List[str]]] = {}
317
+ for r in rows.tolist():
318
+ v = col.value(r)
319
+ msg = self.check_value(cat, item, v)
320
+ if msg:
321
+ entry = by_msg.setdefault(msg, ([], []))
322
+ entry[0].append(r)
323
+ if len(entry[1]) < MAX_EXAMPLES and v not in entry[1]:
324
+ entry[1].append(v)
325
+ for msg, (rlist, vlist) in by_msg.items():
326
+ problems[msg] = ([np.asarray(rlist, dtype=np.int64)], vlist)
327
+ # ranges / mandatory '?' on rows not already checked above
328
+ if rs or kw.mandatory:
329
+ self._dense_ranges(table, item, rs, kw, problems,
330
+ ~nulls & ~mask)
331
+ out = {}
332
+ for msg, (parts, vals) in problems.items():
333
+ arr = (parts[0] if len(parts) == 1
334
+ else np.sort(np.concatenate(parts)))
335
+ out[msg] = (arr, vals)
336
+ return out
337
+ uniques, codes = col.factorize()
338
+ bad: Dict[str, List[int]] = {}
339
+ for code, v in enumerate(uniques):
340
+ msg = self.check_value(cat, item, v)
341
+ if msg:
342
+ bad.setdefault(msg, []).append(code)
343
+ out = {}
344
+ for msg, bcodes in bad.items():
345
+ rows = np.flatnonzero(np.isin(codes, np.asarray(bcodes)))
346
+ out[msg] = (rows, [uniques[c] for c in bcodes[:MAX_EXAMPLES]])
347
+ return out
348
+
349
+ def _dense_ranges(self, table, item, rs, kw, problems, valid_mask):
350
+ arr = table.columns[item].arr
351
+ if kw.mandatory:
352
+ rows = np.flatnonzero(arr == b"?")
353
+ if len(rows):
354
+ msg = "mandatory item has value '?'"
355
+ entry = problems.setdefault(msg, ([], []))
356
+ entry[0].append(rows)
357
+ if "?" not in entry[1]:
358
+ entry[1].append("?")
359
+ if not rs:
360
+ return
361
+ idx = np.flatnonzero(valid_mask)
362
+ if not len(idx):
363
+ return
364
+ try:
365
+ x = arr[idx].astype(np.float64)
366
+ except ValueError:
367
+ return
368
+ ok = np.zeros(len(x), dtype=bool)
369
+ for lo, hi in rs:
370
+ if lo is not None and hi is not None and lo == hi:
371
+ ok |= x == lo
372
+ else:
373
+ m = np.ones(len(x), dtype=bool)
374
+ if lo is not None:
375
+ m &= x > lo
376
+ if hi is not None:
377
+ m &= x < hi
378
+ ok |= m
379
+ bad = idx[~ok]
380
+ if len(bad):
381
+ msg = "out of allowed range (%s)" % self.range_text(table.name,
382
+ item)
383
+ entry = problems.setdefault(msg, ([], []))
384
+ entry[0].append(bad)
385
+ for v in sorted(set(table.columns[item].take(bad[:MAX_EXAMPLES]))):
386
+ if len(entry[1]) >= MAX_EXAMPLES:
387
+ break
388
+ if v not in entry[1]:
389
+ entry[1].append(v)
390
+
391
+ def check_table(self, table: Table) -> List[Issue]:
392
+ issues: List[Issue] = []
393
+ cat = table.name
394
+ dcat = self.dic.ihm.categories.get(cat) if self.dic.ihm else None
395
+ if dcat is None:
396
+ issues.append(Issue(ERROR, "unknown_category", cat,
397
+ "category is not defined in the dictionary",
398
+ count=table.nrows))
399
+ return issues
400
+ for item in table.items:
401
+ if item not in dcat.keywords:
402
+ issues.append(Issue(ERROR, "unknown_item", cat,
403
+ "item is not defined in the dictionary",
404
+ items=[item], count=table.nrows))
405
+ for kname, kw in dcat.keywords.items():
406
+ if kw.mandatory and kname not in table.columns:
407
+ issues.append(Issue(ERROR, "missing_item", cat,
408
+ "mandatory item is missing",
409
+ items=[kname]))
410
+ for item in table.items:
411
+ if item not in dcat.keywords:
412
+ continue
413
+ for msg, (rows, vals) in self._column_problems(table,
414
+ item).items():
415
+ sev = WARNING if msg.startswith("out of allowed range") \
416
+ else ERROR
417
+ issues.append(Issue(sev, "bad_value", cat, msg, items=[item],
418
+ count=len(rows),
419
+ rows=rows[:MAX_EXAMPLES].tolist(),
420
+ values=list(vals)[:MAX_EXAMPLES]))
421
+ issues.extend(self._duplicate_keys(table))
422
+ return issues
423
+
424
+ def _duplicate_keys(self, table: Table) -> List[Issue]:
425
+ cm = self.dic.category(table.name)
426
+ if cm is None or not cm.keys or table.nrows < 2:
427
+ return []
428
+ if not all(k in table.columns for k in cm.keys):
429
+ return []
430
+ if len(cm.keys) == 1 and isinstance(table.columns[cm.keys[0]],
431
+ DenseColumn):
432
+ return self._duplicate_dense(table, cm.keys[0])
433
+ key = _combined_codes(table, cm.keys)
434
+ uniq, first, counts = np.unique(key, return_index=True,
435
+ return_counts=True)
436
+ dup = counts > 1
437
+ if not dup.any():
438
+ return []
439
+ dup_keys = uniq[dup]
440
+ rows = np.flatnonzero(np.isin(key, dup_keys))
441
+ vals = []
442
+ for r in first[dup][:MAX_EXAMPLES].tolist():
443
+ vals.append(", ".join("%s=%s" % (k, table.get(r, k))
444
+ for k in cm.keys))
445
+ return [Issue(ERROR, "duplicate_key", table.name,
446
+ "%d key value(s) used by more than one row"
447
+ % int(dup.sum()), items=list(cm.keys),
448
+ count=len(rows), rows=rows[:MAX_EXAMPLES].tolist(),
449
+ values=vals)]
450
+
451
+ def duplicate_rows(self, data: CifData, table: Table,
452
+ items: Tuple[str, ...]) -> frozenset:
453
+ """Row indices in ``table`` whose ``items`` tuple is not unique."""
454
+ if not items or table.nrows < 2:
455
+ return frozenset()
456
+ cache_key = (table.name, items)
457
+ with self._lock:
458
+ self._bind_data(data)
459
+ cached = self._dup_rows.get(cache_key)
460
+ if cached is not None:
461
+ return cached
462
+ if len(items) == 1 and isinstance(table.columns[items[0]],
463
+ DenseColumn):
464
+ arr = table.columns[items[0]].arr
465
+ uniq, counts = np.unique(arr, return_counts=True)
466
+ dup = uniq[counts > 1]
467
+ rows = (np.flatnonzero(np.isin(arr, dup))
468
+ if len(dup) else np.empty(0, dtype=np.int64))
469
+ else:
470
+ key = _combined_codes(table, items)
471
+ uniq, counts = np.unique(key, return_counts=True)
472
+ dup = uniq[counts > 1]
473
+ rows = (np.flatnonzero(np.isin(key, dup))
474
+ if len(dup) else np.empty(0, dtype=np.int64))
475
+ res = frozenset(rows.tolist())
476
+ with self._lock:
477
+ self._bind_data(data)
478
+ self._dup_rows[cache_key] = res
479
+ return res
480
+
481
+ def _duplicate_dense(self, table: Table, item: str) -> List[Issue]:
482
+ arr = table.columns[item].arr
483
+ uniq, counts = np.unique(arr, return_counts=True)
484
+ dup = uniq[counts > 1]
485
+ if not len(dup):
486
+ return []
487
+ rows = np.flatnonzero(np.isin(arr, dup))
488
+ vals = ["%s=%s" % (item, table.get(int(r), item))
489
+ for r in rows[:MAX_EXAMPLES]]
490
+ return [Issue(ERROR, "duplicate_key", table.name,
491
+ "%d key value(s) used by more than one row" % len(dup),
492
+ items=[item], count=len(rows),
493
+ rows=rows[:MAX_EXAMPLES].tolist(), values=vals)]
494
+
495
+ # -- links ----------------------------------------------------------------
496
+ def parent_keyset(self, data: CifData, parent: Table,
497
+ items: Tuple[str, ...]):
498
+ key = (parent.name, items)
499
+ with self._lock:
500
+ self._bind_data(data)
501
+ ks = self._keysets.get(key)
502
+ if ks is None:
503
+ if len(items) == 1:
504
+ col = parent.columns[items[0]]
505
+ if isinstance(col, CatColumn):
506
+ ks = {(u,) for u in col.uniques}
507
+ elif isinstance(col, ListColumn):
508
+ ks = {(v,) for v in col._values}
509
+ elif isinstance(col, DenseColumn) and col.arr.dtype.kind == "S":
510
+ ks = _DenseKeySet(col.arr)
511
+ else:
512
+ ks = {(v,) for v in col.values(0, parent.nrows)}
513
+ else:
514
+ cols = [parent.columns[i].values(0, parent.nrows)
515
+ for i in items]
516
+ ks = set(zip(*cols))
517
+ with self._lock:
518
+ self._bind_data(data)
519
+ self._keysets[key] = ks
520
+ return ks
521
+
522
+ def check_links(self, data: CifData, table: Table) -> List[Issue]:
523
+ issues = []
524
+ for link in self.dic.parents_of(table.name):
525
+ iss = self._check_link(data, table, link)
526
+ if iss:
527
+ issues.append(iss)
528
+ return issues
529
+
530
+ def _check_link(self, data: CifData, table: Table,
531
+ link: Link) -> Optional[Issue]:
532
+ direct = link.child_items[:link.n_direct]
533
+ if not all(c in table.columns for c in direct):
534
+ return None
535
+ pairs = [(c, p) for c, p in zip(link.child_items, link.parent_items)
536
+ if c in table.columns]
537
+ citems = tuple(c for c, _ in pairs)
538
+ pitems = tuple(p for _, p in pairs)
539
+ facts = [table.columns[c].factorize() for c in citems]
540
+ # rows where all direct items are non-null
541
+ valid = np.ones(table.nrows, dtype=bool)
542
+ for (uniq, codes), c in zip(facts, citems):
543
+ if c in direct:
544
+ nullc = [i for i, u in enumerate(uniq) if u in NULLS]
545
+ if nullc:
546
+ valid &= ~np.isin(codes, nullc)
547
+ idx = np.flatnonzero(valid)
548
+ if len(idx) == 0:
549
+ return None
550
+ key = _mixed_radix([codes[idx] for _, codes in facts],
551
+ [len(u) for u, _ in facts])
552
+ ukeys, first, counts = np.unique(key, return_index=True,
553
+ return_counts=True)
554
+ parent = data.get(link.parent)
555
+ if parent is None:
556
+ if link.parent.startswith("chem_comp_"):
557
+ return None
558
+ sev = WARNING if link.parent.startswith("ihm_") else INFO
559
+ return Issue(sev, "missing_parent_category", table.name,
560
+ "refers to category %s, which is not in the file"
561
+ % link.parent, items=list(direct),
562
+ count=int(counts.sum()),
563
+ rows=idx[first[:MAX_EXAMPLES]].tolist(),
564
+ link=link.id)
565
+ if not all(p in parent.columns for p in pitems[:link.n_direct]):
566
+ return None
567
+ bad_count, bad_vals, bad_u_idx = 0, [], []
568
+ keysets: Dict[Tuple[str, ...], object] = {}
569
+ for u_i, (f, n) in enumerate(zip(first.tolist(), counts.tolist())):
570
+ r = int(idx[f])
571
+ vals = [table.columns[c].value(r) for c in citems]
572
+ use = tuple(i for i, (c, v) in enumerate(zip(citems, vals))
573
+ if c in direct or v not in NULLS)
574
+ use = tuple(i for i in use if pitems[i] in parent.columns)
575
+ sub_p = tuple(pitems[i] for i in use)
576
+ if sub_p not in keysets:
577
+ keysets[sub_p] = self.parent_keyset(data, parent, sub_p)
578
+ if tuple(vals[i] for i in use) not in keysets[sub_p]:
579
+ bad_count += n
580
+ bad_u_idx.append(u_i)
581
+ if len(bad_vals) < MAX_EXAMPLES:
582
+ bad_vals.append(", ".join("%s=%s" % (citems[i], vals[i])
583
+ for i in use))
584
+ if not bad_count:
585
+ return None
586
+ bad_mask = np.isin(key, ukeys[bad_u_idx])
587
+ rows = idx[np.flatnonzero(bad_mask)[:MAX_EXAMPLES]]
588
+ return Issue(ERROR, "broken_link", table.name,
589
+ "%d row(s) refer to %s rows that do not exist (%d "
590
+ "distinct value(s))" % (bad_count, link.parent,
591
+ len(bad_u_idx)),
592
+ items=list(citems), count=bad_count,
593
+ rows=rows.tolist(), values=bad_vals, link=link.id)
594
+
595
+ # -- everything -------------------------------------------------------
596
+ def run(self, data: CifData,
597
+ progress: Callable[[str, float], None] = lambda s, f: None
598
+ ) -> List[Issue]:
599
+ issues: List[Issue] = list(self.check_categories(data))
600
+ if self.dic is None:
601
+ return issues
602
+ tables = list(data.tables.values())
603
+ total = sum(max(t.nrows, 1) for t in tables) or 1
604
+ done = 0
605
+ for t in tables:
606
+ progress("checking %s" % t.name, done / total)
607
+ issues.extend(self.check_table(t))
608
+ issues.extend(self.check_links(data, t))
609
+ done += max(t.nrows, 1)
610
+ progress("checked", 1.0)
611
+ return issues
612
+
613
+ def check_categories(self, data: CifData) -> List[Issue]:
614
+ if self.dic is None or self.dic.ihm is None:
615
+ return []
616
+ out = []
617
+ missing = sorted(c.name for c in self.dic.ihm.categories.values()
618
+ if c.mandatory and c.name not in data.tables)
619
+ for name in missing:
620
+ out.append(Issue(ERROR, "missing_category", name,
621
+ "mandatory category is missing from the file"))
622
+ return out
623
+
624
+
625
+ def _combined_codes(table: Table, items: Sequence[str]) -> np.ndarray:
626
+ facts = [table.columns[i].factorize() for i in items]
627
+ return _mixed_radix([c for _, c in facts], [len(u) for u, _ in facts])
628
+
629
+
630
+ def _mixed_radix(codes: List[np.ndarray], sizes: List[int]) -> np.ndarray:
631
+ """Combine several code arrays into one comparable 1D key per row."""
632
+ if not codes or len(codes[0]) == 0:
633
+ return np.empty(0, dtype=np.int64)
634
+ total = 1
635
+ for s in sizes:
636
+ total *= max(s, 1)
637
+ if total < 2 ** 62:
638
+ key = np.zeros(len(codes[0]), dtype=np.int64)
639
+ for c, s in zip(codes, sizes):
640
+ key = key * max(s, 1) + c.astype(np.int64)
641
+ return key
642
+ mat = np.ascontiguousarray(np.column_stack(codes), dtype=np.int64)
643
+ return mat.view(np.dtype((np.void, mat.strides[0]))).ravel()
644
+
645
+
646
+ def run_ihm_validator(path: str, fmt: str, gzipped: bool,
647
+ dic: DictInfo) -> List[str]:
648
+ """Run python-ihm's dictionary validator; return its error messages."""
649
+ if dic is None or dic.ihm is None:
650
+ return ["no dictionary available"]
651
+ opener = gzip.open if gzipped else open
652
+ if fmt == "BCIF":
653
+ with opener(path, "rb") as fh:
654
+ return _validate(dic, fh, "BCIF")
655
+ with opener(path, "rb") as raw:
656
+ fh = io.TextIOWrapper(raw, encoding="utf-8", errors="replace")
657
+ return _validate(dic, fh, "mmCIF")
658
+
659
+
660
+ def _validate(dic: DictInfo, fh, fmt: str) -> List[str]:
661
+ try:
662
+ dic.ihm.validate(fh, format=fmt)
663
+ except ihm.dictionary.ValidatorError as exc:
664
+ return [m for m in str(exc).split("\n\n") if m.strip()]
665
+ return []