cif-explorer 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cif_explorer/__init__.py +3 -0
- cif_explorer/__main__.py +3 -0
- cif_explorer/checks.py +665 -0
- cif_explorer/cli.py +86 -0
- cif_explorer/dictionary.py +500 -0
- cif_explorer/explorer.py +803 -0
- cif_explorer/prefs.py +82 -0
- cif_explorer/server.py +242 -0
- cif_explorer/static/app.js +1034 -0
- cif_explorer/static/index.html +67 -0
- cif_explorer/static/style.css +252 -0
- cif_explorer/store.py +609 -0
- cif_explorer-0.1.0.dist-info/METADATA +146 -0
- cif_explorer-0.1.0.dist-info/RECORD +17 -0
- cif_explorer-0.1.0.dist-info/WHEEL +4 -0
- cif_explorer-0.1.0.dist-info/entry_points.txt +3 -0
- cif_explorer-0.1.0.dist-info/licenses/LICENSE +21 -0
cif_explorer/__init__.py
ADDED
cif_explorer/__main__.py
ADDED
cif_explorer/checks.py
ADDED
|
@@ -0,0 +1,665 @@
|
|
|
1
|
+
"""Dictionary conformance checks.
|
|
2
|
+
|
|
3
|
+
Two kinds of checks:
|
|
4
|
+
|
|
5
|
+
* :class:`Checker` - fast checks over the data already in memory, using the
|
|
6
|
+
python-ihm dictionary objects (type regular expressions, enumerations,
|
|
7
|
+
mandatory flags) plus ranges, duplicate keys and parent/child integrity.
|
|
8
|
+
Every distinct value is checked once, so this is cheap even for atom_site,
|
|
9
|
+
and every problem points at concrete rows.
|
|
10
|
+
* :func:`run_ihm_validator` - python-ihm's own
|
|
11
|
+
:meth:`ihm.dictionary.Dictionary.validate`, run on demand over the file.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import gzip
|
|
16
|
+
import io
|
|
17
|
+
import re
|
|
18
|
+
import threading
|
|
19
|
+
import weakref
|
|
20
|
+
from dataclasses import dataclass, field
|
|
21
|
+
from typing import Callable, Dict, List, Optional, Sequence, Tuple
|
|
22
|
+
|
|
23
|
+
import numpy as np
|
|
24
|
+
|
|
25
|
+
import ihm.dictionary
|
|
26
|
+
|
|
27
|
+
from .dictionary import DictInfo, Link
|
|
28
|
+
from .store import NULLS, CatColumn, CifData, Column, DenseColumn, ListColumn, Table
|
|
29
|
+
|
|
30
|
+
MAX_EXAMPLES = 25
|
|
31
|
+
|
|
32
|
+
ERROR, WARNING, INFO = "error", "warning", "info"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass
|
|
36
|
+
class Issue:
|
|
37
|
+
severity: str
|
|
38
|
+
kind: str
|
|
39
|
+
category: str
|
|
40
|
+
message: str
|
|
41
|
+
items: List[str] = field(default_factory=list)
|
|
42
|
+
count: int = 0 # affected rows
|
|
43
|
+
rows: List[int] = field(default_factory=list) # example rows
|
|
44
|
+
values: List[str] = field(default_factory=list) # example values
|
|
45
|
+
link: Optional[str] = None
|
|
46
|
+
|
|
47
|
+
def to_json(self) -> dict:
|
|
48
|
+
return {"severity": self.severity, "kind": self.kind,
|
|
49
|
+
"category": self.category, "items": self.items,
|
|
50
|
+
"message": self.message, "count": self.count,
|
|
51
|
+
"rows": self.rows, "values": self.values, "link": self.link}
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _num(s: Optional[str]) -> Optional[float]:
|
|
55
|
+
if s is None or s in NULLS:
|
|
56
|
+
return None
|
|
57
|
+
try:
|
|
58
|
+
return float(s)
|
|
59
|
+
except ValueError:
|
|
60
|
+
return None
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _in_ranges(x: float, ranges) -> bool:
|
|
64
|
+
"""DDL2 semantics: each (min, max) is an open interval, or the single
|
|
65
|
+
value min when min == max; '.' means unbounded."""
|
|
66
|
+
for lo, hi in ranges:
|
|
67
|
+
if lo is not None and hi is not None and lo == hi:
|
|
68
|
+
if x == lo:
|
|
69
|
+
return True
|
|
70
|
+
continue
|
|
71
|
+
if (lo is None or x > lo) and (hi is None or x < hi):
|
|
72
|
+
return True
|
|
73
|
+
return False
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
# bytes allowed in plain decimal numbers (0 = numpy padding)
|
|
77
|
+
_DIGIT_LUT = np.zeros(256, dtype=bool)
|
|
78
|
+
for _c in b"0123456789":
|
|
79
|
+
_DIGIT_LUT[_c] = True
|
|
80
|
+
_FLOAT_LUT = np.zeros(256, dtype=bool)
|
|
81
|
+
for _c in b"0123456789.-+eE":
|
|
82
|
+
_FLOAT_LUT[_c] = True
|
|
83
|
+
_FLOAT_LUT[0] = True
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
_CHARCLASS_STAR = re.compile(r"\[\]?[^\]]*\]\*")
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _charset_lut(item_type) -> Optional[np.ndarray]:
|
|
90
|
+
"""For types whose construct is one character class repeated
|
|
91
|
+
('[...]*', e.g. code, line, text) return a byte -> allowed table, so a
|
|
92
|
+
whole numpy bytes column can be checked at once."""
|
|
93
|
+
if item_type is None or not _CHARCLASS_STAR.fullmatch(
|
|
94
|
+
item_type.construct):
|
|
95
|
+
return None
|
|
96
|
+
try:
|
|
97
|
+
exact = re.compile(item_type.construct + r"\Z")
|
|
98
|
+
except re.error:
|
|
99
|
+
return None
|
|
100
|
+
lut = np.zeros(256, dtype=bool)
|
|
101
|
+
for b in range(1, 128):
|
|
102
|
+
lut[b] = bool(exact.match(chr(b)))
|
|
103
|
+
lut[0] = True # numpy padding
|
|
104
|
+
return lut
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _find_bad_floats(arr: np.ndarray, cand_idx: np.ndarray) -> np.ndarray:
|
|
108
|
+
"""Locate indices in ``cand_idx`` whose bytes fail float64 conversion."""
|
|
109
|
+
bad: List[np.ndarray] = []
|
|
110
|
+
for start in range(0, len(cand_idx), 4096):
|
|
111
|
+
chunk = cand_idx[start:start + 4096]
|
|
112
|
+
try:
|
|
113
|
+
arr[chunk].astype(np.float64)
|
|
114
|
+
except ValueError:
|
|
115
|
+
for s2 in range(0, len(chunk), 64):
|
|
116
|
+
sub = chunk[s2:s2 + 64]
|
|
117
|
+
try:
|
|
118
|
+
arr[sub].astype(np.float64)
|
|
119
|
+
except ValueError:
|
|
120
|
+
bad.append(sub)
|
|
121
|
+
if not bad:
|
|
122
|
+
return np.empty(0, dtype=np.int64)
|
|
123
|
+
return np.concatenate(bad)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
class _DenseKeySet:
|
|
127
|
+
"""Compact membership container for a single-column bytes DenseColumn."""
|
|
128
|
+
|
|
129
|
+
def __init__(self, arr: np.ndarray):
|
|
130
|
+
self._uniq = np.unique(arr)
|
|
131
|
+
self._max_len = arr.dtype.itemsize
|
|
132
|
+
|
|
133
|
+
def __contains__(self, key) -> bool:
|
|
134
|
+
if not isinstance(key, tuple) or len(key) != 1:
|
|
135
|
+
return False
|
|
136
|
+
try:
|
|
137
|
+
b = key[0].encode("ascii")
|
|
138
|
+
except (AttributeError, UnicodeEncodeError):
|
|
139
|
+
return False
|
|
140
|
+
if len(b) > self._max_len or not len(self._uniq):
|
|
141
|
+
return False
|
|
142
|
+
i = int(np.searchsorted(self._uniq, b))
|
|
143
|
+
return i < len(self._uniq) and self._uniq[i] == b
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
class Checker:
|
|
147
|
+
def __init__(self, dic: Optional[DictInfo]):
|
|
148
|
+
self.dic = dic
|
|
149
|
+
self._lock = threading.RLock()
|
|
150
|
+
self._luts: Dict[str, Optional[np.ndarray]] = {}
|
|
151
|
+
self._cell_cache: Dict[Tuple[str, str, str], Optional[str]] = {}
|
|
152
|
+
self._ranges: Dict[Tuple[str, str], list] = {}
|
|
153
|
+
self._data_ref: Optional[weakref.ref] = None
|
|
154
|
+
self._keysets: Dict[Tuple[str, Tuple[str, ...]], object] = {}
|
|
155
|
+
self._dup_rows: Dict[Tuple[str, Tuple[str, ...]], frozenset] = {}
|
|
156
|
+
|
|
157
|
+
def _bind_data(self, data: CifData) -> None:
|
|
158
|
+
if self._data_ref is None or self._data_ref() is not data:
|
|
159
|
+
self._keysets.clear()
|
|
160
|
+
self._dup_rows.clear()
|
|
161
|
+
self._data_ref = weakref.ref(data)
|
|
162
|
+
|
|
163
|
+
# -- single values ------------------------------------------------------
|
|
164
|
+
def _parsed_ranges(self, cat: str, item: str):
|
|
165
|
+
key = (cat, item)
|
|
166
|
+
with self._lock:
|
|
167
|
+
rs = self._ranges.get(key)
|
|
168
|
+
if rs is None:
|
|
169
|
+
im = self.dic.item(cat, item) if self.dic else None
|
|
170
|
+
rs = []
|
|
171
|
+
if im is not None:
|
|
172
|
+
for lo, hi in im.ranges:
|
|
173
|
+
rs.append((_num(lo), _num(hi)))
|
|
174
|
+
with self._lock:
|
|
175
|
+
self._ranges[key] = rs
|
|
176
|
+
return rs
|
|
177
|
+
|
|
178
|
+
def check_value(self, cat: str, item: str, value: str) -> Optional[str]:
|
|
179
|
+
"""Problem with one value, or None. Results are cached."""
|
|
180
|
+
if self.dic is None:
|
|
181
|
+
return None
|
|
182
|
+
key = (cat, item, value)
|
|
183
|
+
with self._lock:
|
|
184
|
+
if key in self._cell_cache:
|
|
185
|
+
return self._cell_cache[key]
|
|
186
|
+
msg = self._check_value(cat, item, value)
|
|
187
|
+
with self._lock:
|
|
188
|
+
if len(self._cell_cache) > 500000:
|
|
189
|
+
self._cell_cache.clear()
|
|
190
|
+
self._cell_cache[key] = msg
|
|
191
|
+
return msg
|
|
192
|
+
|
|
193
|
+
def _check_value(self, cat, item, value):
|
|
194
|
+
kw = self.dic.keyword(cat, item)
|
|
195
|
+
if kw is None:
|
|
196
|
+
return None
|
|
197
|
+
if value in NULLS:
|
|
198
|
+
if value == "?" and kw.mandatory:
|
|
199
|
+
return "mandatory item has value '?'"
|
|
200
|
+
return None
|
|
201
|
+
if kw.enumeration and value not in kw.enumeration:
|
|
202
|
+
opts = sorted(kw.enumeration)
|
|
203
|
+
shown = ", ".join(opts[:12]) + (", ..." if len(opts) > 12
|
|
204
|
+
else "")
|
|
205
|
+
return "not an allowed value (%s)" % shown
|
|
206
|
+
it = kw.item_type
|
|
207
|
+
if it is not None and not it.regex.match(value):
|
|
208
|
+
return "does not match type '%s'" % it.name
|
|
209
|
+
rs = self._parsed_ranges(cat, item)
|
|
210
|
+
if rs:
|
|
211
|
+
x = _num(value)
|
|
212
|
+
if x is not None and not _in_ranges(x, rs):
|
|
213
|
+
return "out of allowed range (%s)" % self.range_text(cat,
|
|
214
|
+
item)
|
|
215
|
+
return None
|
|
216
|
+
|
|
217
|
+
def range_text(self, cat, item) -> str:
|
|
218
|
+
im = self.dic.item(cat, item) if self.dic else None
|
|
219
|
+
if im is None:
|
|
220
|
+
return ""
|
|
221
|
+
parts = []
|
|
222
|
+
for lo, hi in im.ranges:
|
|
223
|
+
if lo == hi and lo not in (None, "."):
|
|
224
|
+
parts.append("= %s" % lo)
|
|
225
|
+
else:
|
|
226
|
+
parts.append("%s < x < %s" % (
|
|
227
|
+
lo if lo not in (None, ".") else "-inf",
|
|
228
|
+
hi if hi not in (None, ".") else "inf"))
|
|
229
|
+
return " or ".join(parts)
|
|
230
|
+
|
|
231
|
+
# -- whole columns ------------------------------------------------------
|
|
232
|
+
def _numeric_fast_ok(self, col: Column, kw, has_ranges: bool
|
|
233
|
+
) -> Optional[Tuple[np.ndarray, np.ndarray]]:
|
|
234
|
+
"""For a dense bytes column, return (maybe_bad, nulls) masks if a fast
|
|
235
|
+
vectorized path applies, or None to fall back to factorize()."""
|
|
236
|
+
if not isinstance(col, DenseColumn) or col.arr.dtype.kind != "S":
|
|
237
|
+
return None
|
|
238
|
+
it = kw.item_type
|
|
239
|
+
if it is None or kw.enumeration:
|
|
240
|
+
return None
|
|
241
|
+
arr = col.arr
|
|
242
|
+
n = len(arr)
|
|
243
|
+
if n == 0:
|
|
244
|
+
return None
|
|
245
|
+
width = arr.dtype.itemsize
|
|
246
|
+
u8 = arr.view(np.uint8).reshape(n, width)
|
|
247
|
+
nulls = (arr == b"?") | (arr == b".")
|
|
248
|
+
embedded_nul = (
|
|
249
|
+
((u8[:, :-1] == 0) & (u8[:, 1:] != 0)).any(axis=1)
|
|
250
|
+
if width > 1 else np.zeros(n, dtype=bool))
|
|
251
|
+
if it.primitive_code == "numb":
|
|
252
|
+
if it.name == "int" and it.construct == "[+-]?[0-9]+":
|
|
253
|
+
digits = _DIGIT_LUT[u8]
|
|
254
|
+
pad = u8 == 0
|
|
255
|
+
first = u8[:, 0]
|
|
256
|
+
has_sign = (first == ord("+")) | (first == ord("-"))
|
|
257
|
+
if width == 1:
|
|
258
|
+
ok = digits[:, 0]
|
|
259
|
+
else:
|
|
260
|
+
ok = ((digits[:, 0] | (has_sign & digits[:, 1]))
|
|
261
|
+
& (digits[:, 1:] | pad[:, 1:]).all(axis=1)
|
|
262
|
+
& ~embedded_nul)
|
|
263
|
+
maybe_bad = ~nulls & ~ok
|
|
264
|
+
elif it.name == "positive_int" and it.construct == "[1-9][0-9]*":
|
|
265
|
+
digits = _DIGIT_LUT[u8]
|
|
266
|
+
pad = u8 == 0
|
|
267
|
+
first = u8[:, 0]
|
|
268
|
+
ok = (first >= ord("1")) & (first <= ord("9"))
|
|
269
|
+
if width > 1:
|
|
270
|
+
ok &= ((digits[:, 1:] | pad[:, 1:]).all(axis=1)
|
|
271
|
+
& ~embedded_nul)
|
|
272
|
+
maybe_bad = ~nulls & ~ok
|
|
273
|
+
elif it.name == "float":
|
|
274
|
+
chars_ok = (_FLOAT_LUT[u8].all(axis=1)
|
|
275
|
+
& (u8[:, 0] != 0) & ~embedded_nul)
|
|
276
|
+
if not it.regex.match("+1"):
|
|
277
|
+
chars_ok &= u8[:, 0] != ord("+")
|
|
278
|
+
maybe_bad = ~nulls & ~chars_ok
|
|
279
|
+
cand = ~nulls & chars_ok
|
|
280
|
+
try:
|
|
281
|
+
arr[cand].astype(np.float64)
|
|
282
|
+
except ValueError:
|
|
283
|
+
bad_idx = _find_bad_floats(arr, np.flatnonzero(cand))
|
|
284
|
+
maybe_bad[bad_idx] = True
|
|
285
|
+
else:
|
|
286
|
+
return None
|
|
287
|
+
else:
|
|
288
|
+
if has_ranges:
|
|
289
|
+
return None
|
|
290
|
+
with self._lock:
|
|
291
|
+
if it.name not in self._luts:
|
|
292
|
+
self._luts[it.name] = _charset_lut(it)
|
|
293
|
+
lut = self._luts[it.name]
|
|
294
|
+
if lut is None:
|
|
295
|
+
return None
|
|
296
|
+
chars_ok = lut[u8].all(axis=1) & ~embedded_nul
|
|
297
|
+
maybe_bad = ~nulls & ~chars_ok
|
|
298
|
+
if int(maybe_bad.sum()) > 50000:
|
|
299
|
+
return None
|
|
300
|
+
return maybe_bad, nulls
|
|
301
|
+
|
|
302
|
+
def _column_problems(self, table: Table, item: str
|
|
303
|
+
) -> Dict[str, Tuple[np.ndarray, List[str]]]:
|
|
304
|
+
"""message -> (rows, example values) for one column."""
|
|
305
|
+
col = table.columns[item]
|
|
306
|
+
cat = table.name
|
|
307
|
+
kw = self.dic.keyword(cat, item)
|
|
308
|
+
if kw is None:
|
|
309
|
+
return {}
|
|
310
|
+
rs = self._parsed_ranges(cat, item)
|
|
311
|
+
fast = self._numeric_fast_ok(col, kw, bool(rs))
|
|
312
|
+
if fast is not None:
|
|
313
|
+
mask, nulls = fast
|
|
314
|
+
problems: Dict[str, Tuple[List[np.ndarray], List[str]]] = {}
|
|
315
|
+
rows = np.flatnonzero(mask)
|
|
316
|
+
by_msg: Dict[str, Tuple[List[int], List[str]]] = {}
|
|
317
|
+
for r in rows.tolist():
|
|
318
|
+
v = col.value(r)
|
|
319
|
+
msg = self.check_value(cat, item, v)
|
|
320
|
+
if msg:
|
|
321
|
+
entry = by_msg.setdefault(msg, ([], []))
|
|
322
|
+
entry[0].append(r)
|
|
323
|
+
if len(entry[1]) < MAX_EXAMPLES and v not in entry[1]:
|
|
324
|
+
entry[1].append(v)
|
|
325
|
+
for msg, (rlist, vlist) in by_msg.items():
|
|
326
|
+
problems[msg] = ([np.asarray(rlist, dtype=np.int64)], vlist)
|
|
327
|
+
# ranges / mandatory '?' on rows not already checked above
|
|
328
|
+
if rs or kw.mandatory:
|
|
329
|
+
self._dense_ranges(table, item, rs, kw, problems,
|
|
330
|
+
~nulls & ~mask)
|
|
331
|
+
out = {}
|
|
332
|
+
for msg, (parts, vals) in problems.items():
|
|
333
|
+
arr = (parts[0] if len(parts) == 1
|
|
334
|
+
else np.sort(np.concatenate(parts)))
|
|
335
|
+
out[msg] = (arr, vals)
|
|
336
|
+
return out
|
|
337
|
+
uniques, codes = col.factorize()
|
|
338
|
+
bad: Dict[str, List[int]] = {}
|
|
339
|
+
for code, v in enumerate(uniques):
|
|
340
|
+
msg = self.check_value(cat, item, v)
|
|
341
|
+
if msg:
|
|
342
|
+
bad.setdefault(msg, []).append(code)
|
|
343
|
+
out = {}
|
|
344
|
+
for msg, bcodes in bad.items():
|
|
345
|
+
rows = np.flatnonzero(np.isin(codes, np.asarray(bcodes)))
|
|
346
|
+
out[msg] = (rows, [uniques[c] for c in bcodes[:MAX_EXAMPLES]])
|
|
347
|
+
return out
|
|
348
|
+
|
|
349
|
+
def _dense_ranges(self, table, item, rs, kw, problems, valid_mask):
|
|
350
|
+
arr = table.columns[item].arr
|
|
351
|
+
if kw.mandatory:
|
|
352
|
+
rows = np.flatnonzero(arr == b"?")
|
|
353
|
+
if len(rows):
|
|
354
|
+
msg = "mandatory item has value '?'"
|
|
355
|
+
entry = problems.setdefault(msg, ([], []))
|
|
356
|
+
entry[0].append(rows)
|
|
357
|
+
if "?" not in entry[1]:
|
|
358
|
+
entry[1].append("?")
|
|
359
|
+
if not rs:
|
|
360
|
+
return
|
|
361
|
+
idx = np.flatnonzero(valid_mask)
|
|
362
|
+
if not len(idx):
|
|
363
|
+
return
|
|
364
|
+
try:
|
|
365
|
+
x = arr[idx].astype(np.float64)
|
|
366
|
+
except ValueError:
|
|
367
|
+
return
|
|
368
|
+
ok = np.zeros(len(x), dtype=bool)
|
|
369
|
+
for lo, hi in rs:
|
|
370
|
+
if lo is not None and hi is not None and lo == hi:
|
|
371
|
+
ok |= x == lo
|
|
372
|
+
else:
|
|
373
|
+
m = np.ones(len(x), dtype=bool)
|
|
374
|
+
if lo is not None:
|
|
375
|
+
m &= x > lo
|
|
376
|
+
if hi is not None:
|
|
377
|
+
m &= x < hi
|
|
378
|
+
ok |= m
|
|
379
|
+
bad = idx[~ok]
|
|
380
|
+
if len(bad):
|
|
381
|
+
msg = "out of allowed range (%s)" % self.range_text(table.name,
|
|
382
|
+
item)
|
|
383
|
+
entry = problems.setdefault(msg, ([], []))
|
|
384
|
+
entry[0].append(bad)
|
|
385
|
+
for v in sorted(set(table.columns[item].take(bad[:MAX_EXAMPLES]))):
|
|
386
|
+
if len(entry[1]) >= MAX_EXAMPLES:
|
|
387
|
+
break
|
|
388
|
+
if v not in entry[1]:
|
|
389
|
+
entry[1].append(v)
|
|
390
|
+
|
|
391
|
+
def check_table(self, table: Table) -> List[Issue]:
|
|
392
|
+
issues: List[Issue] = []
|
|
393
|
+
cat = table.name
|
|
394
|
+
dcat = self.dic.ihm.categories.get(cat) if self.dic.ihm else None
|
|
395
|
+
if dcat is None:
|
|
396
|
+
issues.append(Issue(ERROR, "unknown_category", cat,
|
|
397
|
+
"category is not defined in the dictionary",
|
|
398
|
+
count=table.nrows))
|
|
399
|
+
return issues
|
|
400
|
+
for item in table.items:
|
|
401
|
+
if item not in dcat.keywords:
|
|
402
|
+
issues.append(Issue(ERROR, "unknown_item", cat,
|
|
403
|
+
"item is not defined in the dictionary",
|
|
404
|
+
items=[item], count=table.nrows))
|
|
405
|
+
for kname, kw in dcat.keywords.items():
|
|
406
|
+
if kw.mandatory and kname not in table.columns:
|
|
407
|
+
issues.append(Issue(ERROR, "missing_item", cat,
|
|
408
|
+
"mandatory item is missing",
|
|
409
|
+
items=[kname]))
|
|
410
|
+
for item in table.items:
|
|
411
|
+
if item not in dcat.keywords:
|
|
412
|
+
continue
|
|
413
|
+
for msg, (rows, vals) in self._column_problems(table,
|
|
414
|
+
item).items():
|
|
415
|
+
sev = WARNING if msg.startswith("out of allowed range") \
|
|
416
|
+
else ERROR
|
|
417
|
+
issues.append(Issue(sev, "bad_value", cat, msg, items=[item],
|
|
418
|
+
count=len(rows),
|
|
419
|
+
rows=rows[:MAX_EXAMPLES].tolist(),
|
|
420
|
+
values=list(vals)[:MAX_EXAMPLES]))
|
|
421
|
+
issues.extend(self._duplicate_keys(table))
|
|
422
|
+
return issues
|
|
423
|
+
|
|
424
|
+
def _duplicate_keys(self, table: Table) -> List[Issue]:
|
|
425
|
+
cm = self.dic.category(table.name)
|
|
426
|
+
if cm is None or not cm.keys or table.nrows < 2:
|
|
427
|
+
return []
|
|
428
|
+
if not all(k in table.columns for k in cm.keys):
|
|
429
|
+
return []
|
|
430
|
+
if len(cm.keys) == 1 and isinstance(table.columns[cm.keys[0]],
|
|
431
|
+
DenseColumn):
|
|
432
|
+
return self._duplicate_dense(table, cm.keys[0])
|
|
433
|
+
key = _combined_codes(table, cm.keys)
|
|
434
|
+
uniq, first, counts = np.unique(key, return_index=True,
|
|
435
|
+
return_counts=True)
|
|
436
|
+
dup = counts > 1
|
|
437
|
+
if not dup.any():
|
|
438
|
+
return []
|
|
439
|
+
dup_keys = uniq[dup]
|
|
440
|
+
rows = np.flatnonzero(np.isin(key, dup_keys))
|
|
441
|
+
vals = []
|
|
442
|
+
for r in first[dup][:MAX_EXAMPLES].tolist():
|
|
443
|
+
vals.append(", ".join("%s=%s" % (k, table.get(r, k))
|
|
444
|
+
for k in cm.keys))
|
|
445
|
+
return [Issue(ERROR, "duplicate_key", table.name,
|
|
446
|
+
"%d key value(s) used by more than one row"
|
|
447
|
+
% int(dup.sum()), items=list(cm.keys),
|
|
448
|
+
count=len(rows), rows=rows[:MAX_EXAMPLES].tolist(),
|
|
449
|
+
values=vals)]
|
|
450
|
+
|
|
451
|
+
def duplicate_rows(self, data: CifData, table: Table,
|
|
452
|
+
items: Tuple[str, ...]) -> frozenset:
|
|
453
|
+
"""Row indices in ``table`` whose ``items`` tuple is not unique."""
|
|
454
|
+
if not items or table.nrows < 2:
|
|
455
|
+
return frozenset()
|
|
456
|
+
cache_key = (table.name, items)
|
|
457
|
+
with self._lock:
|
|
458
|
+
self._bind_data(data)
|
|
459
|
+
cached = self._dup_rows.get(cache_key)
|
|
460
|
+
if cached is not None:
|
|
461
|
+
return cached
|
|
462
|
+
if len(items) == 1 and isinstance(table.columns[items[0]],
|
|
463
|
+
DenseColumn):
|
|
464
|
+
arr = table.columns[items[0]].arr
|
|
465
|
+
uniq, counts = np.unique(arr, return_counts=True)
|
|
466
|
+
dup = uniq[counts > 1]
|
|
467
|
+
rows = (np.flatnonzero(np.isin(arr, dup))
|
|
468
|
+
if len(dup) else np.empty(0, dtype=np.int64))
|
|
469
|
+
else:
|
|
470
|
+
key = _combined_codes(table, items)
|
|
471
|
+
uniq, counts = np.unique(key, return_counts=True)
|
|
472
|
+
dup = uniq[counts > 1]
|
|
473
|
+
rows = (np.flatnonzero(np.isin(key, dup))
|
|
474
|
+
if len(dup) else np.empty(0, dtype=np.int64))
|
|
475
|
+
res = frozenset(rows.tolist())
|
|
476
|
+
with self._lock:
|
|
477
|
+
self._bind_data(data)
|
|
478
|
+
self._dup_rows[cache_key] = res
|
|
479
|
+
return res
|
|
480
|
+
|
|
481
|
+
def _duplicate_dense(self, table: Table, item: str) -> List[Issue]:
|
|
482
|
+
arr = table.columns[item].arr
|
|
483
|
+
uniq, counts = np.unique(arr, return_counts=True)
|
|
484
|
+
dup = uniq[counts > 1]
|
|
485
|
+
if not len(dup):
|
|
486
|
+
return []
|
|
487
|
+
rows = np.flatnonzero(np.isin(arr, dup))
|
|
488
|
+
vals = ["%s=%s" % (item, table.get(int(r), item))
|
|
489
|
+
for r in rows[:MAX_EXAMPLES]]
|
|
490
|
+
return [Issue(ERROR, "duplicate_key", table.name,
|
|
491
|
+
"%d key value(s) used by more than one row" % len(dup),
|
|
492
|
+
items=[item], count=len(rows),
|
|
493
|
+
rows=rows[:MAX_EXAMPLES].tolist(), values=vals)]
|
|
494
|
+
|
|
495
|
+
# -- links ----------------------------------------------------------------
|
|
496
|
+
def parent_keyset(self, data: CifData, parent: Table,
|
|
497
|
+
items: Tuple[str, ...]):
|
|
498
|
+
key = (parent.name, items)
|
|
499
|
+
with self._lock:
|
|
500
|
+
self._bind_data(data)
|
|
501
|
+
ks = self._keysets.get(key)
|
|
502
|
+
if ks is None:
|
|
503
|
+
if len(items) == 1:
|
|
504
|
+
col = parent.columns[items[0]]
|
|
505
|
+
if isinstance(col, CatColumn):
|
|
506
|
+
ks = {(u,) for u in col.uniques}
|
|
507
|
+
elif isinstance(col, ListColumn):
|
|
508
|
+
ks = {(v,) for v in col._values}
|
|
509
|
+
elif isinstance(col, DenseColumn) and col.arr.dtype.kind == "S":
|
|
510
|
+
ks = _DenseKeySet(col.arr)
|
|
511
|
+
else:
|
|
512
|
+
ks = {(v,) for v in col.values(0, parent.nrows)}
|
|
513
|
+
else:
|
|
514
|
+
cols = [parent.columns[i].values(0, parent.nrows)
|
|
515
|
+
for i in items]
|
|
516
|
+
ks = set(zip(*cols))
|
|
517
|
+
with self._lock:
|
|
518
|
+
self._bind_data(data)
|
|
519
|
+
self._keysets[key] = ks
|
|
520
|
+
return ks
|
|
521
|
+
|
|
522
|
+
def check_links(self, data: CifData, table: Table) -> List[Issue]:
|
|
523
|
+
issues = []
|
|
524
|
+
for link in self.dic.parents_of(table.name):
|
|
525
|
+
iss = self._check_link(data, table, link)
|
|
526
|
+
if iss:
|
|
527
|
+
issues.append(iss)
|
|
528
|
+
return issues
|
|
529
|
+
|
|
530
|
+
def _check_link(self, data: CifData, table: Table,
|
|
531
|
+
link: Link) -> Optional[Issue]:
|
|
532
|
+
direct = link.child_items[:link.n_direct]
|
|
533
|
+
if not all(c in table.columns for c in direct):
|
|
534
|
+
return None
|
|
535
|
+
pairs = [(c, p) for c, p in zip(link.child_items, link.parent_items)
|
|
536
|
+
if c in table.columns]
|
|
537
|
+
citems = tuple(c for c, _ in pairs)
|
|
538
|
+
pitems = tuple(p for _, p in pairs)
|
|
539
|
+
facts = [table.columns[c].factorize() for c in citems]
|
|
540
|
+
# rows where all direct items are non-null
|
|
541
|
+
valid = np.ones(table.nrows, dtype=bool)
|
|
542
|
+
for (uniq, codes), c in zip(facts, citems):
|
|
543
|
+
if c in direct:
|
|
544
|
+
nullc = [i for i, u in enumerate(uniq) if u in NULLS]
|
|
545
|
+
if nullc:
|
|
546
|
+
valid &= ~np.isin(codes, nullc)
|
|
547
|
+
idx = np.flatnonzero(valid)
|
|
548
|
+
if len(idx) == 0:
|
|
549
|
+
return None
|
|
550
|
+
key = _mixed_radix([codes[idx] for _, codes in facts],
|
|
551
|
+
[len(u) for u, _ in facts])
|
|
552
|
+
ukeys, first, counts = np.unique(key, return_index=True,
|
|
553
|
+
return_counts=True)
|
|
554
|
+
parent = data.get(link.parent)
|
|
555
|
+
if parent is None:
|
|
556
|
+
if link.parent.startswith("chem_comp_"):
|
|
557
|
+
return None
|
|
558
|
+
sev = WARNING if link.parent.startswith("ihm_") else INFO
|
|
559
|
+
return Issue(sev, "missing_parent_category", table.name,
|
|
560
|
+
"refers to category %s, which is not in the file"
|
|
561
|
+
% link.parent, items=list(direct),
|
|
562
|
+
count=int(counts.sum()),
|
|
563
|
+
rows=idx[first[:MAX_EXAMPLES]].tolist(),
|
|
564
|
+
link=link.id)
|
|
565
|
+
if not all(p in parent.columns for p in pitems[:link.n_direct]):
|
|
566
|
+
return None
|
|
567
|
+
bad_count, bad_vals, bad_u_idx = 0, [], []
|
|
568
|
+
keysets: Dict[Tuple[str, ...], object] = {}
|
|
569
|
+
for u_i, (f, n) in enumerate(zip(first.tolist(), counts.tolist())):
|
|
570
|
+
r = int(idx[f])
|
|
571
|
+
vals = [table.columns[c].value(r) for c in citems]
|
|
572
|
+
use = tuple(i for i, (c, v) in enumerate(zip(citems, vals))
|
|
573
|
+
if c in direct or v not in NULLS)
|
|
574
|
+
use = tuple(i for i in use if pitems[i] in parent.columns)
|
|
575
|
+
sub_p = tuple(pitems[i] for i in use)
|
|
576
|
+
if sub_p not in keysets:
|
|
577
|
+
keysets[sub_p] = self.parent_keyset(data, parent, sub_p)
|
|
578
|
+
if tuple(vals[i] for i in use) not in keysets[sub_p]:
|
|
579
|
+
bad_count += n
|
|
580
|
+
bad_u_idx.append(u_i)
|
|
581
|
+
if len(bad_vals) < MAX_EXAMPLES:
|
|
582
|
+
bad_vals.append(", ".join("%s=%s" % (citems[i], vals[i])
|
|
583
|
+
for i in use))
|
|
584
|
+
if not bad_count:
|
|
585
|
+
return None
|
|
586
|
+
bad_mask = np.isin(key, ukeys[bad_u_idx])
|
|
587
|
+
rows = idx[np.flatnonzero(bad_mask)[:MAX_EXAMPLES]]
|
|
588
|
+
return Issue(ERROR, "broken_link", table.name,
|
|
589
|
+
"%d row(s) refer to %s rows that do not exist (%d "
|
|
590
|
+
"distinct value(s))" % (bad_count, link.parent,
|
|
591
|
+
len(bad_u_idx)),
|
|
592
|
+
items=list(citems), count=bad_count,
|
|
593
|
+
rows=rows.tolist(), values=bad_vals, link=link.id)
|
|
594
|
+
|
|
595
|
+
# -- everything -------------------------------------------------------
|
|
596
|
+
def run(self, data: CifData,
|
|
597
|
+
progress: Callable[[str, float], None] = lambda s, f: None
|
|
598
|
+
) -> List[Issue]:
|
|
599
|
+
issues: List[Issue] = list(self.check_categories(data))
|
|
600
|
+
if self.dic is None:
|
|
601
|
+
return issues
|
|
602
|
+
tables = list(data.tables.values())
|
|
603
|
+
total = sum(max(t.nrows, 1) for t in tables) or 1
|
|
604
|
+
done = 0
|
|
605
|
+
for t in tables:
|
|
606
|
+
progress("checking %s" % t.name, done / total)
|
|
607
|
+
issues.extend(self.check_table(t))
|
|
608
|
+
issues.extend(self.check_links(data, t))
|
|
609
|
+
done += max(t.nrows, 1)
|
|
610
|
+
progress("checked", 1.0)
|
|
611
|
+
return issues
|
|
612
|
+
|
|
613
|
+
def check_categories(self, data: CifData) -> List[Issue]:
|
|
614
|
+
if self.dic is None or self.dic.ihm is None:
|
|
615
|
+
return []
|
|
616
|
+
out = []
|
|
617
|
+
missing = sorted(c.name for c in self.dic.ihm.categories.values()
|
|
618
|
+
if c.mandatory and c.name not in data.tables)
|
|
619
|
+
for name in missing:
|
|
620
|
+
out.append(Issue(ERROR, "missing_category", name,
|
|
621
|
+
"mandatory category is missing from the file"))
|
|
622
|
+
return out
|
|
623
|
+
|
|
624
|
+
|
|
625
|
+
def _combined_codes(table: Table, items: Sequence[str]) -> np.ndarray:
|
|
626
|
+
facts = [table.columns[i].factorize() for i in items]
|
|
627
|
+
return _mixed_radix([c for _, c in facts], [len(u) for u, _ in facts])
|
|
628
|
+
|
|
629
|
+
|
|
630
|
+
def _mixed_radix(codes: List[np.ndarray], sizes: List[int]) -> np.ndarray:
|
|
631
|
+
"""Combine several code arrays into one comparable 1D key per row."""
|
|
632
|
+
if not codes or len(codes[0]) == 0:
|
|
633
|
+
return np.empty(0, dtype=np.int64)
|
|
634
|
+
total = 1
|
|
635
|
+
for s in sizes:
|
|
636
|
+
total *= max(s, 1)
|
|
637
|
+
if total < 2 ** 62:
|
|
638
|
+
key = np.zeros(len(codes[0]), dtype=np.int64)
|
|
639
|
+
for c, s in zip(codes, sizes):
|
|
640
|
+
key = key * max(s, 1) + c.astype(np.int64)
|
|
641
|
+
return key
|
|
642
|
+
mat = np.ascontiguousarray(np.column_stack(codes), dtype=np.int64)
|
|
643
|
+
return mat.view(np.dtype((np.void, mat.strides[0]))).ravel()
|
|
644
|
+
|
|
645
|
+
|
|
646
|
+
def run_ihm_validator(path: str, fmt: str, gzipped: bool,
|
|
647
|
+
dic: DictInfo) -> List[str]:
|
|
648
|
+
"""Run python-ihm's dictionary validator; return its error messages."""
|
|
649
|
+
if dic is None or dic.ihm is None:
|
|
650
|
+
return ["no dictionary available"]
|
|
651
|
+
opener = gzip.open if gzipped else open
|
|
652
|
+
if fmt == "BCIF":
|
|
653
|
+
with opener(path, "rb") as fh:
|
|
654
|
+
return _validate(dic, fh, "BCIF")
|
|
655
|
+
with opener(path, "rb") as raw:
|
|
656
|
+
fh = io.TextIOWrapper(raw, encoding="utf-8", errors="replace")
|
|
657
|
+
return _validate(dic, fh, "mmCIF")
|
|
658
|
+
|
|
659
|
+
|
|
660
|
+
def _validate(dic: DictInfo, fh, fmt: str) -> List[str]:
|
|
661
|
+
try:
|
|
662
|
+
dic.ihm.validate(fh, format=fmt)
|
|
663
|
+
except ihm.dictionary.ValidatorError as exc:
|
|
664
|
+
return [m for m in str(exc).split("\n\n") if m.strip()]
|
|
665
|
+
return []
|