dazdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dazdiff/__init__.py +34 -0
- dazdiff/__main__.py +5 -0
- dazdiff/cli.py +248 -0
- dazdiff/core.py +1357 -0
- dazdiff/py.typed +0 -0
- dazdiff/report.py +483 -0
- dazdiff-0.1.0.dist-info/METADATA +167 -0
- dazdiff-0.1.0.dist-info/RECORD +11 -0
- dazdiff-0.1.0.dist-info/WHEEL +4 -0
- dazdiff-0.1.0.dist-info/entry_points.txt +2 -0
- dazdiff-0.1.0.dist-info/licenses/LICENSE +21 -0
dazdiff/core.py
ADDED
|
@@ -0,0 +1,1357 @@
|
|
|
1
|
+
"""Compare two tables value by value."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
import re
|
|
7
|
+
import warnings
|
|
8
|
+
from collections.abc import Sequence
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
import pandas as pd
|
|
12
|
+
|
|
13
|
+
STATUSES = ["same", "changed", "only_x", "only_y"]
|
|
14
|
+
_MODES = ("benchmark", "equal")
|
|
15
|
+
_DUPLICATES = ("disambiguate", "aggregate", "error")
|
|
16
|
+
_ROW_KEYS = ("none", "position", "sorted")
|
|
17
|
+
_MEASURE_KEYWORDS = ("numeric", "numeric+integer")
|
|
18
|
+
_TRUE = {"true", "t", "yes", "1"}
|
|
19
|
+
_FALSE = {"false", "f", "no", "0"}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class DiffError(ValueError):
|
|
23
|
+
"""The two tables cannot be compared as asked."""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class DiffWarning(UserWarning):
|
|
27
|
+
"""The answer may be wrong, e.g. rows were paired by arrival order."""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class _PairingWarning(DiffWarning):
|
|
31
|
+
"""Repeated keys were paired in arrival order (a DiffWarning; the subclass lets
|
|
32
|
+
assert_table_equal() drop it when it refuses the pairing outright)."""
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class DiffNote(UserWarning):
|
|
36
|
+
"""An expected side effect, e.g. a column was dropped or a value truncated."""
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
# --- argument checks ---------------------------------------------------------
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _check_frame(v, name: str) -> None:
|
|
43
|
+
if not isinstance(v, pd.DataFrame):
|
|
44
|
+
raise DiffError(f"`{name}` must be a pandas DataFrame, not {type(v).__name__}.")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _check_result(d) -> None:
|
|
48
|
+
if not isinstance(d, pd.DataFrame) or "status" not in d:
|
|
49
|
+
raise DiffError("`d` must be a result of diff_table().")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _check_threshold(v, name: str, allow_none: bool = False) -> None:
|
|
53
|
+
if v is None and allow_none:
|
|
54
|
+
return
|
|
55
|
+
ok = (
|
|
56
|
+
isinstance(v, (int, float, np.integer, np.floating))
|
|
57
|
+
and not isinstance(v, (bool, np.bool_))
|
|
58
|
+
and bool(np.isfinite(v))
|
|
59
|
+
and v >= 0
|
|
60
|
+
)
|
|
61
|
+
if not ok:
|
|
62
|
+
none = "None or " if allow_none else ""
|
|
63
|
+
raise DiffError(f"`{name}` must be {none}a single finite number >= 0, not {v!r}.")
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _check_columns_arg(v, name: str, allow_empty: bool) -> None:
|
|
67
|
+
"""A column name, or a list or tuple of them."""
|
|
68
|
+
if isinstance(v, str):
|
|
69
|
+
return
|
|
70
|
+
ok = isinstance(v, (list, tuple)) and all(isinstance(c, str) for c in v)
|
|
71
|
+
if not ok or (not v and not allow_empty):
|
|
72
|
+
what = (
|
|
73
|
+
"a column name or a non-empty list of them"
|
|
74
|
+
if not allow_empty
|
|
75
|
+
else ("a keyword, a column name or a list of them")
|
|
76
|
+
)
|
|
77
|
+
raise DiffError(f"`{name}` must be {what}, not {v!r}.")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
# --- names -------------------------------------------------------------------
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _canon(name) -> str:
|
|
84
|
+
return re.sub(r"[^A-Z0-9]+", "_", str(name).upper()).strip("_")
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _join(names) -> str:
|
|
88
|
+
return ", ".join(str(n) for n in names)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def normalize_names(names: Sequence[str]) -> list[str]:
|
|
92
|
+
"""Normalize column names as :func:`diff_table` does.
|
|
93
|
+
|
|
94
|
+
Upper-case, every run of characters other than ``A-Z`` and ``0-9`` replaced by
|
|
95
|
+
one ``_``, and ``_`` trimmed from both ends. A name that ends up empty, or two
|
|
96
|
+
names that end up the same, are an error.
|
|
97
|
+
"""
|
|
98
|
+
out = [_canon(n) for n in names]
|
|
99
|
+
_check_names(list(names), out, label="names")
|
|
100
|
+
return out
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _check_names(original: list, canon: list[str], label: str) -> None:
|
|
104
|
+
empty = [o for o, c in zip(original, canon, strict=True) if not c]
|
|
105
|
+
if empty:
|
|
106
|
+
raise DiffError(f"Column name(s) of {label} are empty once normalized: {empty!r}.")
|
|
107
|
+
groups: dict[str, list] = {}
|
|
108
|
+
for o, c in zip(original, canon, strict=True):
|
|
109
|
+
groups.setdefault(c, []).append(o)
|
|
110
|
+
clashes = [g for g in groups.values() if len(g) > 1]
|
|
111
|
+
if clashes:
|
|
112
|
+
listed = "; ".join(_join(g) for g in clashes)
|
|
113
|
+
raise DiffError(f"Column names of {label} clash once normalized: {listed}.")
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
# --- casting -----------------------------------------------------------------
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _is_float(s: pd.Series) -> bool:
|
|
120
|
+
return pd.api.types.is_float_dtype(s)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _is_int(s: pd.Series) -> bool:
|
|
124
|
+
return pd.api.types.is_integer_dtype(s) and not pd.api.types.is_bool_dtype(s)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _is_bool(s: pd.Series) -> bool:
|
|
128
|
+
return pd.api.types.is_bool_dtype(s)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _is_datetime(s: pd.Series) -> bool:
|
|
132
|
+
return pd.api.types.is_datetime64_any_dtype(s)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _is_text(s: pd.Series) -> bool:
|
|
136
|
+
return pd.api.types.is_object_dtype(s) or isinstance(s.dtype, pd.StringDtype)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _show(v) -> str:
|
|
140
|
+
"""A value as an error message shows it: strings quoted, numbers plain."""
|
|
141
|
+
if isinstance(v, str):
|
|
142
|
+
return repr(v)
|
|
143
|
+
return str(v.item() if isinstance(v, np.generic) else v)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _first_bad(original: pd.Series, converted: pd.Series):
|
|
147
|
+
bad = original.notna() & converted.isna()
|
|
148
|
+
return original[bad].iloc[0]
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _to_text(v):
|
|
152
|
+
if isinstance(v, float) and v.is_integer():
|
|
153
|
+
return str(int(v + 0.0)) # + 0.0 writes -0.0 as "0"
|
|
154
|
+
return str(v)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _examples(values, n: int = 3) -> str:
|
|
158
|
+
shown = [_show(v) if not pd.isna(v) else "NA" for v in list(values)[:n]]
|
|
159
|
+
return ", ".join(shown) + (", ..." if len(values) > n else "")
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
_PLAIN_DECIMAL = re.compile(r"-?(0|[1-9][0-9]*)(\.[0-9]+)?")
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _text_decimals(ref: pd.Series) -> tuple[int | None, list[int], bool]:
|
|
166
|
+
"""The one number of decimals every value of the text column ``ref`` uses.
|
|
167
|
+
|
|
168
|
+
Read off every distinct value, not a sample: one ``"1.00"`` among a million
|
|
169
|
+
``"1.0"`` is the value a sample misses. Only plain decimal text counts; a sign,
|
|
170
|
+
leading zeros, separators or an exponent mean there is no format to follow.
|
|
171
|
+
Returns the decimals (or None), the counts of decimals seen, and whether every
|
|
172
|
+
value is plain decimal text.
|
|
173
|
+
"""
|
|
174
|
+
u = [str(v) for v in pd.unique(ref.dropna())]
|
|
175
|
+
plain = [bool(_PLAIN_DECIMAL.fullmatch(v)) for v in u]
|
|
176
|
+
seen = sorted({len(v.partition(".")[2]) for v, p in zip(u, plain, strict=True) if p})
|
|
177
|
+
k = seen[0] if u and all(plain) and len(seen) == 1 else None
|
|
178
|
+
return k, seen, all(plain)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _number_as_text_like(v: pd.Series, out: pd.Series, ref: pd.Series, name: str, alerts: list):
|
|
182
|
+
"""Write ``y``'s numbers ``v`` (already text in ``out``) as the benchmark ``ref``
|
|
183
|
+
writes them, where every value of ``ref`` has the same number of decimals.
|
|
184
|
+
|
|
185
|
+
A value is reformatted only when its text reads back as the same number: 1.25 at
|
|
186
|
+
one decimal would become "1.2" and join the benchmark's 1.2, a silent false
|
|
187
|
+
match, so it keeps its own text and shows as unmatched instead.
|
|
188
|
+
"""
|
|
189
|
+
k, seen, plain = _text_decimals(ref)
|
|
190
|
+
num = v.to_numpy(dtype=float, na_value=np.nan)
|
|
191
|
+
finite = np.isfinite(num)
|
|
192
|
+
fits = np.zeros(len(v), dtype=bool)
|
|
193
|
+
kept = np.zeros(len(v), dtype=bool)
|
|
194
|
+
out = out.copy()
|
|
195
|
+
if k:
|
|
196
|
+
txt = np.array([f"{n + 0.0:.{k}f}" if f else "" for n, f in zip(num, finite, strict=True)])
|
|
197
|
+
fits = finite & np.array(
|
|
198
|
+
[f and float(t) == n for t, n, f in zip(txt, num, finite, strict=True)]
|
|
199
|
+
)
|
|
200
|
+
kept = finite & ~fits
|
|
201
|
+
out[fits] = txt[fits]
|
|
202
|
+
|
|
203
|
+
# Values of x that are numbers y has, written differently: those rows will not align.
|
|
204
|
+
first_text = {}
|
|
205
|
+
for n, t in zip(num, out, strict=True):
|
|
206
|
+
if np.isfinite(n):
|
|
207
|
+
first_text.setdefault(n + 0.0, t)
|
|
208
|
+
have = set(out.dropna())
|
|
209
|
+
u = [str(s) for s in pd.unique(ref.dropna())]
|
|
210
|
+
u_num = pd.to_numeric(pd.Series(u, dtype=object), errors="coerce").to_numpy(dtype=float)
|
|
211
|
+
skew = [
|
|
212
|
+
(s, first_text[n + 0.0])
|
|
213
|
+
for s, n in zip(u, u_num, strict=True)
|
|
214
|
+
if np.isfinite(n) and n + 0.0 in first_text and s not in have
|
|
215
|
+
]
|
|
216
|
+
if skew:
|
|
217
|
+
pairs = ", ".join(f"{a!r} vs {b!r}" for a, b in skew[:3]) + (
|
|
218
|
+
", ..." if len(skew) > 3 else ""
|
|
219
|
+
)
|
|
220
|
+
if k:
|
|
221
|
+
why = f"`y`'s numbers were written with {k} decimal place{'s' if k > 1 else ''}, as `x` writes them"
|
|
222
|
+
elif not plain:
|
|
223
|
+
why = "no decimal places were inferred, since not every value of `x` is plain decimal text"
|
|
224
|
+
elif len(seen) > 1:
|
|
225
|
+
why = f"no decimal places were inferred, since `x` writes them with {_join(seen)} decimal places"
|
|
226
|
+
else:
|
|
227
|
+
why = "`x` writes them with no decimal places"
|
|
228
|
+
alerts.append(
|
|
229
|
+
(
|
|
230
|
+
DiffWarning,
|
|
231
|
+
f"Column `{name}`: {len(skew)} value(s) of `x` equal a number of `y` but not its "
|
|
232
|
+
f"text ({pairs}), so those rows will not align; {why}; cast the column of `x` "
|
|
233
|
+
"to a number to compare by value.",
|
|
234
|
+
)
|
|
235
|
+
)
|
|
236
|
+
elif fits.any():
|
|
237
|
+
firsts = pd.Series(num[fits]).drop_duplicates().index[:3]
|
|
238
|
+
shown = [(num[fits][i], out[fits].iloc[i]) for i in firsts]
|
|
239
|
+
listed = ", ".join(f"{n:g} -> {t!r}" for n, t in shown)
|
|
240
|
+
more = ", ..." if int(fits.sum()) > len(shown) else ""
|
|
241
|
+
extra = ""
|
|
242
|
+
if kept.any():
|
|
243
|
+
kept_text = ", ".join(repr(t) for t in pd.unique(out[kept])[:3])
|
|
244
|
+
extra = f"; {int(kept.sum())} value(s) needed more decimals and kept their own digits ({kept_text})"
|
|
245
|
+
alerts.append(
|
|
246
|
+
(
|
|
247
|
+
DiffNote,
|
|
248
|
+
f"Wrote column `{name}` of `y` as text with {k} decimal place{'s' if k > 1 else ''}, "
|
|
249
|
+
f"as `x` writes it: {int(fits.sum())} value(s) ({listed}{more}){extra}.",
|
|
250
|
+
)
|
|
251
|
+
)
|
|
252
|
+
return out
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def _stamp_to_text(y: pd.Series) -> pd.Series:
|
|
256
|
+
"""Date-times as ``%Y-%m-%d %H:%M:%S``, midnight included, with the fraction of a
|
|
257
|
+
second to the microsecond, without trailing zeros, when there is one."""
|
|
258
|
+
s = y.dt.round("us")
|
|
259
|
+
txt = s.dt.strftime("%Y-%m-%d %H:%M:%S")
|
|
260
|
+
us = s.dt.microsecond
|
|
261
|
+
frac = s.notna() & (us > 0)
|
|
262
|
+
txt[frac] = txt[frac] + "." + us[frac].map(lambda u: f"{int(u):06d}".rstrip("0"))
|
|
263
|
+
return txt
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
_INT_TEXT = re.compile(r"[+-]?[0-9]+")
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _nullable(dtype) -> str:
|
|
270
|
+
"""The pandas nullable integer dtype for an integer dtype: int8 -> Int8."""
|
|
271
|
+
if isinstance(dtype, pd.api.extensions.ExtensionDtype):
|
|
272
|
+
return str(dtype)
|
|
273
|
+
return str(dtype).replace("uint", "UInt").replace("int", "Int")
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _cast_to_int(
|
|
277
|
+
y: pd.Series, x: pd.Series, name: str, truncate: bool, truncated: dict
|
|
278
|
+
) -> pd.Series:
|
|
279
|
+
"""Cast ``y`` to the integer dtype of ``x`` exactly.
|
|
280
|
+
|
|
281
|
+
Integer dtypes and integer text become Python ints, so a value above 2**53 keeps
|
|
282
|
+
every digit; other numbers go through float, where a fraction truncates. A value
|
|
283
|
+
outside the target dtype's range is refused rather than wrapped around.
|
|
284
|
+
"""
|
|
285
|
+
target = x.dtype.numpy_dtype if hasattr(x.dtype, "numpy_dtype") else x.dtype
|
|
286
|
+
info = np.iinfo(target)
|
|
287
|
+
missing = y.isna().to_numpy()
|
|
288
|
+
if _is_int(y) or _is_bool(y):
|
|
289
|
+
values = [None if m else int(v) for v, m in zip(y.tolist(), missing, strict=True)]
|
|
290
|
+
frac: list = []
|
|
291
|
+
else:
|
|
292
|
+
values, frac = [], []
|
|
293
|
+
for v, m in zip(y.tolist(), missing, strict=True):
|
|
294
|
+
if m:
|
|
295
|
+
values.append(None)
|
|
296
|
+
continue
|
|
297
|
+
if isinstance(v, str) and _INT_TEXT.fullmatch(v.strip()):
|
|
298
|
+
values.append(int(v.strip()))
|
|
299
|
+
continue
|
|
300
|
+
try:
|
|
301
|
+
f = float(v)
|
|
302
|
+
except (TypeError, ValueError):
|
|
303
|
+
raise DiffError(
|
|
304
|
+
f"Cannot cast `{name}` to a number: {_show(v)} is not one."
|
|
305
|
+
) from None
|
|
306
|
+
if math.isnan(f):
|
|
307
|
+
raise DiffError(f"Cannot cast `{name}` to a number: {_show(v)} is not one.")
|
|
308
|
+
if math.isinf(f):
|
|
309
|
+
raise DiffError(f"Cannot cast `{name}` to integer: it holds infinite values.")
|
|
310
|
+
t = math.trunc(f)
|
|
311
|
+
if t != f:
|
|
312
|
+
frac.append((f, float(t)))
|
|
313
|
+
values.append(t)
|
|
314
|
+
if frac:
|
|
315
|
+
if not truncate:
|
|
316
|
+
raise DiffError(
|
|
317
|
+
f"Casting `{name}` to integer would truncate {len(frac)} value(s), "
|
|
318
|
+
f"e.g. {_show(frac[0][0])}; pass truncate=True to allow it."
|
|
319
|
+
)
|
|
320
|
+
truncated[name] = frac
|
|
321
|
+
bad = next((v for v in values if v is not None and not info.min <= v <= info.max), None)
|
|
322
|
+
if bad is not None:
|
|
323
|
+
raise DiffError(
|
|
324
|
+
f"Cannot cast `{name}` to {target}: {bad} is outside the range "
|
|
325
|
+
f"{info.min} to {info.max}."
|
|
326
|
+
)
|
|
327
|
+
out = pd.Series(pd.array(values, dtype=_nullable(target)), index=y.index)
|
|
328
|
+
if isinstance(x.dtype, pd.api.extensions.ExtensionDtype) or out.isna().any():
|
|
329
|
+
return out.astype(_nullable(x.dtype))
|
|
330
|
+
return out.astype(x.dtype)
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def _object_needs_text(y: pd.Series, x: pd.Series) -> bool:
|
|
334
|
+
"""Two ``object`` columns share a dtype but not a type: pandas 2 reads text as
|
|
335
|
+
object, and an object column of y may hold numbers. When x holds only text and y
|
|
336
|
+
does not, y is cast to text like any other column."""
|
|
337
|
+
if not pd.api.types.is_object_dtype(x):
|
|
338
|
+
return False
|
|
339
|
+
texts = all(isinstance(v, str) for v in x.dropna())
|
|
340
|
+
return texts and not all(isinstance(v, str) for v in y.dropna())
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def _cast_to(
|
|
344
|
+
y: pd.Series, x: pd.Series, name: str, truncate: bool, truncated: dict, alerts: list
|
|
345
|
+
) -> pd.Series:
|
|
346
|
+
"""Cast column ``y`` to the dtype of the benchmark column ``x``."""
|
|
347
|
+
if y.dtype == x.dtype and not _object_needs_text(y, x):
|
|
348
|
+
return y
|
|
349
|
+
|
|
350
|
+
# A column with no values (how a reader types an empty column) casts to anything.
|
|
351
|
+
if (
|
|
352
|
+
len(y)
|
|
353
|
+
and y.isna().all()
|
|
354
|
+
and (_is_int(x) or _is_float(x) or _is_bool(x) or _is_datetime(x) or _is_text(x))
|
|
355
|
+
):
|
|
356
|
+
target = "Int64" if _is_int(x) else "boolean" if _is_bool(x) else x.dtype
|
|
357
|
+
missing = np.nan if pd.api.types.is_object_dtype(x) or _is_float(x) else pd.NA
|
|
358
|
+
return pd.Series([missing] * len(y), index=y.index, dtype=object).astype(target)
|
|
359
|
+
|
|
360
|
+
if _is_int(x):
|
|
361
|
+
return _cast_to_int(y, x, name, truncate, truncated)
|
|
362
|
+
|
|
363
|
+
if _is_float(x):
|
|
364
|
+
if _is_bool(y):
|
|
365
|
+
num = y.astype(float)
|
|
366
|
+
else:
|
|
367
|
+
num = pd.to_numeric(y, errors="coerce")
|
|
368
|
+
if (y.notna() & num.isna()).any():
|
|
369
|
+
v = _first_bad(y, num)
|
|
370
|
+
raise DiffError(f"Cannot cast `{name}` to a number: {_show(v)} is not one.")
|
|
371
|
+
return num.astype(float).astype(x.dtype)
|
|
372
|
+
|
|
373
|
+
if _is_bool(x):
|
|
374
|
+
if _is_int(y) or _is_float(y):
|
|
375
|
+
ok = y.isna() | y.isin([0, 1])
|
|
376
|
+
if not ok.all():
|
|
377
|
+
raise DiffError(
|
|
378
|
+
f"Cannot cast `{name}` to bool: {_show(y[~ok].iloc[0])} is not 0 or 1."
|
|
379
|
+
)
|
|
380
|
+
out = y.map(lambda v: v if pd.isna(v) else bool(v))
|
|
381
|
+
else:
|
|
382
|
+
low = y.map(lambda v: v if pd.isna(v) else str(v).strip().lower())
|
|
383
|
+
out = low.map(lambda v: True if v in _TRUE else False if v in _FALSE else None)
|
|
384
|
+
if (y.notna() & out.isna()).any():
|
|
385
|
+
v = _first_bad(y, out)
|
|
386
|
+
raise DiffError(f"Cannot cast `{name}` to bool: {_show(v)} is not true or false.")
|
|
387
|
+
return out.astype("boolean" if out.isna().any() else x.dtype)
|
|
388
|
+
|
|
389
|
+
if _is_datetime(x):
|
|
390
|
+
# pandas reads a number as nanoseconds since 1970, which is never what a
|
|
391
|
+
# number in a date column means, so numbers are refused rather than guessed.
|
|
392
|
+
if _is_int(y) or _is_float(y) or _is_bool(y):
|
|
393
|
+
raise DiffError(
|
|
394
|
+
f"Cannot cast `{name}` to a date-time: it holds numbers ({y.dtype}), "
|
|
395
|
+
"not dates; convert it first."
|
|
396
|
+
)
|
|
397
|
+
try:
|
|
398
|
+
with warnings.catch_warnings():
|
|
399
|
+
# pandas 2 warns about text in several UTC offsets where pandas 3
|
|
400
|
+
# raises; both become the same DiffError.
|
|
401
|
+
warnings.simplefilter("error", FutureWarning)
|
|
402
|
+
conv = pd.to_datetime(y, errors="coerce")
|
|
403
|
+
except (TypeError, ValueError, FutureWarning) as e:
|
|
404
|
+
raise DiffError(f"Cannot cast `{name}` to a date-time: {e}") from None
|
|
405
|
+
if (y.notna() & conv.isna()).any():
|
|
406
|
+
v = _first_bad(y, conv)
|
|
407
|
+
raise DiffError(f"Cannot cast `{name}` to a date-time: {_show(v)} is not one.")
|
|
408
|
+
try:
|
|
409
|
+
return conv.astype(x.dtype)
|
|
410
|
+
except (TypeError, ValueError) as e:
|
|
411
|
+
raise DiffError(f"Cannot cast `{name}` to {x.dtype}: {e}") from e
|
|
412
|
+
|
|
413
|
+
if _is_text(x):
|
|
414
|
+
if _is_datetime(y):
|
|
415
|
+
return _stamp_to_text(y).astype(x.dtype)
|
|
416
|
+
out = y.map(lambda v: v if pd.isna(v) else _to_text(v)).astype(object)
|
|
417
|
+
if _is_int(y) or _is_float(y):
|
|
418
|
+
out = _number_as_text_like(y, out, x, name, alerts)
|
|
419
|
+
return out.astype(x.dtype)
|
|
420
|
+
|
|
421
|
+
raise DiffError(
|
|
422
|
+
f"Cannot cast `{name}` from {y.dtype} to {x.dtype}: supported target types are "
|
|
423
|
+
"integer, float, bool, string and date-time."
|
|
424
|
+
)
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
# --- measures and identity ---------------------------------------------------
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
def _select_measures(x: pd.DataFrame, measures, x_names: dict) -> list[str]:
|
|
431
|
+
if isinstance(measures, str) and measures in _MEASURE_KEYWORDS:
|
|
432
|
+
if measures == "numeric":
|
|
433
|
+
return [c for c in x.columns if _is_float(x[c])]
|
|
434
|
+
return [c for c in x.columns if _is_float(x[c]) or _is_int(x[c])]
|
|
435
|
+
given = [measures] if isinstance(measures, str) else list(measures)
|
|
436
|
+
canon = [_canon(m) for m in given]
|
|
437
|
+
missing = [g for g, c in zip(given, canon, strict=True) if c not in x.columns]
|
|
438
|
+
if missing:
|
|
439
|
+
raise DiffError(f"Measure column(s) not found in `x`: {_join(missing)}.")
|
|
440
|
+
bad = [x_names[c] for c in canon if not (_is_float(x[c]) or _is_int(x[c]))]
|
|
441
|
+
if bad:
|
|
442
|
+
raise DiffError(f"Measure column(s) are not numeric: {_join(bad)}.")
|
|
443
|
+
return list(dict.fromkeys(canon))
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
def _add_seq(df: pd.DataFrame, by: list[str], duplicates: str, label: str, alerts: list):
|
|
447
|
+
seq = df.groupby(by, dropna=False, sort=False).cumcount() + 1
|
|
448
|
+
n_dup = int((seq > 1).sum())
|
|
449
|
+
if n_dup and duplicates != "ignore":
|
|
450
|
+
if duplicates == "error":
|
|
451
|
+
raise DiffError(f"`{label}` has {n_dup} row(s) with a repeated key.")
|
|
452
|
+
alerts.append(
|
|
453
|
+
(
|
|
454
|
+
_PairingWarning,
|
|
455
|
+
f"`{label}` has {n_dup} row(s) with a repeated key; "
|
|
456
|
+
"they are paired in order of arrival.",
|
|
457
|
+
)
|
|
458
|
+
)
|
|
459
|
+
df["key_seq"] = seq.astype("int64").to_numpy()
|
|
460
|
+
return df
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
# --- values ------------------------------------------------------------------
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
def _int_diff(vx: pd.Series, vy: pd.Series):
|
|
467
|
+
"""``vy - vx`` for two integer columns, exact as an int64 where it fits, and
|
|
468
|
+
``vy == vx`` exactly; NaN where either is missing."""
|
|
469
|
+
miss = vx.isna().to_numpy() | vy.isna().to_numpy()
|
|
470
|
+
if "uint64" in str(vx.dtype).lower() or "uint64" in str(vy.dtype).lower():
|
|
471
|
+
pairs = zip(vx.tolist(), vy.tolist(), miss, strict=True)
|
|
472
|
+
exact = [0 if m else int(b) - int(a) for a, b, m in pairs]
|
|
473
|
+
diff = np.array([float(d) for d in exact])
|
|
474
|
+
equal = ~miss & np.array([d == 0 for d in exact], dtype=bool)
|
|
475
|
+
else:
|
|
476
|
+
a = vx.to_numpy(dtype="int64", na_value=0)
|
|
477
|
+
b = vy.to_numpy(dtype="int64", na_value=0)
|
|
478
|
+
with np.errstate(over="ignore"):
|
|
479
|
+
d = b - a
|
|
480
|
+
# Two's complement: b - a overflowed when b and a differ in sign and the
|
|
481
|
+
# result's sign is not b's.
|
|
482
|
+
over = ((b ^ a) & (b ^ d)) < 0
|
|
483
|
+
diff = d.astype(float)
|
|
484
|
+
diff[over] = b[over].astype(float) - a[over].astype(float)
|
|
485
|
+
equal = ~miss & ~over & (d == 0)
|
|
486
|
+
diff[miss] = np.nan
|
|
487
|
+
return diff, equal
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
def _classify(vx: pd.Series, vy: pd.Series, tolerance: float, rel_tol: float):
|
|
491
|
+
if _is_int(vx) and _is_int(vy):
|
|
492
|
+
# Compared as integers: as floats, values above 2**53 that differ by a few
|
|
493
|
+
# units are the same number.
|
|
494
|
+
both_na = (vx.isna() & vy.isna()).to_numpy()
|
|
495
|
+
diff, equal = _int_diff(vx, vy)
|
|
496
|
+
fx = vx.to_numpy(dtype=float, na_value=np.nan)
|
|
497
|
+
fy = vy.to_numpy(dtype=float, na_value=np.nan)
|
|
498
|
+
finite = ~np.isnan(diff)
|
|
499
|
+
abs_diff = np.abs(diff)
|
|
500
|
+
with np.errstate(invalid="ignore", divide="ignore"):
|
|
501
|
+
scale = np.maximum(np.abs(fx), np.abs(fy))
|
|
502
|
+
rel = np.where(scale > 0, abs_diff / np.where(scale > 0, scale, 1.0), 0.0)
|
|
503
|
+
rel_diff = np.where(equal, 0.0, np.where(finite, rel, np.nan))
|
|
504
|
+
close = finite & ((abs_diff <= tolerance) | (rel_diff <= rel_tol))
|
|
505
|
+
return (
|
|
506
|
+
np.where(equal, 0.0, diff),
|
|
507
|
+
np.where(equal, 0.0, abs_diff),
|
|
508
|
+
rel_diff,
|
|
509
|
+
both_na | equal | close,
|
|
510
|
+
)
|
|
511
|
+
vx = vx.to_numpy(dtype=float, na_value=np.nan)
|
|
512
|
+
vy = vy.to_numpy(dtype=float, na_value=np.nan)
|
|
513
|
+
with np.errstate(over="ignore", invalid="ignore", divide="ignore"):
|
|
514
|
+
equal = vx == vy # also equal infinities
|
|
515
|
+
finite = np.isfinite(vx) & np.isfinite(vy)
|
|
516
|
+
diff = np.where(equal, 0.0, vy - vx)
|
|
517
|
+
abs_diff = np.abs(diff)
|
|
518
|
+
scale = np.maximum(np.abs(vx), np.abs(vy))
|
|
519
|
+
safe = np.where(scale > 0, scale, 1.0)
|
|
520
|
+
rel = np.abs(vy / safe - vx / safe) # scaled first, so it cannot overflow
|
|
521
|
+
rel_diff = np.where(equal, 0.0, np.where(finite, rel, np.nan))
|
|
522
|
+
both_na = np.isnan(vx) & np.isnan(vy)
|
|
523
|
+
close = finite & ((abs_diff <= tolerance) | (rel_diff <= rel_tol))
|
|
524
|
+
same = both_na | equal | close
|
|
525
|
+
return diff, abs_diff, rel_diff, same
|
|
526
|
+
|
|
527
|
+
|
|
528
|
+
# --- diff_table --------------------------------------------------------------
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
def diff_table(
|
|
532
|
+
x: pd.DataFrame,
|
|
533
|
+
y: pd.DataFrame,
|
|
534
|
+
*,
|
|
535
|
+
mode: str = "benchmark",
|
|
536
|
+
by: Sequence[str] | str | None = None,
|
|
537
|
+
exclude: Sequence[str] | str | None = None,
|
|
538
|
+
row_key: str = "none",
|
|
539
|
+
measures: Sequence[str] | str = "numeric",
|
|
540
|
+
tolerance: float = 0.0,
|
|
541
|
+
rel_tol: float = 0.0,
|
|
542
|
+
duplicates: str = "disambiguate",
|
|
543
|
+
cast: bool | None = None,
|
|
544
|
+
truncate: bool = True,
|
|
545
|
+
) -> pd.DataFrame:
|
|
546
|
+
"""Compare two tables value by value.
|
|
547
|
+
|
|
548
|
+
Parameters
|
|
549
|
+
----------
|
|
550
|
+
x, y
|
|
551
|
+
The tables. In ``mode="benchmark"``, ``x`` is the benchmark: ``y`` is cast to
|
|
552
|
+
its column types and ``y``'s extra columns are dropped. In ``mode="equal"`` the
|
|
553
|
+
tables must have the same columns with the same types.
|
|
554
|
+
mode
|
|
555
|
+
``"benchmark"`` (default) or ``"equal"``.
|
|
556
|
+
by
|
|
557
|
+
Key columns that identify a row. Defaults to every column that is not a measure.
|
|
558
|
+
With ``by`` given, other non-measure columns are ignored with a note in
|
|
559
|
+
benchmark mode, and are an error in equal mode.
|
|
560
|
+
exclude
|
|
561
|
+
Columns, in either spelling, removed from both tables before anything else, in
|
|
562
|
+
either mode. They need not be in both tables and are not announced as dropped
|
|
563
|
+
or ignored. A name in neither table is a :class:`DiffWarning`; a name ``by`` or
|
|
564
|
+
``measures`` also names is an error.
|
|
565
|
+
row_key
|
|
566
|
+
``"none"`` (default), or a virtual ``row_number`` key added to the identity:
|
|
567
|
+
``"position"`` numbers rows as they come, ``"sorted"`` numbers them after
|
|
568
|
+
sorting both tables by their compared columns. Use it when every column is a
|
|
569
|
+
measure. It cannot be combined with ``by``.
|
|
570
|
+
measures
|
|
571
|
+
``"numeric"`` (float columns), ``"numeric+integer"``, or column names.
|
|
572
|
+
tolerance, rel_tol
|
|
573
|
+
For a finite pair, a value is ``same`` when ``abs_diff <= tolerance`` or
|
|
574
|
+
``rel_diff <= rel_tol``.
|
|
575
|
+
duplicates
|
|
576
|
+
``"disambiguate"`` (default) pairs repeated keys in arrival order, with a
|
|
577
|
+
warning; ``"aggregate"`` compares the multiset of values per key and measure
|
|
578
|
+
(tolerances must be 0); ``"error"`` refuses repeated keys.
|
|
579
|
+
cast
|
|
580
|
+
Whether to cast the compared columns of ``y`` to ``x``'s types. ``None``
|
|
581
|
+
(default) casts in benchmark mode; ``False`` requires the types to agree.
|
|
582
|
+
Equal mode never casts, so ``cast=True`` with it is an error. A number cast
|
|
583
|
+
to text is written with the decimals ``x`` uses, when every value of ``x``'s
|
|
584
|
+
column uses the same number of them.
|
|
585
|
+
truncate
|
|
586
|
+
Allow truncating non-integral values when casting to an integer column.
|
|
587
|
+
|
|
588
|
+
Returns
|
|
589
|
+
-------
|
|
590
|
+
pandas.DataFrame
|
|
591
|
+
Sorted by the key. With measures (``attrs["shape"] == "cells"``), one row per
|
|
592
|
+
key and measure, with ``metric``, ``row_id_x``, ``row_id_y``, ``value_x``,
|
|
593
|
+
``value_y``, ``diff``, ``abs_diff``, ``rel_diff`` and ``status``. With
|
|
594
|
+
``duplicates="aggregate"`` (``"groups"``), one row per key and measure, with
|
|
595
|
+
``n_rows_x``, ``n_rows_y``, ``values_x`` and ``values_y`` (sorted tuples). With no
|
|
596
|
+
measures (``"rows"``), one row per source row, with ``key_seq``. ``attrs``
|
|
597
|
+
also holds ``mode``, ``by``, ``measures``, ``tolerance``, ``name_map``,
|
|
598
|
+
``dropped``, ``excluded``, ``ignored`` and ``truncated``.
|
|
599
|
+
|
|
600
|
+
Raises
|
|
601
|
+
------
|
|
602
|
+
DiffError
|
|
603
|
+
When the question cannot be answered: missing columns, type differences in
|
|
604
|
+
equal mode, clashing names, no key, or no key in common.
|
|
605
|
+
|
|
606
|
+
Warns
|
|
607
|
+
-----
|
|
608
|
+
DiffWarning
|
|
609
|
+
When the answer may be wrong: repeated keys paired by arrival order.
|
|
610
|
+
DiffNote
|
|
611
|
+
For expected side effects: dropped or ignored columns, truncation, no measures.
|
|
612
|
+
"""
|
|
613
|
+
_check_frame(x, "x")
|
|
614
|
+
_check_frame(y, "y")
|
|
615
|
+
if mode not in _MODES:
|
|
616
|
+
raise DiffError(f"`mode` must be one of {_MODES}, not {mode!r}.")
|
|
617
|
+
if duplicates not in _DUPLICATES:
|
|
618
|
+
raise DiffError(f"`duplicates` must be one of {_DUPLICATES}, not {duplicates!r}.")
|
|
619
|
+
if row_key not in _ROW_KEYS:
|
|
620
|
+
raise DiffError(f"`row_key` must be one of {_ROW_KEYS}, not {row_key!r}.")
|
|
621
|
+
if cast is not None and not isinstance(cast, bool):
|
|
622
|
+
raise DiffError("`cast` must be None, True or False.")
|
|
623
|
+
if mode == "equal" and cast:
|
|
624
|
+
raise DiffError('mode = "equal" never casts: neither table is the reference for the other.')
|
|
625
|
+
if cast is None:
|
|
626
|
+
cast = mode == "benchmark"
|
|
627
|
+
if row_key != "none" and by is not None:
|
|
628
|
+
raise DiffError("`row_key` adds a key of its own, so it cannot be combined with `by`.")
|
|
629
|
+
_check_threshold(tolerance, "tolerance")
|
|
630
|
+
_check_threshold(rel_tol, "rel_tol")
|
|
631
|
+
_check_columns_arg(measures, "measures", allow_empty=True)
|
|
632
|
+
if by is not None:
|
|
633
|
+
_check_columns_arg(by, "by", allow_empty=False)
|
|
634
|
+
if exclude is not None:
|
|
635
|
+
_check_columns_arg(exclude, "exclude", allow_empty=True)
|
|
636
|
+
if duplicates == "aggregate" and (tolerance or rel_tol):
|
|
637
|
+
raise DiffError(
|
|
638
|
+
'duplicates = "aggregate" compares multisets exactly; `tolerance` and `rel_tol` '
|
|
639
|
+
"must be 0."
|
|
640
|
+
)
|
|
641
|
+
|
|
642
|
+
alerts: list[tuple[type[Warning], str]] = []
|
|
643
|
+
try:
|
|
644
|
+
out = _diff_table(
|
|
645
|
+
x,
|
|
646
|
+
y,
|
|
647
|
+
mode,
|
|
648
|
+
by,
|
|
649
|
+
exclude,
|
|
650
|
+
row_key,
|
|
651
|
+
measures,
|
|
652
|
+
tolerance,
|
|
653
|
+
rel_tol,
|
|
654
|
+
duplicates,
|
|
655
|
+
cast,
|
|
656
|
+
truncate,
|
|
657
|
+
alerts,
|
|
658
|
+
)
|
|
659
|
+
finally:
|
|
660
|
+
for category, message in alerts:
|
|
661
|
+
warnings.warn(message, category, stacklevel=2)
|
|
662
|
+
return out
|
|
663
|
+
|
|
664
|
+
|
|
665
|
+
def _exclude(x, y, exclude, by, measures, alerts):
|
|
666
|
+
"""Remove the ``exclude`` columns from both tables, before anything else looks at
|
|
667
|
+
them: they are neither required, dropped with a note, nor checked for clashes."""
|
|
668
|
+
given = [exclude] if isinstance(exclude, str) else list(exclude)
|
|
669
|
+
canon = [_canon(e) for e in given]
|
|
670
|
+
cols_x = [_canon(c) for c in x.columns]
|
|
671
|
+
cols_y = [_canon(c) for c in y.columns]
|
|
672
|
+
# A warning, not an error: one list of technical columns (load stamps, run ids)
|
|
673
|
+
# serves many tables, not all of which have each one. Still a warning, since a
|
|
674
|
+
# typo leaves a column compared.
|
|
675
|
+
unknown = [g for g, c in zip(given, canon, strict=True) if c not in cols_x + cols_y]
|
|
676
|
+
if unknown:
|
|
677
|
+
alerts.append(
|
|
678
|
+
(
|
|
679
|
+
DiffWarning,
|
|
680
|
+
f"`exclude` names column(s) in neither table, so nothing was excluded for "
|
|
681
|
+
f"them: {_join(unknown)}.",
|
|
682
|
+
)
|
|
683
|
+
)
|
|
684
|
+
named = [] if by is None else [by] if isinstance(by, str) else list(by)
|
|
685
|
+
if not (isinstance(measures, str) and measures in _MEASURE_KEYWORDS):
|
|
686
|
+
named += [measures] if isinstance(measures, str) else list(measures)
|
|
687
|
+
named_canon = {_canon(n) for n in named}
|
|
688
|
+
both = [g for g, c in zip(given, canon, strict=True) if c in named_canon]
|
|
689
|
+
if both:
|
|
690
|
+
raise DiffError(
|
|
691
|
+
f"`exclude` names column(s) that `by` or `measures` also name: {_join(both)}."
|
|
692
|
+
)
|
|
693
|
+
drop = set(canon)
|
|
694
|
+
excluded = list(
|
|
695
|
+
dict.fromkeys(
|
|
696
|
+
[o for o, c in zip(x.columns, cols_x, strict=True) if c in drop]
|
|
697
|
+
+ [o for o, c in zip(y.columns, cols_y, strict=True) if c in drop]
|
|
698
|
+
)
|
|
699
|
+
)
|
|
700
|
+
x = x.iloc[:, [i for i, c in enumerate(cols_x) if c not in drop]]
|
|
701
|
+
y = y.iloc[:, [i for i, c in enumerate(cols_y) if c not in drop]]
|
|
702
|
+
return x, y, excluded
|
|
703
|
+
|
|
704
|
+
|
|
705
|
+
def _diff_table(
|
|
706
|
+
x,
|
|
707
|
+
y,
|
|
708
|
+
mode,
|
|
709
|
+
by,
|
|
710
|
+
exclude,
|
|
711
|
+
row_key,
|
|
712
|
+
measures,
|
|
713
|
+
tolerance,
|
|
714
|
+
rel_tol,
|
|
715
|
+
duplicates,
|
|
716
|
+
cast,
|
|
717
|
+
truncate,
|
|
718
|
+
alerts,
|
|
719
|
+
):
|
|
720
|
+
excluded: list = []
|
|
721
|
+
if exclude is not None:
|
|
722
|
+
x, y, excluded = _exclude(x, y, exclude, by, measures, alerts)
|
|
723
|
+
|
|
724
|
+
# Names: x is checked whole; y only for the columns that are compared.
|
|
725
|
+
x_canon = [_canon(c) for c in x.columns]
|
|
726
|
+
_check_names(list(x.columns), x_canon, "`x`")
|
|
727
|
+
y_canon = [_canon(c) for c in y.columns]
|
|
728
|
+
if mode == "equal":
|
|
729
|
+
_check_names(list(y.columns), y_canon, "`y`")
|
|
730
|
+
else:
|
|
731
|
+
used = [(o, c) for o, c in zip(y.columns, y_canon, strict=True) if c in set(x_canon)]
|
|
732
|
+
_check_names([o for o, _ in used], [c for _, c in used], "`y`")
|
|
733
|
+
x_names = dict(zip(x_canon, x.columns, strict=True))
|
|
734
|
+
y_names = {c: o for o, c in zip(y.columns, y_canon, strict=True) if c}
|
|
735
|
+
# Every column of y that x lacks, as written: also those whose names normalize
|
|
736
|
+
# to nothing or clash, which y_names cannot hold.
|
|
737
|
+
y_extra = [o for o, c in zip(y.columns, y_canon, strict=True) if c not in set(x_canon)]
|
|
738
|
+
name_map = {c: {"x": o, "y": y_names.get(c)} for c, o in x_names.items()}
|
|
739
|
+
name_map.update({c: {"x": None, "y": o} for c, o in y_names.items() if c not in x_names})
|
|
740
|
+
|
|
741
|
+
x = x.copy()
|
|
742
|
+
x.columns = x_canon
|
|
743
|
+
x = x.reset_index(drop=True)
|
|
744
|
+
y_keep = [i for i, c in enumerate(y_canon) if c in y_names and y_names[c] == y.columns[i]]
|
|
745
|
+
y = y.iloc[:, y_keep].copy()
|
|
746
|
+
y.columns = [y_canon[i] for i in y_keep]
|
|
747
|
+
y = y.reset_index(drop=True)
|
|
748
|
+
|
|
749
|
+
only_x = [c for c in x.columns if c not in y.columns]
|
|
750
|
+
only_y = [c for c in y.columns if c not in x.columns]
|
|
751
|
+
dropped: list = []
|
|
752
|
+
|
|
753
|
+
if mode == "equal":
|
|
754
|
+
if only_x or only_y:
|
|
755
|
+
raise DiffError(
|
|
756
|
+
'mode = "equal" needs the same columns in both tables.\n'
|
|
757
|
+
f" only in `x`: {_join(x_names[c] for c in only_x) or '<none>'}\n"
|
|
758
|
+
f" only in `y`: {_join(y_names[c] for c in only_y) or '<none>'}"
|
|
759
|
+
)
|
|
760
|
+
else:
|
|
761
|
+
if only_x:
|
|
762
|
+
missing = _join(x_names[c] for c in only_x)
|
|
763
|
+
raise DiffError(f"`y` lacks column(s) of the benchmark `x`: {missing}.")
|
|
764
|
+
dropped = y_extra
|
|
765
|
+
if dropped:
|
|
766
|
+
alerts.append(
|
|
767
|
+
(
|
|
768
|
+
DiffNote,
|
|
769
|
+
f"Dropped column(s) of `y` that the benchmark `x` does not have: "
|
|
770
|
+
f"{_join(dropped)}.",
|
|
771
|
+
)
|
|
772
|
+
)
|
|
773
|
+
y = y[list(x.columns)]
|
|
774
|
+
|
|
775
|
+
# Roles are decided on x, whose types a cast never changes, and before casting,
|
|
776
|
+
# so a column the call ignores can neither refuse the comparison nor be truncated.
|
|
777
|
+
rest: list = []
|
|
778
|
+
if by is not None:
|
|
779
|
+
given = [by] if isinstance(by, str) else list(by)
|
|
780
|
+
by = list(dict.fromkeys(_canon(b) for b in given))
|
|
781
|
+
missing = [g for g in given if _canon(g) not in x.columns]
|
|
782
|
+
if missing:
|
|
783
|
+
raise DiffError(f"Key column(s) not found in `x`: {_join(missing)}.")
|
|
784
|
+
if not (isinstance(measures, str) and measures in _MEASURE_KEYWORDS):
|
|
785
|
+
named = [measures] if isinstance(measures, str) else list(measures)
|
|
786
|
+
both = [m for m in named if _canon(m) in by]
|
|
787
|
+
if both:
|
|
788
|
+
raise DiffError(f"`by` and `measures` both name: {_join(both)}.")
|
|
789
|
+
meas = [m for m in _select_measures(x, measures, x_names) if m not in by]
|
|
790
|
+
rest = [c for c in x.columns if c not in by and c not in meas]
|
|
791
|
+
if rest:
|
|
792
|
+
if mode == "equal":
|
|
793
|
+
raise DiffError(
|
|
794
|
+
f"Column(s) that are neither keys nor measures: {_join(x_names[c] for c in rest)}. "
|
|
795
|
+
'mode = "equal" compares every column; add them to `by` or `measures`.'
|
|
796
|
+
)
|
|
797
|
+
alerts.append(
|
|
798
|
+
(
|
|
799
|
+
DiffNote,
|
|
800
|
+
f"Ignored column(s) that are neither keys nor measures: "
|
|
801
|
+
f"{_join(x_names[c] for c in rest)}.",
|
|
802
|
+
)
|
|
803
|
+
)
|
|
804
|
+
else:
|
|
805
|
+
meas = _select_measures(x, measures, x_names)
|
|
806
|
+
by = [c for c in x.columns if c not in meas]
|
|
807
|
+
compared = by + meas
|
|
808
|
+
|
|
809
|
+
# Types, of the compared columns only.
|
|
810
|
+
truncated: dict = {}
|
|
811
|
+
if cast:
|
|
812
|
+
y = y.assign(
|
|
813
|
+
**{c: _cast_to(y[c], x[c], y_names[c], truncate, truncated, alerts) for c in compared}
|
|
814
|
+
)
|
|
815
|
+
if truncated:
|
|
816
|
+
lines = [f" {c}: {len(v)} ({v[0][0]:g} -> {v[0][1]:g})" for c, v in truncated.items()]
|
|
817
|
+
n = sum(len(v) for v in truncated.values())
|
|
818
|
+
alerts.append(
|
|
819
|
+
(
|
|
820
|
+
DiffNote,
|
|
821
|
+
f"Truncated {n} value(s) while casting to the reference types:\n"
|
|
822
|
+
+ "\n".join(lines),
|
|
823
|
+
)
|
|
824
|
+
)
|
|
825
|
+
else:
|
|
826
|
+
differ = [
|
|
827
|
+
f"{x_names[c]} ({x[c].dtype} vs {y[c].dtype})"
|
|
828
|
+
for c in compared
|
|
829
|
+
if x[c].dtype != y[c].dtype
|
|
830
|
+
]
|
|
831
|
+
if differ:
|
|
832
|
+
lead = (
|
|
833
|
+
'mode = "equal" needs the same column types'
|
|
834
|
+
if mode == "equal"
|
|
835
|
+
else ("cast=False, but x and y disagree on the type of")
|
|
836
|
+
)
|
|
837
|
+
raise DiffError(f"{lead}: " + ", ".join(differ))
|
|
838
|
+
|
|
839
|
+
if row_key != "none":
|
|
840
|
+
x = x.assign(row_number=_row_numbers(x, row_key, compared))
|
|
841
|
+
y = y.assign(row_number=_row_numbers(y, row_key, compared))
|
|
842
|
+
x_names["row_number"] = "row_number"
|
|
843
|
+
by = [*by, "row_number"]
|
|
844
|
+
if not by:
|
|
845
|
+
raise DiffError(
|
|
846
|
+
"Every column is a measure, so rows have no key; pass `by`, fewer `measures`, "
|
|
847
|
+
'or `row_key="position"` or `"sorted"`.'
|
|
848
|
+
)
|
|
849
|
+
|
|
850
|
+
if not meas:
|
|
851
|
+
alerts.append(
|
|
852
|
+
(
|
|
853
|
+
DiffNote,
|
|
854
|
+
"No measure columns, so rows are compared as wholes: "
|
|
855
|
+
"do the two tables have the same rows?",
|
|
856
|
+
)
|
|
857
|
+
)
|
|
858
|
+
|
|
859
|
+
x_names_y = {c: y_names.get(c) for c in x.columns}
|
|
860
|
+
x, y = x[by + meas].copy(), y[by + meas].copy()
|
|
861
|
+
paired = False
|
|
862
|
+
if duplicates == "aggregate" and meas:
|
|
863
|
+
out = _aggregate(x, y, by, meas, x_names)
|
|
864
|
+
sort, shape = [*by, "metric"], "groups"
|
|
865
|
+
else:
|
|
866
|
+
# With no measures a row is its own key, so repeats are copies, not clashes,
|
|
867
|
+
# unless the caller asked for repeats to be refused.
|
|
868
|
+
dup_rule = duplicates if meas or duplicates == "error" else "ignore"
|
|
869
|
+
out, sort, shape = _pairwise(x, y, by, meas, dup_rule, tolerance, rel_tol, x_names, alerts)
|
|
870
|
+
# The cells shape drops key_seq, so whether repeated keys were paired is kept here.
|
|
871
|
+
paired = bool(x.duplicated(by).any() or y.duplicated(by).any())
|
|
872
|
+
|
|
873
|
+
out = out.sort_values(sort, kind="stable", ignore_index=True, na_position="last")
|
|
874
|
+
out.attrs.update(
|
|
875
|
+
{
|
|
876
|
+
"shape": shape,
|
|
877
|
+
"mode": mode,
|
|
878
|
+
"by": by,
|
|
879
|
+
"measures": meas,
|
|
880
|
+
"tolerance": {"abs": float(tolerance), "rel": float(rel_tol)},
|
|
881
|
+
"name_map": name_map,
|
|
882
|
+
"dropped": dropped,
|
|
883
|
+
"excluded": excluded,
|
|
884
|
+
"ignored": [x_names[c] for c in rest],
|
|
885
|
+
"paired": paired,
|
|
886
|
+
"truncated": {
|
|
887
|
+
c: truncated[x_names_y[c]] for c in by + meas if x_names_y.get(c) in truncated
|
|
888
|
+
},
|
|
889
|
+
}
|
|
890
|
+
)
|
|
891
|
+
return out
|
|
892
|
+
|
|
893
|
+
|
|
894
|
+
def _row_numbers(df: pd.DataFrame, row_key: str, compared: list[str]) -> np.ndarray:
|
|
895
|
+
n = len(df)
|
|
896
|
+
if row_key == "position" or n == 0:
|
|
897
|
+
return np.arange(1, n + 1)
|
|
898
|
+
order = df.sort_values(compared, kind="stable", na_position="last").index.to_numpy()
|
|
899
|
+
numbers = np.empty(n, dtype="int64")
|
|
900
|
+
numbers[order] = np.arange(1, n + 1)
|
|
901
|
+
return numbers
|
|
902
|
+
|
|
903
|
+
|
|
904
|
+
def _no_overlap(x: pd.DataFrame, y: pd.DataFrame, x_names: dict, by: list[str]) -> DiffError:
|
|
905
|
+
"""The error for tables that share no key, naming the key columns that alone share
|
|
906
|
+
no value: most often one that labels the table (a run id, a source) and belongs in
|
|
907
|
+
``exclude``."""
|
|
908
|
+
msg = (
|
|
909
|
+
f"No row of `x` has a key in common with `y` (key: {_join(x_names[c] for c in by)}); "
|
|
910
|
+
"check `by`, and the format of the key values"
|
|
911
|
+
)
|
|
912
|
+
apart, constant = [], []
|
|
913
|
+
many = max(len(x), len(y)) > 1
|
|
914
|
+
for c in by:
|
|
915
|
+
ux, uy = pd.Series(pd.unique(x[c])), pd.Series(pd.unique(y[c]))
|
|
916
|
+
if not ux.isin(uy).any():
|
|
917
|
+
apart.append(f"{x_names[c]} (x: {_examples(ux)}; y: {_examples(uy)})")
|
|
918
|
+
if many and len(ux) == 1 and len(uy) == 1:
|
|
919
|
+
constant.append(x_names[c])
|
|
920
|
+
if apart:
|
|
921
|
+
msg += f"; key column(s) that share no value between the tables: {', '.join(apart)}"
|
|
922
|
+
# Offered only when a key is left once they are excluded.
|
|
923
|
+
if constant and len(constant) < len(by):
|
|
924
|
+
each = "each hold" if len(constant) > 1 else "holds"
|
|
925
|
+
msg += (
|
|
926
|
+
f"; {_join(constant)} {each} one value in each table, so "
|
|
927
|
+
f"{'they label' if len(constant) > 1 else 'it labels'} the table rather than its "
|
|
928
|
+
f"rows: leave {'them' if len(constant) > 1 else 'it'} out with exclude={constant!r}"
|
|
929
|
+
)
|
|
930
|
+
err = DiffError(msg + ".")
|
|
931
|
+
err.disjoint = True # assert_table_equal() reports it as a failure
|
|
932
|
+
return err
|
|
933
|
+
|
|
934
|
+
|
|
935
|
+
def _pairwise(x, y, by, meas, dup_rule, tolerance, rel_tol, x_names, alerts):
|
|
936
|
+
# An outer merge turns an integer column with a missing partner into float64;
|
|
937
|
+
# nullable integers keep the values exact for _classify().
|
|
938
|
+
for c in meas:
|
|
939
|
+
if _is_int(x[c]) and _is_int(y[c]):
|
|
940
|
+
x[c] = x[c].astype(_nullable(x[c].dtype))
|
|
941
|
+
y[c] = y[c].astype(_nullable(y[c].dtype))
|
|
942
|
+
x = _add_seq(x, by, dup_rule, "x", alerts)
|
|
943
|
+
y = _add_seq(y, by, dup_rule, "y", alerts)
|
|
944
|
+
keys = [*by, "key_seq"]
|
|
945
|
+
x["row_id_x"] = np.arange(1, len(x) + 1)
|
|
946
|
+
y["row_id_y"] = np.arange(1, len(y) + 1)
|
|
947
|
+
|
|
948
|
+
m = x.merge(y, on=keys, how="outer", suffixes=("_x", "_y"), sort=False)
|
|
949
|
+
matched = m["row_id_x"].notna() & m["row_id_y"].notna()
|
|
950
|
+
if len(x) and len(y) and not matched.any():
|
|
951
|
+
raise _no_overlap(x, y, x_names, by)
|
|
952
|
+
for c in ("row_id_x", "row_id_y"):
|
|
953
|
+
m[c] = m[c].astype("Int64")
|
|
954
|
+
only_in_y, only_in_x = m["row_id_x"].isna().to_numpy(), m["row_id_y"].isna().to_numpy()
|
|
955
|
+
|
|
956
|
+
if not meas:
|
|
957
|
+
status = np.select([only_in_y, only_in_x], ["only_y", "only_x"], "same")
|
|
958
|
+
out = m[[*keys, "row_id_x", "row_id_y"]].copy()
|
|
959
|
+
out["status"] = pd.Categorical(status, categories=STATUSES)
|
|
960
|
+
return out, keys, "rows"
|
|
961
|
+
|
|
962
|
+
parts = []
|
|
963
|
+
for metric in meas:
|
|
964
|
+
p = m[[*keys, "row_id_x", "row_id_y"]].copy()
|
|
965
|
+
p.insert(len(keys), "metric", metric)
|
|
966
|
+
vx, vy = m[f"{metric}_x"], m[f"{metric}_y"]
|
|
967
|
+
diff, abs_diff, rel_diff, same = _classify(vx, vy, tolerance, rel_tol)
|
|
968
|
+
p["value_x"] = vx.to_numpy(dtype=float, na_value=np.nan)
|
|
969
|
+
p["value_y"] = vy.to_numpy(dtype=float, na_value=np.nan)
|
|
970
|
+
p["diff"], p["abs_diff"], p["rel_diff"] = diff, abs_diff, rel_diff
|
|
971
|
+
status = np.select([only_in_y, only_in_x, same], ["only_y", "only_x", "same"], "changed")
|
|
972
|
+
p["status"] = pd.Categorical(status, categories=STATUSES)
|
|
973
|
+
parts.append(p)
|
|
974
|
+
out = pd.concat(parts, ignore_index=True).drop(columns="key_seq")
|
|
975
|
+
return out, [*by, "metric"], "cells"
|
|
976
|
+
|
|
977
|
+
|
|
978
|
+
def _multiset(values: pd.Series) -> tuple:
|
|
979
|
+
"""The values of one group as a sorted tuple; missing values last, as NaN.
|
|
980
|
+
|
|
981
|
+
Integers stay Python ints, so values above 2**53 that differ stay different.
|
|
982
|
+
"""
|
|
983
|
+
if _is_int(values):
|
|
984
|
+
present = sorted(int(v) for v in values.dropna().tolist())
|
|
985
|
+
return (*present, *([math.nan] * int(values.isna().sum())))
|
|
986
|
+
v = values.to_numpy(dtype=float, na_value=np.nan)
|
|
987
|
+
v = np.where(v == 0, 0.0, v) # -0.0 and 0.0 are the same value
|
|
988
|
+
return tuple(np.sort(v).tolist()) # np.sort puts NaN last
|
|
989
|
+
|
|
990
|
+
|
|
991
|
+
def _same_multiset(a: tuple, b: tuple) -> bool:
|
|
992
|
+
if len(a) != len(b):
|
|
993
|
+
return False
|
|
994
|
+
return all(u == v or (u != u and v != v) for u, v in zip(a, b, strict=True))
|
|
995
|
+
|
|
996
|
+
|
|
997
|
+
def _aggregate(x, y, by, meas, x_names):
|
|
998
|
+
def groups(df, side):
|
|
999
|
+
g = df.groupby(by, dropna=False, sort=False)
|
|
1000
|
+
out = g[meas].agg(_multiset)
|
|
1001
|
+
out.columns = [f"{m}_{side}" for m in meas]
|
|
1002
|
+
out.insert(0, f"n_rows_{side}", g.size())
|
|
1003
|
+
return out.reset_index()
|
|
1004
|
+
|
|
1005
|
+
gx, gy = groups(x, "x"), groups(y, "y")
|
|
1006
|
+
m = gx.merge(gy, on=by, how="outer", sort=False)
|
|
1007
|
+
in_x, in_y = m["n_rows_x"].notna().to_numpy(), m["n_rows_y"].notna().to_numpy()
|
|
1008
|
+
if len(x) and len(y) and not (in_x & in_y).any():
|
|
1009
|
+
raise _no_overlap(x, y, x_names, by)
|
|
1010
|
+
for c in ("n_rows_x", "n_rows_y"):
|
|
1011
|
+
m[c] = m[c].astype("Int64")
|
|
1012
|
+
|
|
1013
|
+
parts = []
|
|
1014
|
+
for metric in meas:
|
|
1015
|
+
p = m[[*by, "n_rows_x", "n_rows_y"]].copy()
|
|
1016
|
+
p.insert(len(by), "metric", metric)
|
|
1017
|
+
vx, vy = m[f"{metric}_x"], m[f"{metric}_y"]
|
|
1018
|
+
p["values_x"], p["values_y"] = vx.to_numpy(), vy.to_numpy()
|
|
1019
|
+
same = np.array(
|
|
1020
|
+
[a == a and b == b and _same_multiset(a, b) for a, b in zip(vx, vy, strict=True)],
|
|
1021
|
+
dtype=bool,
|
|
1022
|
+
)
|
|
1023
|
+
status = np.select([~in_x, ~in_y, same], ["only_y", "only_x", "same"], "changed")
|
|
1024
|
+
p["status"] = pd.Categorical(status, categories=STATUSES)
|
|
1025
|
+
parts.append(p)
|
|
1026
|
+
return pd.concat(parts, ignore_index=True)
|
|
1027
|
+
|
|
1028
|
+
|
|
1029
|
+
def diff_summary(d: pd.DataFrame) -> pd.DataFrame:
|
|
1030
|
+
"""Condense a :func:`diff_table` result per measure."""
|
|
1031
|
+
_check_result(d)
|
|
1032
|
+
groups = d["metric"] if "metric" in d else pd.Series("<rows>", index=d.index, name="metric")
|
|
1033
|
+
counts = pd.crosstab(groups, d["status"]).reindex(columns=STATUSES, fill_value=0)
|
|
1034
|
+
counts.columns = [f"n_{s}" for s in STATUSES]
|
|
1035
|
+
counts.insert(0, "n", counts.sum(axis=1))
|
|
1036
|
+
if "abs_diff" in d:
|
|
1037
|
+
g = d.groupby("metric")
|
|
1038
|
+
stats = pd.DataFrame(
|
|
1039
|
+
{
|
|
1040
|
+
"max_abs_diff": g["abs_diff"].max(),
|
|
1041
|
+
"mean_abs_diff": g["abs_diff"].mean(),
|
|
1042
|
+
"rmse": g["abs_diff"].apply(lambda s: float(np.sqrt((s.dropna() ** 2).mean()))),
|
|
1043
|
+
"max_rel_diff": g["rel_diff"].max(),
|
|
1044
|
+
}
|
|
1045
|
+
)
|
|
1046
|
+
counts = counts.join(stats)
|
|
1047
|
+
counts.columns.name = None
|
|
1048
|
+
counts.index.name = "metric"
|
|
1049
|
+
return counts.reset_index()
|
|
1050
|
+
|
|
1051
|
+
|
|
1052
|
+
def _result_ids(d: pd.DataFrame) -> list[str]:
|
|
1053
|
+
"""The key columns of a diff_table() result. User columns are normalized to
|
|
1054
|
+
upper case; internal ones hold a lowercase letter, and ``row_number`` is the one
|
|
1055
|
+
internal key. Read off the names, so a filtered or re-sorted result still works."""
|
|
1056
|
+
return [c for c in d.columns if not re.search("[a-z]", str(c)) or c == "row_number"]
|
|
1057
|
+
|
|
1058
|
+
|
|
1059
|
+
def _result_shape(d: pd.DataFrame) -> str:
|
|
1060
|
+
"""The shape of a diff_table() result, from ``attrs`` or, when a pandas operation
|
|
1061
|
+
dropped them, from its columns."""
|
|
1062
|
+
shape = d.attrs.get("shape")
|
|
1063
|
+
if shape:
|
|
1064
|
+
return shape
|
|
1065
|
+
if "n_rows_x" in d:
|
|
1066
|
+
return "groups"
|
|
1067
|
+
return "cells" if "abs_diff" in d else "rows"
|
|
1068
|
+
|
|
1069
|
+
|
|
1070
|
+
def is_matching(
|
|
1071
|
+
d: pd.DataFrame, *, tolerance: float | None = None, rel_tol: float | None = None
|
|
1072
|
+
) -> bool:
|
|
1073
|
+
"""``True`` when every row of a :func:`diff_table` result is ``same``: no value
|
|
1074
|
+
changed and no row is found in only one table. Two empty tables match.
|
|
1075
|
+
|
|
1076
|
+
With ``tolerance`` or ``rel_tol`` the values are judged again at those thresholds,
|
|
1077
|
+
looser or stricter, from the stored ``abs_diff`` and ``rel_diff``, by
|
|
1078
|
+
:func:`diff_table`'s rule: equal or both-missing values always match, and the
|
|
1079
|
+
tolerances apply to finite pairs only. When only one is given, the other is 0.
|
|
1080
|
+
A row found in only one table never matches.
|
|
1081
|
+
"""
|
|
1082
|
+
_check_result(d)
|
|
1083
|
+
_check_threshold(tolerance, "tolerance", allow_none=True)
|
|
1084
|
+
_check_threshold(rel_tol, "rel_tol", allow_none=True)
|
|
1085
|
+
status = d["status"].astype(str).to_numpy()
|
|
1086
|
+
if tolerance is None and rel_tol is None:
|
|
1087
|
+
return bool((status == "same").all())
|
|
1088
|
+
tolerance = 0.0 if tolerance is None else float(tolerance)
|
|
1089
|
+
rel_tol = 0.0 if rel_tol is None else float(rel_tol)
|
|
1090
|
+
|
|
1091
|
+
shape = _result_shape(d)
|
|
1092
|
+
# The multisets of "aggregate" are compared exactly and hold no difference to
|
|
1093
|
+
# re-judge, as diff_table() refuses a tolerance with them.
|
|
1094
|
+
if shape == "groups" and (tolerance > 0 or rel_tol > 0):
|
|
1095
|
+
raise DiffError(
|
|
1096
|
+
'A result of duplicates = "aggregate" compares values exactly; '
|
|
1097
|
+
"`tolerance` and `rel_tol` must be 0."
|
|
1098
|
+
)
|
|
1099
|
+
if shape != "cells":
|
|
1100
|
+
return bool((status == "same").all())
|
|
1101
|
+
|
|
1102
|
+
joined = np.isin(status, ["same", "changed"])
|
|
1103
|
+
# rel_diff is defined where both values are finite, which is where the
|
|
1104
|
+
# tolerances apply. Elsewhere a value matched by equality or by being missing on
|
|
1105
|
+
# both sides, which no threshold changes.
|
|
1106
|
+
abs_diff = d["abs_diff"].to_numpy(dtype=float, na_value=np.nan)
|
|
1107
|
+
rel_diff = d["rel_diff"].to_numpy(dtype=float, na_value=np.nan)
|
|
1108
|
+
finite = ~np.isnan(rel_diff)
|
|
1109
|
+
with np.errstate(invalid="ignore"):
|
|
1110
|
+
close = (abs_diff <= tolerance) | (rel_diff <= rel_tol)
|
|
1111
|
+
same = joined & np.where(finite, close, status == "same")
|
|
1112
|
+
return bool(same.all())
|
|
1113
|
+
|
|
1114
|
+
|
|
1115
|
+
# --- diff_explain -----------------------------------------------------------------
|
|
1116
|
+
|
|
1117
|
+
_AMONG = ("all", "joined")
|
|
1118
|
+
_OVERVIEW = ["column", "n_values", "same_only", "differ_only", "mixed", "separates", "identifies"]
|
|
1119
|
+
|
|
1120
|
+
|
|
1121
|
+
def diff_explain(d: pd.DataFrame, column: str | None = None, *, among: str = "all") -> pd.DataFrame:
|
|
1122
|
+
"""Which values of the key columns separate the rows that match from those that
|
|
1123
|
+
do not.
|
|
1124
|
+
|
|
1125
|
+
With no ``column``, one row per key column (and per ``metric`` when the result
|
|
1126
|
+
has measures): ``column``, ``n_values``, and how many of its values are
|
|
1127
|
+
``same_only`` (every record holding it matches), ``differ_only`` (none does) or
|
|
1128
|
+
``mixed``. A column ``separates`` when no value is mixed; one with a different
|
|
1129
|
+
value on every record ``identifies`` and is ranked last. The others come first,
|
|
1130
|
+
fewest mixed values first, then fewest values.
|
|
1131
|
+
|
|
1132
|
+
With a ``column`` (a key column in either spelling, or ``"metric"``), one row per
|
|
1133
|
+
value of it: ``n``, ``n_same``, ``n_changed``, ``n_only_x``, ``n_only_y`` and
|
|
1134
|
+
``class``, most differences first.
|
|
1135
|
+
|
|
1136
|
+
For a key column the unit is the row, which matches only when every one of its
|
|
1137
|
+
measures does; with ``duplicates="aggregate"`` it is the key; for ``metric`` it
|
|
1138
|
+
is the value. ``among="joined"`` counts only rows found in both tables, so the
|
|
1139
|
+
explanation is of the changed values alone.
|
|
1140
|
+
"""
|
|
1141
|
+
_check_result(d)
|
|
1142
|
+
if among not in _AMONG:
|
|
1143
|
+
raise DiffError(f"`among` must be one of {_AMONG}, not {among!r}.")
|
|
1144
|
+
ids = _result_ids(d)
|
|
1145
|
+
has_metric = "metric" in d
|
|
1146
|
+
candidates = ids + (["metric"] if has_metric else [])
|
|
1147
|
+
|
|
1148
|
+
if column is not None:
|
|
1149
|
+
if not isinstance(column, str):
|
|
1150
|
+
raise DiffError("`column` must be None or a single column name.")
|
|
1151
|
+
given = column
|
|
1152
|
+
column = column if column in candidates else _canon(column)
|
|
1153
|
+
if column not in candidates:
|
|
1154
|
+
raise DiffError(
|
|
1155
|
+
"`column` must be a key column of the result"
|
|
1156
|
+
+ (' or "metric"' if has_metric else "")
|
|
1157
|
+
+ f"; got {given}; the key columns are {_join(ids)}."
|
|
1158
|
+
)
|
|
1159
|
+
|
|
1160
|
+
units = _explain_units(d, ids, _result_shape(d))
|
|
1161
|
+
cells = (
|
|
1162
|
+
pd.DataFrame({"metric": d["metric"].astype(str), "status": d["status"].astype(str)})
|
|
1163
|
+
if has_metric
|
|
1164
|
+
else None
|
|
1165
|
+
)
|
|
1166
|
+
if among == "joined":
|
|
1167
|
+
units = units[units["status"].isin(["same", "changed"])]
|
|
1168
|
+
if cells is not None:
|
|
1169
|
+
cells = cells[cells["status"].isin(["same", "changed"])]
|
|
1170
|
+
|
|
1171
|
+
def records(col):
|
|
1172
|
+
return cells if col == "metric" else units
|
|
1173
|
+
|
|
1174
|
+
if column is not None:
|
|
1175
|
+
return _explain_values(records(column), column)
|
|
1176
|
+
|
|
1177
|
+
rows = []
|
|
1178
|
+
for col in candidates:
|
|
1179
|
+
r = records(col)
|
|
1180
|
+
v = _explain_values(r, col)
|
|
1181
|
+
rows.append(
|
|
1182
|
+
{
|
|
1183
|
+
"column": col,
|
|
1184
|
+
"n_values": len(v),
|
|
1185
|
+
"same_only": int((v["class"] == "same").sum()),
|
|
1186
|
+
"differ_only": int((v["class"] == "differ").sum()),
|
|
1187
|
+
"mixed": int((v["class"] == "mixed").sum()),
|
|
1188
|
+
"separates": not (v["class"] == "mixed").any(),
|
|
1189
|
+
# metric is never an identifier: it repeats on every row.
|
|
1190
|
+
"identifies": col != "metric" and len(v) > 1 and len(v) == len(r),
|
|
1191
|
+
}
|
|
1192
|
+
)
|
|
1193
|
+
out = pd.DataFrame(rows, columns=_OVERVIEW)
|
|
1194
|
+
out = out.astype(
|
|
1195
|
+
{
|
|
1196
|
+
"n_values": "int64",
|
|
1197
|
+
"same_only": "int64",
|
|
1198
|
+
"differ_only": "int64",
|
|
1199
|
+
"mixed": "int64",
|
|
1200
|
+
"separates": bool,
|
|
1201
|
+
"identifies": bool,
|
|
1202
|
+
}
|
|
1203
|
+
)
|
|
1204
|
+
return out.sort_values(["identifies", "mixed", "n_values"], kind="stable", ignore_index=True)
|
|
1205
|
+
|
|
1206
|
+
|
|
1207
|
+
def _check_one_comparison(d: pd.DataFrame, ids: list[str], unit: list[str], shape: str):
|
|
1208
|
+
"""Refuse a frame holding several diff_table() results, such as two joined with
|
|
1209
|
+
``pd.concat``: row ids restart at 1 in each, so rows of different comparisons
|
|
1210
|
+
would be read as one (decided 2026-11-13, #41)."""
|
|
1211
|
+
record = unit + (["metric"] if "metric" in d else [])
|
|
1212
|
+
clash = bool(d.duplicated(record).any())
|
|
1213
|
+
if not clash and shape != "groups" and ids and len(d):
|
|
1214
|
+
keys = d.groupby(unit, dropna=False)[ids].nunique(dropna=False)
|
|
1215
|
+
clash = bool((keys > 1).to_numpy().any())
|
|
1216
|
+
if clash:
|
|
1217
|
+
raise DiffError(
|
|
1218
|
+
"`d` holds more than one comparison (results joined with pd.concat?): "
|
|
1219
|
+
"its row ids repeat. Explain or report each diff_table() result on its own."
|
|
1220
|
+
)
|
|
1221
|
+
|
|
1222
|
+
|
|
1223
|
+
def _explain_units(d: pd.DataFrame, ids: list[str], shape: str) -> pd.DataFrame:
|
|
1224
|
+
"""One record per row (per key with ``"groups"``), with the status of the row as
|
|
1225
|
+
a whole: ``only_x`` or ``only_y`` by membership, which comes from ``row_id`` or
|
|
1226
|
+
``n_rows`` and never from the value; otherwise ``changed`` when any of its
|
|
1227
|
+
measures changed."""
|
|
1228
|
+
present = "n_rows" if shape == "groups" else "row_id"
|
|
1229
|
+
# A row is its pair of source rows, which tells paired duplicates apart.
|
|
1230
|
+
unit = ids if shape == "groups" else ["row_id_x", "row_id_y"]
|
|
1231
|
+
_check_one_comparison(d, ids, unit, shape)
|
|
1232
|
+
w = d[list(dict.fromkeys(ids + unit))].copy()
|
|
1233
|
+
w["_in_x"] = d[f"{present}_x"].notna().to_numpy()
|
|
1234
|
+
w["_in_y"] = d[f"{present}_y"].notna().to_numpy()
|
|
1235
|
+
w["_changed"] = (d["status"].astype(str) == "changed").to_numpy()
|
|
1236
|
+
if len(w):
|
|
1237
|
+
agg = {c: "first" for c in ids if c not in unit}
|
|
1238
|
+
agg.update({"_in_x": "max", "_in_y": "max", "_changed": "max"})
|
|
1239
|
+
w = w.groupby(unit, dropna=False, sort=False).agg(agg).reset_index()
|
|
1240
|
+
status = np.select(
|
|
1241
|
+
[~w["_in_y"].astype(bool), ~w["_in_x"].astype(bool), w["_changed"].astype(bool)],
|
|
1242
|
+
["only_x", "only_y", "changed"],
|
|
1243
|
+
"same",
|
|
1244
|
+
)
|
|
1245
|
+
out = w[ids].reset_index(drop=True)
|
|
1246
|
+
out["status"] = status
|
|
1247
|
+
return out
|
|
1248
|
+
|
|
1249
|
+
|
|
1250
|
+
def _explain_values(r: pd.DataFrame, col: str) -> pd.DataFrame:
|
|
1251
|
+
"""The counts of each status per value of ``col``, with the value's class: most
|
|
1252
|
+
differences first, then the values that only differ before the mixed ones, since
|
|
1253
|
+
they are the cleaner lead."""
|
|
1254
|
+
s = r["status"].to_numpy()
|
|
1255
|
+
v = pd.DataFrame(
|
|
1256
|
+
{
|
|
1257
|
+
col: r[col].to_numpy(),
|
|
1258
|
+
"n": 1,
|
|
1259
|
+
"n_same": s == "same",
|
|
1260
|
+
"n_changed": s == "changed",
|
|
1261
|
+
"n_only_x": s == "only_x",
|
|
1262
|
+
"n_only_y": s == "only_y",
|
|
1263
|
+
}
|
|
1264
|
+
)
|
|
1265
|
+
out = (
|
|
1266
|
+
v.groupby(col, dropna=False, sort=False)
|
|
1267
|
+
.sum()
|
|
1268
|
+
.reset_index()
|
|
1269
|
+
.astype({k: "int64" for k in ["n", "n_same", "n_changed", "n_only_x", "n_only_y"]})
|
|
1270
|
+
)
|
|
1271
|
+
out["class"] = np.select(
|
|
1272
|
+
[out["n_same"] == out["n"], out["n_same"] == 0], ["same", "differ"], "mixed"
|
|
1273
|
+
)
|
|
1274
|
+
out["_d"] = out["n"] - out["n_same"]
|
|
1275
|
+
out["_r"] = out["class"].map({"differ": 0, "mixed": 1, "same": 2})
|
|
1276
|
+
out = out.sort_values(["_r", col], kind="stable", na_position="last")
|
|
1277
|
+
out = out.sort_values("_d", ascending=False, kind="stable", ignore_index=True)
|
|
1278
|
+
return out.drop(columns=["_d", "_r"])
|
|
1279
|
+
|
|
1280
|
+
|
|
1281
|
+
# --- assert_table_equal -----------------------------------------------------------
|
|
1282
|
+
|
|
1283
|
+
_SQRT_EPS = float(np.sqrt(np.finfo(float).eps))
|
|
1284
|
+
_UNSET = object()
|
|
1285
|
+
|
|
1286
|
+
|
|
1287
|
+
def assert_table_equal(
|
|
1288
|
+
actual: pd.DataFrame,
|
|
1289
|
+
expected: pd.DataFrame,
|
|
1290
|
+
*,
|
|
1291
|
+
rel_tol: float = _UNSET, # type: ignore[assignment]
|
|
1292
|
+
max_rows: int = 10,
|
|
1293
|
+
**kwargs,
|
|
1294
|
+
) -> pd.DataFrame:
|
|
1295
|
+
"""Assert that ``actual`` matches ``expected``, for use in tests.
|
|
1296
|
+
|
|
1297
|
+
Runs ``diff_table(expected, actual, ...)``, with ``expected`` as the benchmark,
|
|
1298
|
+
and raises :class:`AssertionError` with :func:`diff_summary` and the first
|
|
1299
|
+
``max_rows`` records that differ. Tables that share no key fail too. Other
|
|
1300
|
+
keyword arguments go to :func:`diff_table`.
|
|
1301
|
+
|
|
1302
|
+
``rel_tol`` defaults to ``sqrt(eps)`` (about 1.5e-8), as pytest and testthat
|
|
1303
|
+
compare floats, where :func:`diff_table` is exact; with
|
|
1304
|
+
``duplicates="aggregate"``, which compares exactly, it defaults to 0. Repeated
|
|
1305
|
+
keys of rows with measures are a :class:`DiffError` unless ``duplicates`` is
|
|
1306
|
+
given: an assertion must not decide identity by arrival order. Repeated whole
|
|
1307
|
+
rows are paired, since they are identical.
|
|
1308
|
+
|
|
1309
|
+
Returns the comparison when the tables match.
|
|
1310
|
+
"""
|
|
1311
|
+
if isinstance(max_rows, bool) or not isinstance(max_rows, (int, np.integer)) or max_rows < 0:
|
|
1312
|
+
raise DiffError("`max_rows` must be a single whole number >= 0.")
|
|
1313
|
+
duplicates = kwargs.get("duplicates")
|
|
1314
|
+
if rel_tol is _UNSET:
|
|
1315
|
+
rel_tol = 0.0 if duplicates == "aggregate" else _SQRT_EPS
|
|
1316
|
+
|
|
1317
|
+
caught: list = []
|
|
1318
|
+
refused = False
|
|
1319
|
+
try:
|
|
1320
|
+
with warnings.catch_warnings(record=True) as caught:
|
|
1321
|
+
warnings.simplefilter("always")
|
|
1322
|
+
d = diff_table(expected, actual, rel_tol=rel_tol, **kwargs)
|
|
1323
|
+
# Pairing repeated keys of rows with measures decides identity by arrival
|
|
1324
|
+
# order, which an assertion must not do unless the caller accepted it.
|
|
1325
|
+
if duplicates is None and d.attrs.get("paired") and d.attrs.get("measures"):
|
|
1326
|
+
refused = True
|
|
1327
|
+
raise DiffError(
|
|
1328
|
+
"`actual` or `expected` has repeated keys, which diff_table() pairs in "
|
|
1329
|
+
"arrival order.\nassert_table_equal() does not pair duplicate keys; pass "
|
|
1330
|
+
'duplicates="disambiguate" or "aggregate" to accept them.'
|
|
1331
|
+
)
|
|
1332
|
+
except DiffError as e:
|
|
1333
|
+
if getattr(e, "disjoint", False):
|
|
1334
|
+
raise AssertionError(
|
|
1335
|
+
f"`actual` does not match `expected` (as `y` and `x`): {e}"
|
|
1336
|
+
) from None
|
|
1337
|
+
raise
|
|
1338
|
+
finally:
|
|
1339
|
+
# Passed on whatever happens: a warning (an `exclude` typo, a dropped column)
|
|
1340
|
+
# often explains the failure that follows (#45).
|
|
1341
|
+
for w in caught:
|
|
1342
|
+
if refused and issubclass(w.category, _PairingWarning):
|
|
1343
|
+
continue # the DiffError says it
|
|
1344
|
+
warnings.warn_explicit(w.message, w.category, w.filename, w.lineno)
|
|
1345
|
+
|
|
1346
|
+
off = d[d["status"] != "same"]
|
|
1347
|
+
if not len(off):
|
|
1348
|
+
return d
|
|
1349
|
+
shown = off.head(max_rows)
|
|
1350
|
+
msg = (
|
|
1351
|
+
f"`actual` does not match `expected`: {len(off):,} of {len(d):,} records differ.\n\n"
|
|
1352
|
+
+ diff_summary(d).to_string(index=False)
|
|
1353
|
+
)
|
|
1354
|
+
if len(shown):
|
|
1355
|
+
lead = f"First {len(shown)} records that differ:\n" if len(off) > len(shown) else ""
|
|
1356
|
+
msg += "\n\n" + lead + shown.to_string(index=False)
|
|
1357
|
+
raise AssertionError(msg)
|