dazdiff 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
dazdiff/core.py ADDED
@@ -0,0 +1,1357 @@
1
+ """Compare two tables value by value."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+ import re
7
+ import warnings
8
+ from collections.abc import Sequence
9
+
10
+ import numpy as np
11
+ import pandas as pd
12
+
13
+ STATUSES = ["same", "changed", "only_x", "only_y"]
14
+ _MODES = ("benchmark", "equal")
15
+ _DUPLICATES = ("disambiguate", "aggregate", "error")
16
+ _ROW_KEYS = ("none", "position", "sorted")
17
+ _MEASURE_KEYWORDS = ("numeric", "numeric+integer")
18
+ _TRUE = {"true", "t", "yes", "1"}
19
+ _FALSE = {"false", "f", "no", "0"}
20
+
21
+
22
+ class DiffError(ValueError):
23
+ """The two tables cannot be compared as asked."""
24
+
25
+
26
+ class DiffWarning(UserWarning):
27
+ """The answer may be wrong, e.g. rows were paired by arrival order."""
28
+
29
+
30
+ class _PairingWarning(DiffWarning):
31
+ """Repeated keys were paired in arrival order (a DiffWarning; the subclass lets
32
+ assert_table_equal() drop it when it refuses the pairing outright)."""
33
+
34
+
35
+ class DiffNote(UserWarning):
36
+ """An expected side effect, e.g. a column was dropped or a value truncated."""
37
+
38
+
39
+ # --- argument checks ---------------------------------------------------------
40
+
41
+
42
+ def _check_frame(v, name: str) -> None:
43
+ if not isinstance(v, pd.DataFrame):
44
+ raise DiffError(f"`{name}` must be a pandas DataFrame, not {type(v).__name__}.")
45
+
46
+
47
+ def _check_result(d) -> None:
48
+ if not isinstance(d, pd.DataFrame) or "status" not in d:
49
+ raise DiffError("`d` must be a result of diff_table().")
50
+
51
+
52
+ def _check_threshold(v, name: str, allow_none: bool = False) -> None:
53
+ if v is None and allow_none:
54
+ return
55
+ ok = (
56
+ isinstance(v, (int, float, np.integer, np.floating))
57
+ and not isinstance(v, (bool, np.bool_))
58
+ and bool(np.isfinite(v))
59
+ and v >= 0
60
+ )
61
+ if not ok:
62
+ none = "None or " if allow_none else ""
63
+ raise DiffError(f"`{name}` must be {none}a single finite number >= 0, not {v!r}.")
64
+
65
+
66
+ def _check_columns_arg(v, name: str, allow_empty: bool) -> None:
67
+ """A column name, or a list or tuple of them."""
68
+ if isinstance(v, str):
69
+ return
70
+ ok = isinstance(v, (list, tuple)) and all(isinstance(c, str) for c in v)
71
+ if not ok or (not v and not allow_empty):
72
+ what = (
73
+ "a column name or a non-empty list of them"
74
+ if not allow_empty
75
+ else ("a keyword, a column name or a list of them")
76
+ )
77
+ raise DiffError(f"`{name}` must be {what}, not {v!r}.")
78
+
79
+
80
+ # --- names -------------------------------------------------------------------
81
+
82
+
83
+ def _canon(name) -> str:
84
+ return re.sub(r"[^A-Z0-9]+", "_", str(name).upper()).strip("_")
85
+
86
+
87
+ def _join(names) -> str:
88
+ return ", ".join(str(n) for n in names)
89
+
90
+
91
+ def normalize_names(names: Sequence[str]) -> list[str]:
92
+ """Normalize column names as :func:`diff_table` does.
93
+
94
+ Upper-case, every run of characters other than ``A-Z`` and ``0-9`` replaced by
95
+ one ``_``, and ``_`` trimmed from both ends. A name that ends up empty, or two
96
+ names that end up the same, are an error.
97
+ """
98
+ out = [_canon(n) for n in names]
99
+ _check_names(list(names), out, label="names")
100
+ return out
101
+
102
+
103
+ def _check_names(original: list, canon: list[str], label: str) -> None:
104
+ empty = [o for o, c in zip(original, canon, strict=True) if not c]
105
+ if empty:
106
+ raise DiffError(f"Column name(s) of {label} are empty once normalized: {empty!r}.")
107
+ groups: dict[str, list] = {}
108
+ for o, c in zip(original, canon, strict=True):
109
+ groups.setdefault(c, []).append(o)
110
+ clashes = [g for g in groups.values() if len(g) > 1]
111
+ if clashes:
112
+ listed = "; ".join(_join(g) for g in clashes)
113
+ raise DiffError(f"Column names of {label} clash once normalized: {listed}.")
114
+
115
+
116
+ # --- casting -----------------------------------------------------------------
117
+
118
+
119
+ def _is_float(s: pd.Series) -> bool:
120
+ return pd.api.types.is_float_dtype(s)
121
+
122
+
123
+ def _is_int(s: pd.Series) -> bool:
124
+ return pd.api.types.is_integer_dtype(s) and not pd.api.types.is_bool_dtype(s)
125
+
126
+
127
+ def _is_bool(s: pd.Series) -> bool:
128
+ return pd.api.types.is_bool_dtype(s)
129
+
130
+
131
+ def _is_datetime(s: pd.Series) -> bool:
132
+ return pd.api.types.is_datetime64_any_dtype(s)
133
+
134
+
135
+ def _is_text(s: pd.Series) -> bool:
136
+ return pd.api.types.is_object_dtype(s) or isinstance(s.dtype, pd.StringDtype)
137
+
138
+
139
+ def _show(v) -> str:
140
+ """A value as an error message shows it: strings quoted, numbers plain."""
141
+ if isinstance(v, str):
142
+ return repr(v)
143
+ return str(v.item() if isinstance(v, np.generic) else v)
144
+
145
+
146
+ def _first_bad(original: pd.Series, converted: pd.Series):
147
+ bad = original.notna() & converted.isna()
148
+ return original[bad].iloc[0]
149
+
150
+
151
+ def _to_text(v):
152
+ if isinstance(v, float) and v.is_integer():
153
+ return str(int(v + 0.0)) # + 0.0 writes -0.0 as "0"
154
+ return str(v)
155
+
156
+
157
+ def _examples(values, n: int = 3) -> str:
158
+ shown = [_show(v) if not pd.isna(v) else "NA" for v in list(values)[:n]]
159
+ return ", ".join(shown) + (", ..." if len(values) > n else "")
160
+
161
+
162
+ _PLAIN_DECIMAL = re.compile(r"-?(0|[1-9][0-9]*)(\.[0-9]+)?")
163
+
164
+
165
+ def _text_decimals(ref: pd.Series) -> tuple[int | None, list[int], bool]:
166
+ """The one number of decimals every value of the text column ``ref`` uses.
167
+
168
+ Read off every distinct value, not a sample: one ``"1.00"`` among a million
169
+ ``"1.0"`` is the value a sample misses. Only plain decimal text counts; a sign,
170
+ leading zeros, separators or an exponent mean there is no format to follow.
171
+ Returns the decimals (or None), the counts of decimals seen, and whether every
172
+ value is plain decimal text.
173
+ """
174
+ u = [str(v) for v in pd.unique(ref.dropna())]
175
+ plain = [bool(_PLAIN_DECIMAL.fullmatch(v)) for v in u]
176
+ seen = sorted({len(v.partition(".")[2]) for v, p in zip(u, plain, strict=True) if p})
177
+ k = seen[0] if u and all(plain) and len(seen) == 1 else None
178
+ return k, seen, all(plain)
179
+
180
+
181
+ def _number_as_text_like(v: pd.Series, out: pd.Series, ref: pd.Series, name: str, alerts: list):
182
+ """Write ``y``'s numbers ``v`` (already text in ``out``) as the benchmark ``ref``
183
+ writes them, where every value of ``ref`` has the same number of decimals.
184
+
185
+ A value is reformatted only when its text reads back as the same number: 1.25 at
186
+ one decimal would become "1.2" and join the benchmark's 1.2, a silent false
187
+ match, so it keeps its own text and shows as unmatched instead.
188
+ """
189
+ k, seen, plain = _text_decimals(ref)
190
+ num = v.to_numpy(dtype=float, na_value=np.nan)
191
+ finite = np.isfinite(num)
192
+ fits = np.zeros(len(v), dtype=bool)
193
+ kept = np.zeros(len(v), dtype=bool)
194
+ out = out.copy()
195
+ if k:
196
+ txt = np.array([f"{n + 0.0:.{k}f}" if f else "" for n, f in zip(num, finite, strict=True)])
197
+ fits = finite & np.array(
198
+ [f and float(t) == n for t, n, f in zip(txt, num, finite, strict=True)]
199
+ )
200
+ kept = finite & ~fits
201
+ out[fits] = txt[fits]
202
+
203
+ # Values of x that are numbers y has, written differently: those rows will not align.
204
+ first_text = {}
205
+ for n, t in zip(num, out, strict=True):
206
+ if np.isfinite(n):
207
+ first_text.setdefault(n + 0.0, t)
208
+ have = set(out.dropna())
209
+ u = [str(s) for s in pd.unique(ref.dropna())]
210
+ u_num = pd.to_numeric(pd.Series(u, dtype=object), errors="coerce").to_numpy(dtype=float)
211
+ skew = [
212
+ (s, first_text[n + 0.0])
213
+ for s, n in zip(u, u_num, strict=True)
214
+ if np.isfinite(n) and n + 0.0 in first_text and s not in have
215
+ ]
216
+ if skew:
217
+ pairs = ", ".join(f"{a!r} vs {b!r}" for a, b in skew[:3]) + (
218
+ ", ..." if len(skew) > 3 else ""
219
+ )
220
+ if k:
221
+ why = f"`y`'s numbers were written with {k} decimal place{'s' if k > 1 else ''}, as `x` writes them"
222
+ elif not plain:
223
+ why = "no decimal places were inferred, since not every value of `x` is plain decimal text"
224
+ elif len(seen) > 1:
225
+ why = f"no decimal places were inferred, since `x` writes them with {_join(seen)} decimal places"
226
+ else:
227
+ why = "`x` writes them with no decimal places"
228
+ alerts.append(
229
+ (
230
+ DiffWarning,
231
+ f"Column `{name}`: {len(skew)} value(s) of `x` equal a number of `y` but not its "
232
+ f"text ({pairs}), so those rows will not align; {why}; cast the column of `x` "
233
+ "to a number to compare by value.",
234
+ )
235
+ )
236
+ elif fits.any():
237
+ firsts = pd.Series(num[fits]).drop_duplicates().index[:3]
238
+ shown = [(num[fits][i], out[fits].iloc[i]) for i in firsts]
239
+ listed = ", ".join(f"{n:g} -> {t!r}" for n, t in shown)
240
+ more = ", ..." if int(fits.sum()) > len(shown) else ""
241
+ extra = ""
242
+ if kept.any():
243
+ kept_text = ", ".join(repr(t) for t in pd.unique(out[kept])[:3])
244
+ extra = f"; {int(kept.sum())} value(s) needed more decimals and kept their own digits ({kept_text})"
245
+ alerts.append(
246
+ (
247
+ DiffNote,
248
+ f"Wrote column `{name}` of `y` as text with {k} decimal place{'s' if k > 1 else ''}, "
249
+ f"as `x` writes it: {int(fits.sum())} value(s) ({listed}{more}){extra}.",
250
+ )
251
+ )
252
+ return out
253
+
254
+
255
+ def _stamp_to_text(y: pd.Series) -> pd.Series:
256
+ """Date-times as ``%Y-%m-%d %H:%M:%S``, midnight included, with the fraction of a
257
+ second to the microsecond, without trailing zeros, when there is one."""
258
+ s = y.dt.round("us")
259
+ txt = s.dt.strftime("%Y-%m-%d %H:%M:%S")
260
+ us = s.dt.microsecond
261
+ frac = s.notna() & (us > 0)
262
+ txt[frac] = txt[frac] + "." + us[frac].map(lambda u: f"{int(u):06d}".rstrip("0"))
263
+ return txt
264
+
265
+
266
+ _INT_TEXT = re.compile(r"[+-]?[0-9]+")
267
+
268
+
269
+ def _nullable(dtype) -> str:
270
+ """The pandas nullable integer dtype for an integer dtype: int8 -> Int8."""
271
+ if isinstance(dtype, pd.api.extensions.ExtensionDtype):
272
+ return str(dtype)
273
+ return str(dtype).replace("uint", "UInt").replace("int", "Int")
274
+
275
+
276
+ def _cast_to_int(
277
+ y: pd.Series, x: pd.Series, name: str, truncate: bool, truncated: dict
278
+ ) -> pd.Series:
279
+ """Cast ``y`` to the integer dtype of ``x`` exactly.
280
+
281
+ Integer dtypes and integer text become Python ints, so a value above 2**53 keeps
282
+ every digit; other numbers go through float, where a fraction truncates. A value
283
+ outside the target dtype's range is refused rather than wrapped around.
284
+ """
285
+ target = x.dtype.numpy_dtype if hasattr(x.dtype, "numpy_dtype") else x.dtype
286
+ info = np.iinfo(target)
287
+ missing = y.isna().to_numpy()
288
+ if _is_int(y) or _is_bool(y):
289
+ values = [None if m else int(v) for v, m in zip(y.tolist(), missing, strict=True)]
290
+ frac: list = []
291
+ else:
292
+ values, frac = [], []
293
+ for v, m in zip(y.tolist(), missing, strict=True):
294
+ if m:
295
+ values.append(None)
296
+ continue
297
+ if isinstance(v, str) and _INT_TEXT.fullmatch(v.strip()):
298
+ values.append(int(v.strip()))
299
+ continue
300
+ try:
301
+ f = float(v)
302
+ except (TypeError, ValueError):
303
+ raise DiffError(
304
+ f"Cannot cast `{name}` to a number: {_show(v)} is not one."
305
+ ) from None
306
+ if math.isnan(f):
307
+ raise DiffError(f"Cannot cast `{name}` to a number: {_show(v)} is not one.")
308
+ if math.isinf(f):
309
+ raise DiffError(f"Cannot cast `{name}` to integer: it holds infinite values.")
310
+ t = math.trunc(f)
311
+ if t != f:
312
+ frac.append((f, float(t)))
313
+ values.append(t)
314
+ if frac:
315
+ if not truncate:
316
+ raise DiffError(
317
+ f"Casting `{name}` to integer would truncate {len(frac)} value(s), "
318
+ f"e.g. {_show(frac[0][0])}; pass truncate=True to allow it."
319
+ )
320
+ truncated[name] = frac
321
+ bad = next((v for v in values if v is not None and not info.min <= v <= info.max), None)
322
+ if bad is not None:
323
+ raise DiffError(
324
+ f"Cannot cast `{name}` to {target}: {bad} is outside the range "
325
+ f"{info.min} to {info.max}."
326
+ )
327
+ out = pd.Series(pd.array(values, dtype=_nullable(target)), index=y.index)
328
+ if isinstance(x.dtype, pd.api.extensions.ExtensionDtype) or out.isna().any():
329
+ return out.astype(_nullable(x.dtype))
330
+ return out.astype(x.dtype)
331
+
332
+
333
+ def _object_needs_text(y: pd.Series, x: pd.Series) -> bool:
334
+ """Two ``object`` columns share a dtype but not a type: pandas 2 reads text as
335
+ object, and an object column of y may hold numbers. When x holds only text and y
336
+ does not, y is cast to text like any other column."""
337
+ if not pd.api.types.is_object_dtype(x):
338
+ return False
339
+ texts = all(isinstance(v, str) for v in x.dropna())
340
+ return texts and not all(isinstance(v, str) for v in y.dropna())
341
+
342
+
343
+ def _cast_to(
344
+ y: pd.Series, x: pd.Series, name: str, truncate: bool, truncated: dict, alerts: list
345
+ ) -> pd.Series:
346
+ """Cast column ``y`` to the dtype of the benchmark column ``x``."""
347
+ if y.dtype == x.dtype and not _object_needs_text(y, x):
348
+ return y
349
+
350
+ # A column with no values (how a reader types an empty column) casts to anything.
351
+ if (
352
+ len(y)
353
+ and y.isna().all()
354
+ and (_is_int(x) or _is_float(x) or _is_bool(x) or _is_datetime(x) or _is_text(x))
355
+ ):
356
+ target = "Int64" if _is_int(x) else "boolean" if _is_bool(x) else x.dtype
357
+ missing = np.nan if pd.api.types.is_object_dtype(x) or _is_float(x) else pd.NA
358
+ return pd.Series([missing] * len(y), index=y.index, dtype=object).astype(target)
359
+
360
+ if _is_int(x):
361
+ return _cast_to_int(y, x, name, truncate, truncated)
362
+
363
+ if _is_float(x):
364
+ if _is_bool(y):
365
+ num = y.astype(float)
366
+ else:
367
+ num = pd.to_numeric(y, errors="coerce")
368
+ if (y.notna() & num.isna()).any():
369
+ v = _first_bad(y, num)
370
+ raise DiffError(f"Cannot cast `{name}` to a number: {_show(v)} is not one.")
371
+ return num.astype(float).astype(x.dtype)
372
+
373
+ if _is_bool(x):
374
+ if _is_int(y) or _is_float(y):
375
+ ok = y.isna() | y.isin([0, 1])
376
+ if not ok.all():
377
+ raise DiffError(
378
+ f"Cannot cast `{name}` to bool: {_show(y[~ok].iloc[0])} is not 0 or 1."
379
+ )
380
+ out = y.map(lambda v: v if pd.isna(v) else bool(v))
381
+ else:
382
+ low = y.map(lambda v: v if pd.isna(v) else str(v).strip().lower())
383
+ out = low.map(lambda v: True if v in _TRUE else False if v in _FALSE else None)
384
+ if (y.notna() & out.isna()).any():
385
+ v = _first_bad(y, out)
386
+ raise DiffError(f"Cannot cast `{name}` to bool: {_show(v)} is not true or false.")
387
+ return out.astype("boolean" if out.isna().any() else x.dtype)
388
+
389
+ if _is_datetime(x):
390
+ # pandas reads a number as nanoseconds since 1970, which is never what a
391
+ # number in a date column means, so numbers are refused rather than guessed.
392
+ if _is_int(y) or _is_float(y) or _is_bool(y):
393
+ raise DiffError(
394
+ f"Cannot cast `{name}` to a date-time: it holds numbers ({y.dtype}), "
395
+ "not dates; convert it first."
396
+ )
397
+ try:
398
+ with warnings.catch_warnings():
399
+ # pandas 2 warns about text in several UTC offsets where pandas 3
400
+ # raises; both become the same DiffError.
401
+ warnings.simplefilter("error", FutureWarning)
402
+ conv = pd.to_datetime(y, errors="coerce")
403
+ except (TypeError, ValueError, FutureWarning) as e:
404
+ raise DiffError(f"Cannot cast `{name}` to a date-time: {e}") from None
405
+ if (y.notna() & conv.isna()).any():
406
+ v = _first_bad(y, conv)
407
+ raise DiffError(f"Cannot cast `{name}` to a date-time: {_show(v)} is not one.")
408
+ try:
409
+ return conv.astype(x.dtype)
410
+ except (TypeError, ValueError) as e:
411
+ raise DiffError(f"Cannot cast `{name}` to {x.dtype}: {e}") from e
412
+
413
+ if _is_text(x):
414
+ if _is_datetime(y):
415
+ return _stamp_to_text(y).astype(x.dtype)
416
+ out = y.map(lambda v: v if pd.isna(v) else _to_text(v)).astype(object)
417
+ if _is_int(y) or _is_float(y):
418
+ out = _number_as_text_like(y, out, x, name, alerts)
419
+ return out.astype(x.dtype)
420
+
421
+ raise DiffError(
422
+ f"Cannot cast `{name}` from {y.dtype} to {x.dtype}: supported target types are "
423
+ "integer, float, bool, string and date-time."
424
+ )
425
+
426
+
427
+ # --- measures and identity ---------------------------------------------------
428
+
429
+
430
+ def _select_measures(x: pd.DataFrame, measures, x_names: dict) -> list[str]:
431
+ if isinstance(measures, str) and measures in _MEASURE_KEYWORDS:
432
+ if measures == "numeric":
433
+ return [c for c in x.columns if _is_float(x[c])]
434
+ return [c for c in x.columns if _is_float(x[c]) or _is_int(x[c])]
435
+ given = [measures] if isinstance(measures, str) else list(measures)
436
+ canon = [_canon(m) for m in given]
437
+ missing = [g for g, c in zip(given, canon, strict=True) if c not in x.columns]
438
+ if missing:
439
+ raise DiffError(f"Measure column(s) not found in `x`: {_join(missing)}.")
440
+ bad = [x_names[c] for c in canon if not (_is_float(x[c]) or _is_int(x[c]))]
441
+ if bad:
442
+ raise DiffError(f"Measure column(s) are not numeric: {_join(bad)}.")
443
+ return list(dict.fromkeys(canon))
444
+
445
+
446
+ def _add_seq(df: pd.DataFrame, by: list[str], duplicates: str, label: str, alerts: list):
447
+ seq = df.groupby(by, dropna=False, sort=False).cumcount() + 1
448
+ n_dup = int((seq > 1).sum())
449
+ if n_dup and duplicates != "ignore":
450
+ if duplicates == "error":
451
+ raise DiffError(f"`{label}` has {n_dup} row(s) with a repeated key.")
452
+ alerts.append(
453
+ (
454
+ _PairingWarning,
455
+ f"`{label}` has {n_dup} row(s) with a repeated key; "
456
+ "they are paired in order of arrival.",
457
+ )
458
+ )
459
+ df["key_seq"] = seq.astype("int64").to_numpy()
460
+ return df
461
+
462
+
463
+ # --- values ------------------------------------------------------------------
464
+
465
+
466
+ def _int_diff(vx: pd.Series, vy: pd.Series):
467
+ """``vy - vx`` for two integer columns, exact as an int64 where it fits, and
468
+ ``vy == vx`` exactly; NaN where either is missing."""
469
+ miss = vx.isna().to_numpy() | vy.isna().to_numpy()
470
+ if "uint64" in str(vx.dtype).lower() or "uint64" in str(vy.dtype).lower():
471
+ pairs = zip(vx.tolist(), vy.tolist(), miss, strict=True)
472
+ exact = [0 if m else int(b) - int(a) for a, b, m in pairs]
473
+ diff = np.array([float(d) for d in exact])
474
+ equal = ~miss & np.array([d == 0 for d in exact], dtype=bool)
475
+ else:
476
+ a = vx.to_numpy(dtype="int64", na_value=0)
477
+ b = vy.to_numpy(dtype="int64", na_value=0)
478
+ with np.errstate(over="ignore"):
479
+ d = b - a
480
+ # Two's complement: b - a overflowed when b and a differ in sign and the
481
+ # result's sign is not b's.
482
+ over = ((b ^ a) & (b ^ d)) < 0
483
+ diff = d.astype(float)
484
+ diff[over] = b[over].astype(float) - a[over].astype(float)
485
+ equal = ~miss & ~over & (d == 0)
486
+ diff[miss] = np.nan
487
+ return diff, equal
488
+
489
+
490
+ def _classify(vx: pd.Series, vy: pd.Series, tolerance: float, rel_tol: float):
491
+ if _is_int(vx) and _is_int(vy):
492
+ # Compared as integers: as floats, values above 2**53 that differ by a few
493
+ # units are the same number.
494
+ both_na = (vx.isna() & vy.isna()).to_numpy()
495
+ diff, equal = _int_diff(vx, vy)
496
+ fx = vx.to_numpy(dtype=float, na_value=np.nan)
497
+ fy = vy.to_numpy(dtype=float, na_value=np.nan)
498
+ finite = ~np.isnan(diff)
499
+ abs_diff = np.abs(diff)
500
+ with np.errstate(invalid="ignore", divide="ignore"):
501
+ scale = np.maximum(np.abs(fx), np.abs(fy))
502
+ rel = np.where(scale > 0, abs_diff / np.where(scale > 0, scale, 1.0), 0.0)
503
+ rel_diff = np.where(equal, 0.0, np.where(finite, rel, np.nan))
504
+ close = finite & ((abs_diff <= tolerance) | (rel_diff <= rel_tol))
505
+ return (
506
+ np.where(equal, 0.0, diff),
507
+ np.where(equal, 0.0, abs_diff),
508
+ rel_diff,
509
+ both_na | equal | close,
510
+ )
511
+ vx = vx.to_numpy(dtype=float, na_value=np.nan)
512
+ vy = vy.to_numpy(dtype=float, na_value=np.nan)
513
+ with np.errstate(over="ignore", invalid="ignore", divide="ignore"):
514
+ equal = vx == vy # also equal infinities
515
+ finite = np.isfinite(vx) & np.isfinite(vy)
516
+ diff = np.where(equal, 0.0, vy - vx)
517
+ abs_diff = np.abs(diff)
518
+ scale = np.maximum(np.abs(vx), np.abs(vy))
519
+ safe = np.where(scale > 0, scale, 1.0)
520
+ rel = np.abs(vy / safe - vx / safe) # scaled first, so it cannot overflow
521
+ rel_diff = np.where(equal, 0.0, np.where(finite, rel, np.nan))
522
+ both_na = np.isnan(vx) & np.isnan(vy)
523
+ close = finite & ((abs_diff <= tolerance) | (rel_diff <= rel_tol))
524
+ same = both_na | equal | close
525
+ return diff, abs_diff, rel_diff, same
526
+
527
+
528
+ # --- diff_table --------------------------------------------------------------
529
+
530
+
531
+ def diff_table(
532
+ x: pd.DataFrame,
533
+ y: pd.DataFrame,
534
+ *,
535
+ mode: str = "benchmark",
536
+ by: Sequence[str] | str | None = None,
537
+ exclude: Sequence[str] | str | None = None,
538
+ row_key: str = "none",
539
+ measures: Sequence[str] | str = "numeric",
540
+ tolerance: float = 0.0,
541
+ rel_tol: float = 0.0,
542
+ duplicates: str = "disambiguate",
543
+ cast: bool | None = None,
544
+ truncate: bool = True,
545
+ ) -> pd.DataFrame:
546
+ """Compare two tables value by value.
547
+
548
+ Parameters
549
+ ----------
550
+ x, y
551
+ The tables. In ``mode="benchmark"``, ``x`` is the benchmark: ``y`` is cast to
552
+ its column types and ``y``'s extra columns are dropped. In ``mode="equal"`` the
553
+ tables must have the same columns with the same types.
554
+ mode
555
+ ``"benchmark"`` (default) or ``"equal"``.
556
+ by
557
+ Key columns that identify a row. Defaults to every column that is not a measure.
558
+ With ``by`` given, other non-measure columns are ignored with a note in
559
+ benchmark mode, and are an error in equal mode.
560
+ exclude
561
+ Columns, in either spelling, removed from both tables before anything else, in
562
+ either mode. They need not be in both tables and are not announced as dropped
563
+ or ignored. A name in neither table is a :class:`DiffWarning`; a name ``by`` or
564
+ ``measures`` also names is an error.
565
+ row_key
566
+ ``"none"`` (default), or a virtual ``row_number`` key added to the identity:
567
+ ``"position"`` numbers rows as they come, ``"sorted"`` numbers them after
568
+ sorting both tables by their compared columns. Use it when every column is a
569
+ measure. It cannot be combined with ``by``.
570
+ measures
571
+ ``"numeric"`` (float columns), ``"numeric+integer"``, or column names.
572
+ tolerance, rel_tol
573
+ For a finite pair, a value is ``same`` when ``abs_diff <= tolerance`` or
574
+ ``rel_diff <= rel_tol``.
575
+ duplicates
576
+ ``"disambiguate"`` (default) pairs repeated keys in arrival order, with a
577
+ warning; ``"aggregate"`` compares the multiset of values per key and measure
578
+ (tolerances must be 0); ``"error"`` refuses repeated keys.
579
+ cast
580
+ Whether to cast the compared columns of ``y`` to ``x``'s types. ``None``
581
+ (default) casts in benchmark mode; ``False`` requires the types to agree.
582
+ Equal mode never casts, so ``cast=True`` with it is an error. A number cast
583
+ to text is written with the decimals ``x`` uses, when every value of ``x``'s
584
+ column uses the same number of them.
585
+ truncate
586
+ Allow truncating non-integral values when casting to an integer column.
587
+
588
+ Returns
589
+ -------
590
+ pandas.DataFrame
591
+ Sorted by the key. With measures (``attrs["shape"] == "cells"``), one row per
592
+ key and measure, with ``metric``, ``row_id_x``, ``row_id_y``, ``value_x``,
593
+ ``value_y``, ``diff``, ``abs_diff``, ``rel_diff`` and ``status``. With
594
+ ``duplicates="aggregate"`` (``"groups"``), one row per key and measure, with
595
+ ``n_rows_x``, ``n_rows_y``, ``values_x`` and ``values_y`` (sorted tuples). With no
596
+ measures (``"rows"``), one row per source row, with ``key_seq``. ``attrs``
597
+ also holds ``mode``, ``by``, ``measures``, ``tolerance``, ``name_map``,
598
+ ``dropped``, ``excluded``, ``ignored`` and ``truncated``.
599
+
600
+ Raises
601
+ ------
602
+ DiffError
603
+ When the question cannot be answered: missing columns, type differences in
604
+ equal mode, clashing names, no key, or no key in common.
605
+
606
+ Warns
607
+ -----
608
+ DiffWarning
609
+ When the answer may be wrong: repeated keys paired by arrival order.
610
+ DiffNote
611
+ For expected side effects: dropped or ignored columns, truncation, no measures.
612
+ """
613
+ _check_frame(x, "x")
614
+ _check_frame(y, "y")
615
+ if mode not in _MODES:
616
+ raise DiffError(f"`mode` must be one of {_MODES}, not {mode!r}.")
617
+ if duplicates not in _DUPLICATES:
618
+ raise DiffError(f"`duplicates` must be one of {_DUPLICATES}, not {duplicates!r}.")
619
+ if row_key not in _ROW_KEYS:
620
+ raise DiffError(f"`row_key` must be one of {_ROW_KEYS}, not {row_key!r}.")
621
+ if cast is not None and not isinstance(cast, bool):
622
+ raise DiffError("`cast` must be None, True or False.")
623
+ if mode == "equal" and cast:
624
+ raise DiffError('mode = "equal" never casts: neither table is the reference for the other.')
625
+ if cast is None:
626
+ cast = mode == "benchmark"
627
+ if row_key != "none" and by is not None:
628
+ raise DiffError("`row_key` adds a key of its own, so it cannot be combined with `by`.")
629
+ _check_threshold(tolerance, "tolerance")
630
+ _check_threshold(rel_tol, "rel_tol")
631
+ _check_columns_arg(measures, "measures", allow_empty=True)
632
+ if by is not None:
633
+ _check_columns_arg(by, "by", allow_empty=False)
634
+ if exclude is not None:
635
+ _check_columns_arg(exclude, "exclude", allow_empty=True)
636
+ if duplicates == "aggregate" and (tolerance or rel_tol):
637
+ raise DiffError(
638
+ 'duplicates = "aggregate" compares multisets exactly; `tolerance` and `rel_tol` '
639
+ "must be 0."
640
+ )
641
+
642
+ alerts: list[tuple[type[Warning], str]] = []
643
+ try:
644
+ out = _diff_table(
645
+ x,
646
+ y,
647
+ mode,
648
+ by,
649
+ exclude,
650
+ row_key,
651
+ measures,
652
+ tolerance,
653
+ rel_tol,
654
+ duplicates,
655
+ cast,
656
+ truncate,
657
+ alerts,
658
+ )
659
+ finally:
660
+ for category, message in alerts:
661
+ warnings.warn(message, category, stacklevel=2)
662
+ return out
663
+
664
+
665
+ def _exclude(x, y, exclude, by, measures, alerts):
666
+ """Remove the ``exclude`` columns from both tables, before anything else looks at
667
+ them: they are neither required, dropped with a note, nor checked for clashes."""
668
+ given = [exclude] if isinstance(exclude, str) else list(exclude)
669
+ canon = [_canon(e) for e in given]
670
+ cols_x = [_canon(c) for c in x.columns]
671
+ cols_y = [_canon(c) for c in y.columns]
672
+ # A warning, not an error: one list of technical columns (load stamps, run ids)
673
+ # serves many tables, not all of which have each one. Still a warning, since a
674
+ # typo leaves a column compared.
675
+ unknown = [g for g, c in zip(given, canon, strict=True) if c not in cols_x + cols_y]
676
+ if unknown:
677
+ alerts.append(
678
+ (
679
+ DiffWarning,
680
+ f"`exclude` names column(s) in neither table, so nothing was excluded for "
681
+ f"them: {_join(unknown)}.",
682
+ )
683
+ )
684
+ named = [] if by is None else [by] if isinstance(by, str) else list(by)
685
+ if not (isinstance(measures, str) and measures in _MEASURE_KEYWORDS):
686
+ named += [measures] if isinstance(measures, str) else list(measures)
687
+ named_canon = {_canon(n) for n in named}
688
+ both = [g for g, c in zip(given, canon, strict=True) if c in named_canon]
689
+ if both:
690
+ raise DiffError(
691
+ f"`exclude` names column(s) that `by` or `measures` also name: {_join(both)}."
692
+ )
693
+ drop = set(canon)
694
+ excluded = list(
695
+ dict.fromkeys(
696
+ [o for o, c in zip(x.columns, cols_x, strict=True) if c in drop]
697
+ + [o for o, c in zip(y.columns, cols_y, strict=True) if c in drop]
698
+ )
699
+ )
700
+ x = x.iloc[:, [i for i, c in enumerate(cols_x) if c not in drop]]
701
+ y = y.iloc[:, [i for i, c in enumerate(cols_y) if c not in drop]]
702
+ return x, y, excluded
703
+
704
+
705
+ def _diff_table(
706
+ x,
707
+ y,
708
+ mode,
709
+ by,
710
+ exclude,
711
+ row_key,
712
+ measures,
713
+ tolerance,
714
+ rel_tol,
715
+ duplicates,
716
+ cast,
717
+ truncate,
718
+ alerts,
719
+ ):
720
+ excluded: list = []
721
+ if exclude is not None:
722
+ x, y, excluded = _exclude(x, y, exclude, by, measures, alerts)
723
+
724
+ # Names: x is checked whole; y only for the columns that are compared.
725
+ x_canon = [_canon(c) for c in x.columns]
726
+ _check_names(list(x.columns), x_canon, "`x`")
727
+ y_canon = [_canon(c) for c in y.columns]
728
+ if mode == "equal":
729
+ _check_names(list(y.columns), y_canon, "`y`")
730
+ else:
731
+ used = [(o, c) for o, c in zip(y.columns, y_canon, strict=True) if c in set(x_canon)]
732
+ _check_names([o for o, _ in used], [c for _, c in used], "`y`")
733
+ x_names = dict(zip(x_canon, x.columns, strict=True))
734
+ y_names = {c: o for o, c in zip(y.columns, y_canon, strict=True) if c}
735
+ # Every column of y that x lacks, as written: also those whose names normalize
736
+ # to nothing or clash, which y_names cannot hold.
737
+ y_extra = [o for o, c in zip(y.columns, y_canon, strict=True) if c not in set(x_canon)]
738
+ name_map = {c: {"x": o, "y": y_names.get(c)} for c, o in x_names.items()}
739
+ name_map.update({c: {"x": None, "y": o} for c, o in y_names.items() if c not in x_names})
740
+
741
+ x = x.copy()
742
+ x.columns = x_canon
743
+ x = x.reset_index(drop=True)
744
+ y_keep = [i for i, c in enumerate(y_canon) if c in y_names and y_names[c] == y.columns[i]]
745
+ y = y.iloc[:, y_keep].copy()
746
+ y.columns = [y_canon[i] for i in y_keep]
747
+ y = y.reset_index(drop=True)
748
+
749
+ only_x = [c for c in x.columns if c not in y.columns]
750
+ only_y = [c for c in y.columns if c not in x.columns]
751
+ dropped: list = []
752
+
753
+ if mode == "equal":
754
+ if only_x or only_y:
755
+ raise DiffError(
756
+ 'mode = "equal" needs the same columns in both tables.\n'
757
+ f" only in `x`: {_join(x_names[c] for c in only_x) or '<none>'}\n"
758
+ f" only in `y`: {_join(y_names[c] for c in only_y) or '<none>'}"
759
+ )
760
+ else:
761
+ if only_x:
762
+ missing = _join(x_names[c] for c in only_x)
763
+ raise DiffError(f"`y` lacks column(s) of the benchmark `x`: {missing}.")
764
+ dropped = y_extra
765
+ if dropped:
766
+ alerts.append(
767
+ (
768
+ DiffNote,
769
+ f"Dropped column(s) of `y` that the benchmark `x` does not have: "
770
+ f"{_join(dropped)}.",
771
+ )
772
+ )
773
+ y = y[list(x.columns)]
774
+
775
+ # Roles are decided on x, whose types a cast never changes, and before casting,
776
+ # so a column the call ignores can neither refuse the comparison nor be truncated.
777
+ rest: list = []
778
+ if by is not None:
779
+ given = [by] if isinstance(by, str) else list(by)
780
+ by = list(dict.fromkeys(_canon(b) for b in given))
781
+ missing = [g for g in given if _canon(g) not in x.columns]
782
+ if missing:
783
+ raise DiffError(f"Key column(s) not found in `x`: {_join(missing)}.")
784
+ if not (isinstance(measures, str) and measures in _MEASURE_KEYWORDS):
785
+ named = [measures] if isinstance(measures, str) else list(measures)
786
+ both = [m for m in named if _canon(m) in by]
787
+ if both:
788
+ raise DiffError(f"`by` and `measures` both name: {_join(both)}.")
789
+ meas = [m for m in _select_measures(x, measures, x_names) if m not in by]
790
+ rest = [c for c in x.columns if c not in by and c not in meas]
791
+ if rest:
792
+ if mode == "equal":
793
+ raise DiffError(
794
+ f"Column(s) that are neither keys nor measures: {_join(x_names[c] for c in rest)}. "
795
+ 'mode = "equal" compares every column; add them to `by` or `measures`.'
796
+ )
797
+ alerts.append(
798
+ (
799
+ DiffNote,
800
+ f"Ignored column(s) that are neither keys nor measures: "
801
+ f"{_join(x_names[c] for c in rest)}.",
802
+ )
803
+ )
804
+ else:
805
+ meas = _select_measures(x, measures, x_names)
806
+ by = [c for c in x.columns if c not in meas]
807
+ compared = by + meas
808
+
809
+ # Types, of the compared columns only.
810
+ truncated: dict = {}
811
+ if cast:
812
+ y = y.assign(
813
+ **{c: _cast_to(y[c], x[c], y_names[c], truncate, truncated, alerts) for c in compared}
814
+ )
815
+ if truncated:
816
+ lines = [f" {c}: {len(v)} ({v[0][0]:g} -> {v[0][1]:g})" for c, v in truncated.items()]
817
+ n = sum(len(v) for v in truncated.values())
818
+ alerts.append(
819
+ (
820
+ DiffNote,
821
+ f"Truncated {n} value(s) while casting to the reference types:\n"
822
+ + "\n".join(lines),
823
+ )
824
+ )
825
+ else:
826
+ differ = [
827
+ f"{x_names[c]} ({x[c].dtype} vs {y[c].dtype})"
828
+ for c in compared
829
+ if x[c].dtype != y[c].dtype
830
+ ]
831
+ if differ:
832
+ lead = (
833
+ 'mode = "equal" needs the same column types'
834
+ if mode == "equal"
835
+ else ("cast=False, but x and y disagree on the type of")
836
+ )
837
+ raise DiffError(f"{lead}: " + ", ".join(differ))
838
+
839
+ if row_key != "none":
840
+ x = x.assign(row_number=_row_numbers(x, row_key, compared))
841
+ y = y.assign(row_number=_row_numbers(y, row_key, compared))
842
+ x_names["row_number"] = "row_number"
843
+ by = [*by, "row_number"]
844
+ if not by:
845
+ raise DiffError(
846
+ "Every column is a measure, so rows have no key; pass `by`, fewer `measures`, "
847
+ 'or `row_key="position"` or `"sorted"`.'
848
+ )
849
+
850
+ if not meas:
851
+ alerts.append(
852
+ (
853
+ DiffNote,
854
+ "No measure columns, so rows are compared as wholes: "
855
+ "do the two tables have the same rows?",
856
+ )
857
+ )
858
+
859
+ x_names_y = {c: y_names.get(c) for c in x.columns}
860
+ x, y = x[by + meas].copy(), y[by + meas].copy()
861
+ paired = False
862
+ if duplicates == "aggregate" and meas:
863
+ out = _aggregate(x, y, by, meas, x_names)
864
+ sort, shape = [*by, "metric"], "groups"
865
+ else:
866
+ # With no measures a row is its own key, so repeats are copies, not clashes,
867
+ # unless the caller asked for repeats to be refused.
868
+ dup_rule = duplicates if meas or duplicates == "error" else "ignore"
869
+ out, sort, shape = _pairwise(x, y, by, meas, dup_rule, tolerance, rel_tol, x_names, alerts)
870
+ # The cells shape drops key_seq, so whether repeated keys were paired is kept here.
871
+ paired = bool(x.duplicated(by).any() or y.duplicated(by).any())
872
+
873
+ out = out.sort_values(sort, kind="stable", ignore_index=True, na_position="last")
874
+ out.attrs.update(
875
+ {
876
+ "shape": shape,
877
+ "mode": mode,
878
+ "by": by,
879
+ "measures": meas,
880
+ "tolerance": {"abs": float(tolerance), "rel": float(rel_tol)},
881
+ "name_map": name_map,
882
+ "dropped": dropped,
883
+ "excluded": excluded,
884
+ "ignored": [x_names[c] for c in rest],
885
+ "paired": paired,
886
+ "truncated": {
887
+ c: truncated[x_names_y[c]] for c in by + meas if x_names_y.get(c) in truncated
888
+ },
889
+ }
890
+ )
891
+ return out
892
+
893
+
894
+ def _row_numbers(df: pd.DataFrame, row_key: str, compared: list[str]) -> np.ndarray:
895
+ n = len(df)
896
+ if row_key == "position" or n == 0:
897
+ return np.arange(1, n + 1)
898
+ order = df.sort_values(compared, kind="stable", na_position="last").index.to_numpy()
899
+ numbers = np.empty(n, dtype="int64")
900
+ numbers[order] = np.arange(1, n + 1)
901
+ return numbers
902
+
903
+
904
+ def _no_overlap(x: pd.DataFrame, y: pd.DataFrame, x_names: dict, by: list[str]) -> DiffError:
905
+ """The error for tables that share no key, naming the key columns that alone share
906
+ no value: most often one that labels the table (a run id, a source) and belongs in
907
+ ``exclude``."""
908
+ msg = (
909
+ f"No row of `x` has a key in common with `y` (key: {_join(x_names[c] for c in by)}); "
910
+ "check `by`, and the format of the key values"
911
+ )
912
+ apart, constant = [], []
913
+ many = max(len(x), len(y)) > 1
914
+ for c in by:
915
+ ux, uy = pd.Series(pd.unique(x[c])), pd.Series(pd.unique(y[c]))
916
+ if not ux.isin(uy).any():
917
+ apart.append(f"{x_names[c]} (x: {_examples(ux)}; y: {_examples(uy)})")
918
+ if many and len(ux) == 1 and len(uy) == 1:
919
+ constant.append(x_names[c])
920
+ if apart:
921
+ msg += f"; key column(s) that share no value between the tables: {', '.join(apart)}"
922
+ # Offered only when a key is left once they are excluded.
923
+ if constant and len(constant) < len(by):
924
+ each = "each hold" if len(constant) > 1 else "holds"
925
+ msg += (
926
+ f"; {_join(constant)} {each} one value in each table, so "
927
+ f"{'they label' if len(constant) > 1 else 'it labels'} the table rather than its "
928
+ f"rows: leave {'them' if len(constant) > 1 else 'it'} out with exclude={constant!r}"
929
+ )
930
+ err = DiffError(msg + ".")
931
+ err.disjoint = True # assert_table_equal() reports it as a failure
932
+ return err
933
+
934
+
935
+ def _pairwise(x, y, by, meas, dup_rule, tolerance, rel_tol, x_names, alerts):
936
+ # An outer merge turns an integer column with a missing partner into float64;
937
+ # nullable integers keep the values exact for _classify().
938
+ for c in meas:
939
+ if _is_int(x[c]) and _is_int(y[c]):
940
+ x[c] = x[c].astype(_nullable(x[c].dtype))
941
+ y[c] = y[c].astype(_nullable(y[c].dtype))
942
+ x = _add_seq(x, by, dup_rule, "x", alerts)
943
+ y = _add_seq(y, by, dup_rule, "y", alerts)
944
+ keys = [*by, "key_seq"]
945
+ x["row_id_x"] = np.arange(1, len(x) + 1)
946
+ y["row_id_y"] = np.arange(1, len(y) + 1)
947
+
948
+ m = x.merge(y, on=keys, how="outer", suffixes=("_x", "_y"), sort=False)
949
+ matched = m["row_id_x"].notna() & m["row_id_y"].notna()
950
+ if len(x) and len(y) and not matched.any():
951
+ raise _no_overlap(x, y, x_names, by)
952
+ for c in ("row_id_x", "row_id_y"):
953
+ m[c] = m[c].astype("Int64")
954
+ only_in_y, only_in_x = m["row_id_x"].isna().to_numpy(), m["row_id_y"].isna().to_numpy()
955
+
956
+ if not meas:
957
+ status = np.select([only_in_y, only_in_x], ["only_y", "only_x"], "same")
958
+ out = m[[*keys, "row_id_x", "row_id_y"]].copy()
959
+ out["status"] = pd.Categorical(status, categories=STATUSES)
960
+ return out, keys, "rows"
961
+
962
+ parts = []
963
+ for metric in meas:
964
+ p = m[[*keys, "row_id_x", "row_id_y"]].copy()
965
+ p.insert(len(keys), "metric", metric)
966
+ vx, vy = m[f"{metric}_x"], m[f"{metric}_y"]
967
+ diff, abs_diff, rel_diff, same = _classify(vx, vy, tolerance, rel_tol)
968
+ p["value_x"] = vx.to_numpy(dtype=float, na_value=np.nan)
969
+ p["value_y"] = vy.to_numpy(dtype=float, na_value=np.nan)
970
+ p["diff"], p["abs_diff"], p["rel_diff"] = diff, abs_diff, rel_diff
971
+ status = np.select([only_in_y, only_in_x, same], ["only_y", "only_x", "same"], "changed")
972
+ p["status"] = pd.Categorical(status, categories=STATUSES)
973
+ parts.append(p)
974
+ out = pd.concat(parts, ignore_index=True).drop(columns="key_seq")
975
+ return out, [*by, "metric"], "cells"
976
+
977
+
978
+ def _multiset(values: pd.Series) -> tuple:
979
+ """The values of one group as a sorted tuple; missing values last, as NaN.
980
+
981
+ Integers stay Python ints, so values above 2**53 that differ stay different.
982
+ """
983
+ if _is_int(values):
984
+ present = sorted(int(v) for v in values.dropna().tolist())
985
+ return (*present, *([math.nan] * int(values.isna().sum())))
986
+ v = values.to_numpy(dtype=float, na_value=np.nan)
987
+ v = np.where(v == 0, 0.0, v) # -0.0 and 0.0 are the same value
988
+ return tuple(np.sort(v).tolist()) # np.sort puts NaN last
989
+
990
+
991
+ def _same_multiset(a: tuple, b: tuple) -> bool:
992
+ if len(a) != len(b):
993
+ return False
994
+ return all(u == v or (u != u and v != v) for u, v in zip(a, b, strict=True))
995
+
996
+
997
+ def _aggregate(x, y, by, meas, x_names):
998
+ def groups(df, side):
999
+ g = df.groupby(by, dropna=False, sort=False)
1000
+ out = g[meas].agg(_multiset)
1001
+ out.columns = [f"{m}_{side}" for m in meas]
1002
+ out.insert(0, f"n_rows_{side}", g.size())
1003
+ return out.reset_index()
1004
+
1005
+ gx, gy = groups(x, "x"), groups(y, "y")
1006
+ m = gx.merge(gy, on=by, how="outer", sort=False)
1007
+ in_x, in_y = m["n_rows_x"].notna().to_numpy(), m["n_rows_y"].notna().to_numpy()
1008
+ if len(x) and len(y) and not (in_x & in_y).any():
1009
+ raise _no_overlap(x, y, x_names, by)
1010
+ for c in ("n_rows_x", "n_rows_y"):
1011
+ m[c] = m[c].astype("Int64")
1012
+
1013
+ parts = []
1014
+ for metric in meas:
1015
+ p = m[[*by, "n_rows_x", "n_rows_y"]].copy()
1016
+ p.insert(len(by), "metric", metric)
1017
+ vx, vy = m[f"{metric}_x"], m[f"{metric}_y"]
1018
+ p["values_x"], p["values_y"] = vx.to_numpy(), vy.to_numpy()
1019
+ same = np.array(
1020
+ [a == a and b == b and _same_multiset(a, b) for a, b in zip(vx, vy, strict=True)],
1021
+ dtype=bool,
1022
+ )
1023
+ status = np.select([~in_x, ~in_y, same], ["only_y", "only_x", "same"], "changed")
1024
+ p["status"] = pd.Categorical(status, categories=STATUSES)
1025
+ parts.append(p)
1026
+ return pd.concat(parts, ignore_index=True)
1027
+
1028
+
1029
+ def diff_summary(d: pd.DataFrame) -> pd.DataFrame:
1030
+ """Condense a :func:`diff_table` result per measure."""
1031
+ _check_result(d)
1032
+ groups = d["metric"] if "metric" in d else pd.Series("<rows>", index=d.index, name="metric")
1033
+ counts = pd.crosstab(groups, d["status"]).reindex(columns=STATUSES, fill_value=0)
1034
+ counts.columns = [f"n_{s}" for s in STATUSES]
1035
+ counts.insert(0, "n", counts.sum(axis=1))
1036
+ if "abs_diff" in d:
1037
+ g = d.groupby("metric")
1038
+ stats = pd.DataFrame(
1039
+ {
1040
+ "max_abs_diff": g["abs_diff"].max(),
1041
+ "mean_abs_diff": g["abs_diff"].mean(),
1042
+ "rmse": g["abs_diff"].apply(lambda s: float(np.sqrt((s.dropna() ** 2).mean()))),
1043
+ "max_rel_diff": g["rel_diff"].max(),
1044
+ }
1045
+ )
1046
+ counts = counts.join(stats)
1047
+ counts.columns.name = None
1048
+ counts.index.name = "metric"
1049
+ return counts.reset_index()
1050
+
1051
+
1052
+ def _result_ids(d: pd.DataFrame) -> list[str]:
1053
+ """The key columns of a diff_table() result. User columns are normalized to
1054
+ upper case; internal ones hold a lowercase letter, and ``row_number`` is the one
1055
+ internal key. Read off the names, so a filtered or re-sorted result still works."""
1056
+ return [c for c in d.columns if not re.search("[a-z]", str(c)) or c == "row_number"]
1057
+
1058
+
1059
+ def _result_shape(d: pd.DataFrame) -> str:
1060
+ """The shape of a diff_table() result, from ``attrs`` or, when a pandas operation
1061
+ dropped them, from its columns."""
1062
+ shape = d.attrs.get("shape")
1063
+ if shape:
1064
+ return shape
1065
+ if "n_rows_x" in d:
1066
+ return "groups"
1067
+ return "cells" if "abs_diff" in d else "rows"
1068
+
1069
+
1070
+ def is_matching(
1071
+ d: pd.DataFrame, *, tolerance: float | None = None, rel_tol: float | None = None
1072
+ ) -> bool:
1073
+ """``True`` when every row of a :func:`diff_table` result is ``same``: no value
1074
+ changed and no row is found in only one table. Two empty tables match.
1075
+
1076
+ With ``tolerance`` or ``rel_tol`` the values are judged again at those thresholds,
1077
+ looser or stricter, from the stored ``abs_diff`` and ``rel_diff``, by
1078
+ :func:`diff_table`'s rule: equal or both-missing values always match, and the
1079
+ tolerances apply to finite pairs only. When only one is given, the other is 0.
1080
+ A row found in only one table never matches.
1081
+ """
1082
+ _check_result(d)
1083
+ _check_threshold(tolerance, "tolerance", allow_none=True)
1084
+ _check_threshold(rel_tol, "rel_tol", allow_none=True)
1085
+ status = d["status"].astype(str).to_numpy()
1086
+ if tolerance is None and rel_tol is None:
1087
+ return bool((status == "same").all())
1088
+ tolerance = 0.0 if tolerance is None else float(tolerance)
1089
+ rel_tol = 0.0 if rel_tol is None else float(rel_tol)
1090
+
1091
+ shape = _result_shape(d)
1092
+ # The multisets of "aggregate" are compared exactly and hold no difference to
1093
+ # re-judge, as diff_table() refuses a tolerance with them.
1094
+ if shape == "groups" and (tolerance > 0 or rel_tol > 0):
1095
+ raise DiffError(
1096
+ 'A result of duplicates = "aggregate" compares values exactly; '
1097
+ "`tolerance` and `rel_tol` must be 0."
1098
+ )
1099
+ if shape != "cells":
1100
+ return bool((status == "same").all())
1101
+
1102
+ joined = np.isin(status, ["same", "changed"])
1103
+ # rel_diff is defined where both values are finite, which is where the
1104
+ # tolerances apply. Elsewhere a value matched by equality or by being missing on
1105
+ # both sides, which no threshold changes.
1106
+ abs_diff = d["abs_diff"].to_numpy(dtype=float, na_value=np.nan)
1107
+ rel_diff = d["rel_diff"].to_numpy(dtype=float, na_value=np.nan)
1108
+ finite = ~np.isnan(rel_diff)
1109
+ with np.errstate(invalid="ignore"):
1110
+ close = (abs_diff <= tolerance) | (rel_diff <= rel_tol)
1111
+ same = joined & np.where(finite, close, status == "same")
1112
+ return bool(same.all())
1113
+
1114
+
1115
+ # --- diff_explain -----------------------------------------------------------------
1116
+
1117
+ _AMONG = ("all", "joined")
1118
+ _OVERVIEW = ["column", "n_values", "same_only", "differ_only", "mixed", "separates", "identifies"]
1119
+
1120
+
1121
+ def diff_explain(d: pd.DataFrame, column: str | None = None, *, among: str = "all") -> pd.DataFrame:
1122
+ """Which values of the key columns separate the rows that match from those that
1123
+ do not.
1124
+
1125
+ With no ``column``, one row per key column (and per ``metric`` when the result
1126
+ has measures): ``column``, ``n_values``, and how many of its values are
1127
+ ``same_only`` (every record holding it matches), ``differ_only`` (none does) or
1128
+ ``mixed``. A column ``separates`` when no value is mixed; one with a different
1129
+ value on every record ``identifies`` and is ranked last. The others come first,
1130
+ fewest mixed values first, then fewest values.
1131
+
1132
+ With a ``column`` (a key column in either spelling, or ``"metric"``), one row per
1133
+ value of it: ``n``, ``n_same``, ``n_changed``, ``n_only_x``, ``n_only_y`` and
1134
+ ``class``, most differences first.
1135
+
1136
+ For a key column the unit is the row, which matches only when every one of its
1137
+ measures does; with ``duplicates="aggregate"`` it is the key; for ``metric`` it
1138
+ is the value. ``among="joined"`` counts only rows found in both tables, so the
1139
+ explanation is of the changed values alone.
1140
+ """
1141
+ _check_result(d)
1142
+ if among not in _AMONG:
1143
+ raise DiffError(f"`among` must be one of {_AMONG}, not {among!r}.")
1144
+ ids = _result_ids(d)
1145
+ has_metric = "metric" in d
1146
+ candidates = ids + (["metric"] if has_metric else [])
1147
+
1148
+ if column is not None:
1149
+ if not isinstance(column, str):
1150
+ raise DiffError("`column` must be None or a single column name.")
1151
+ given = column
1152
+ column = column if column in candidates else _canon(column)
1153
+ if column not in candidates:
1154
+ raise DiffError(
1155
+ "`column` must be a key column of the result"
1156
+ + (' or "metric"' if has_metric else "")
1157
+ + f"; got {given}; the key columns are {_join(ids)}."
1158
+ )
1159
+
1160
+ units = _explain_units(d, ids, _result_shape(d))
1161
+ cells = (
1162
+ pd.DataFrame({"metric": d["metric"].astype(str), "status": d["status"].astype(str)})
1163
+ if has_metric
1164
+ else None
1165
+ )
1166
+ if among == "joined":
1167
+ units = units[units["status"].isin(["same", "changed"])]
1168
+ if cells is not None:
1169
+ cells = cells[cells["status"].isin(["same", "changed"])]
1170
+
1171
+ def records(col):
1172
+ return cells if col == "metric" else units
1173
+
1174
+ if column is not None:
1175
+ return _explain_values(records(column), column)
1176
+
1177
+ rows = []
1178
+ for col in candidates:
1179
+ r = records(col)
1180
+ v = _explain_values(r, col)
1181
+ rows.append(
1182
+ {
1183
+ "column": col,
1184
+ "n_values": len(v),
1185
+ "same_only": int((v["class"] == "same").sum()),
1186
+ "differ_only": int((v["class"] == "differ").sum()),
1187
+ "mixed": int((v["class"] == "mixed").sum()),
1188
+ "separates": not (v["class"] == "mixed").any(),
1189
+ # metric is never an identifier: it repeats on every row.
1190
+ "identifies": col != "metric" and len(v) > 1 and len(v) == len(r),
1191
+ }
1192
+ )
1193
+ out = pd.DataFrame(rows, columns=_OVERVIEW)
1194
+ out = out.astype(
1195
+ {
1196
+ "n_values": "int64",
1197
+ "same_only": "int64",
1198
+ "differ_only": "int64",
1199
+ "mixed": "int64",
1200
+ "separates": bool,
1201
+ "identifies": bool,
1202
+ }
1203
+ )
1204
+ return out.sort_values(["identifies", "mixed", "n_values"], kind="stable", ignore_index=True)
1205
+
1206
+
1207
+ def _check_one_comparison(d: pd.DataFrame, ids: list[str], unit: list[str], shape: str):
1208
+ """Refuse a frame holding several diff_table() results, such as two joined with
1209
+ ``pd.concat``: row ids restart at 1 in each, so rows of different comparisons
1210
+ would be read as one (decided 2026-11-13, #41)."""
1211
+ record = unit + (["metric"] if "metric" in d else [])
1212
+ clash = bool(d.duplicated(record).any())
1213
+ if not clash and shape != "groups" and ids and len(d):
1214
+ keys = d.groupby(unit, dropna=False)[ids].nunique(dropna=False)
1215
+ clash = bool((keys > 1).to_numpy().any())
1216
+ if clash:
1217
+ raise DiffError(
1218
+ "`d` holds more than one comparison (results joined with pd.concat?): "
1219
+ "its row ids repeat. Explain or report each diff_table() result on its own."
1220
+ )
1221
+
1222
+
1223
+ def _explain_units(d: pd.DataFrame, ids: list[str], shape: str) -> pd.DataFrame:
1224
+ """One record per row (per key with ``"groups"``), with the status of the row as
1225
+ a whole: ``only_x`` or ``only_y`` by membership, which comes from ``row_id`` or
1226
+ ``n_rows`` and never from the value; otherwise ``changed`` when any of its
1227
+ measures changed."""
1228
+ present = "n_rows" if shape == "groups" else "row_id"
1229
+ # A row is its pair of source rows, which tells paired duplicates apart.
1230
+ unit = ids if shape == "groups" else ["row_id_x", "row_id_y"]
1231
+ _check_one_comparison(d, ids, unit, shape)
1232
+ w = d[list(dict.fromkeys(ids + unit))].copy()
1233
+ w["_in_x"] = d[f"{present}_x"].notna().to_numpy()
1234
+ w["_in_y"] = d[f"{present}_y"].notna().to_numpy()
1235
+ w["_changed"] = (d["status"].astype(str) == "changed").to_numpy()
1236
+ if len(w):
1237
+ agg = {c: "first" for c in ids if c not in unit}
1238
+ agg.update({"_in_x": "max", "_in_y": "max", "_changed": "max"})
1239
+ w = w.groupby(unit, dropna=False, sort=False).agg(agg).reset_index()
1240
+ status = np.select(
1241
+ [~w["_in_y"].astype(bool), ~w["_in_x"].astype(bool), w["_changed"].astype(bool)],
1242
+ ["only_x", "only_y", "changed"],
1243
+ "same",
1244
+ )
1245
+ out = w[ids].reset_index(drop=True)
1246
+ out["status"] = status
1247
+ return out
1248
+
1249
+
1250
+ def _explain_values(r: pd.DataFrame, col: str) -> pd.DataFrame:
1251
+ """The counts of each status per value of ``col``, with the value's class: most
1252
+ differences first, then the values that only differ before the mixed ones, since
1253
+ they are the cleaner lead."""
1254
+ s = r["status"].to_numpy()
1255
+ v = pd.DataFrame(
1256
+ {
1257
+ col: r[col].to_numpy(),
1258
+ "n": 1,
1259
+ "n_same": s == "same",
1260
+ "n_changed": s == "changed",
1261
+ "n_only_x": s == "only_x",
1262
+ "n_only_y": s == "only_y",
1263
+ }
1264
+ )
1265
+ out = (
1266
+ v.groupby(col, dropna=False, sort=False)
1267
+ .sum()
1268
+ .reset_index()
1269
+ .astype({k: "int64" for k in ["n", "n_same", "n_changed", "n_only_x", "n_only_y"]})
1270
+ )
1271
+ out["class"] = np.select(
1272
+ [out["n_same"] == out["n"], out["n_same"] == 0], ["same", "differ"], "mixed"
1273
+ )
1274
+ out["_d"] = out["n"] - out["n_same"]
1275
+ out["_r"] = out["class"].map({"differ": 0, "mixed": 1, "same": 2})
1276
+ out = out.sort_values(["_r", col], kind="stable", na_position="last")
1277
+ out = out.sort_values("_d", ascending=False, kind="stable", ignore_index=True)
1278
+ return out.drop(columns=["_d", "_r"])
1279
+
1280
+
1281
+ # --- assert_table_equal -----------------------------------------------------------
1282
+
1283
+ _SQRT_EPS = float(np.sqrt(np.finfo(float).eps))
1284
+ _UNSET = object()
1285
+
1286
+
1287
+ def assert_table_equal(
1288
+ actual: pd.DataFrame,
1289
+ expected: pd.DataFrame,
1290
+ *,
1291
+ rel_tol: float = _UNSET, # type: ignore[assignment]
1292
+ max_rows: int = 10,
1293
+ **kwargs,
1294
+ ) -> pd.DataFrame:
1295
+ """Assert that ``actual`` matches ``expected``, for use in tests.
1296
+
1297
+ Runs ``diff_table(expected, actual, ...)``, with ``expected`` as the benchmark,
1298
+ and raises :class:`AssertionError` with :func:`diff_summary` and the first
1299
+ ``max_rows`` records that differ. Tables that share no key fail too. Other
1300
+ keyword arguments go to :func:`diff_table`.
1301
+
1302
+ ``rel_tol`` defaults to ``sqrt(eps)`` (about 1.5e-8), as pytest and testthat
1303
+ compare floats, where :func:`diff_table` is exact; with
1304
+ ``duplicates="aggregate"``, which compares exactly, it defaults to 0. Repeated
1305
+ keys of rows with measures are a :class:`DiffError` unless ``duplicates`` is
1306
+ given: an assertion must not decide identity by arrival order. Repeated whole
1307
+ rows are paired, since they are identical.
1308
+
1309
+ Returns the comparison when the tables match.
1310
+ """
1311
+ if isinstance(max_rows, bool) or not isinstance(max_rows, (int, np.integer)) or max_rows < 0:
1312
+ raise DiffError("`max_rows` must be a single whole number >= 0.")
1313
+ duplicates = kwargs.get("duplicates")
1314
+ if rel_tol is _UNSET:
1315
+ rel_tol = 0.0 if duplicates == "aggregate" else _SQRT_EPS
1316
+
1317
+ caught: list = []
1318
+ refused = False
1319
+ try:
1320
+ with warnings.catch_warnings(record=True) as caught:
1321
+ warnings.simplefilter("always")
1322
+ d = diff_table(expected, actual, rel_tol=rel_tol, **kwargs)
1323
+ # Pairing repeated keys of rows with measures decides identity by arrival
1324
+ # order, which an assertion must not do unless the caller accepted it.
1325
+ if duplicates is None and d.attrs.get("paired") and d.attrs.get("measures"):
1326
+ refused = True
1327
+ raise DiffError(
1328
+ "`actual` or `expected` has repeated keys, which diff_table() pairs in "
1329
+ "arrival order.\nassert_table_equal() does not pair duplicate keys; pass "
1330
+ 'duplicates="disambiguate" or "aggregate" to accept them.'
1331
+ )
1332
+ except DiffError as e:
1333
+ if getattr(e, "disjoint", False):
1334
+ raise AssertionError(
1335
+ f"`actual` does not match `expected` (as `y` and `x`): {e}"
1336
+ ) from None
1337
+ raise
1338
+ finally:
1339
+ # Passed on whatever happens: a warning (an `exclude` typo, a dropped column)
1340
+ # often explains the failure that follows (#45).
1341
+ for w in caught:
1342
+ if refused and issubclass(w.category, _PairingWarning):
1343
+ continue # the DiffError says it
1344
+ warnings.warn_explicit(w.message, w.category, w.filename, w.lineno)
1345
+
1346
+ off = d[d["status"] != "same"]
1347
+ if not len(off):
1348
+ return d
1349
+ shown = off.head(max_rows)
1350
+ msg = (
1351
+ f"`actual` does not match `expected`: {len(off):,} of {len(d):,} records differ.\n\n"
1352
+ + diff_summary(d).to_string(index=False)
1353
+ )
1354
+ if len(shown):
1355
+ lead = f"First {len(shown)} records that differ:\n" if len(off) > len(shown) else ""
1356
+ msg += "\n\n" + lead + shown.to_string(index=False)
1357
+ raise AssertionError(msg)