dazdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dazdiff/__init__.py +34 -0
- dazdiff/__main__.py +5 -0
- dazdiff/cli.py +248 -0
- dazdiff/core.py +1357 -0
- dazdiff/py.typed +0 -0
- dazdiff/report.py +483 -0
- dazdiff-0.1.0.dist-info/METADATA +167 -0
- dazdiff-0.1.0.dist-info/RECORD +11 -0
- dazdiff-0.1.0.dist-info/WHEEL +4 -0
- dazdiff-0.1.0.dist-info/entry_points.txt +2 -0
- dazdiff-0.1.0.dist-info/licenses/LICENSE +21 -0
dazdiff/py.typed
ADDED
|
File without changes
|
dazdiff/report.py
ADDED
|
@@ -0,0 +1,483 @@
|
|
|
1
|
+
"""A text report of a comparison, after the R package's ``diff_report()``.
|
|
2
|
+
|
|
3
|
+
The layout follows the summary of the Python package diffly (Quantco): a section
|
|
4
|
+
each for the columns, the rows and the measures, and a picture of how the rows of
|
|
5
|
+
the two tables align. The report is computed first, as plain data, and drawn by
|
|
6
|
+
:meth:`DiffReport.format`, so the numbers can be tested apart from the drawing.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import math
|
|
12
|
+
import unicodedata
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
import pandas as pd
|
|
17
|
+
|
|
18
|
+
from .core import DiffError, _explain_units, _result_ids, _result_shape, diff_summary
|
|
19
|
+
|
|
20
|
+
_TITLE = "dazdiff report"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass
|
|
24
|
+
class DiffReport:
|
|
25
|
+
"""What :func:`diff_report` found; ``str(report)`` draws it in Unicode and
|
|
26
|
+
``report.format(unicode=False)`` in ASCII."""
|
|
27
|
+
|
|
28
|
+
x_name: str
|
|
29
|
+
y_name: str
|
|
30
|
+
mode: str
|
|
31
|
+
tolerance: dict
|
|
32
|
+
shape: str
|
|
33
|
+
unit: str
|
|
34
|
+
identity: list[str]
|
|
35
|
+
measures: list[str]
|
|
36
|
+
dropped: list[str]
|
|
37
|
+
excluded: list[str]
|
|
38
|
+
ignored: list[str]
|
|
39
|
+
truncated: dict
|
|
40
|
+
paired: bool
|
|
41
|
+
equal: bool
|
|
42
|
+
rows: dict
|
|
43
|
+
by_measure: pd.DataFrame | None
|
|
44
|
+
only_x: pd.DataFrame = field(repr=False)
|
|
45
|
+
only_y: pd.DataFrame = field(repr=False)
|
|
46
|
+
# Distinct keys found in one table only: "(...and N more)" counts these, not
|
|
47
|
+
# rows, since a key can repeat.
|
|
48
|
+
n_keys_only_x: int = 0
|
|
49
|
+
n_keys_only_y: int = 0
|
|
50
|
+
|
|
51
|
+
def format(self, unicode: bool = True) -> str:
|
|
52
|
+
"""The report as text: box-drawing characters, or ASCII with
|
|
53
|
+
``unicode=False``."""
|
|
54
|
+
if not isinstance(unicode, bool):
|
|
55
|
+
raise DiffError("`unicode` must be True or False.")
|
|
56
|
+
return "\n".join(_draw(self, _GLYPHS if unicode else _ASCII))
|
|
57
|
+
|
|
58
|
+
def __str__(self) -> str:
|
|
59
|
+
return self.format(unicode=True)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def diff_report(
|
|
63
|
+
d: pd.DataFrame, x_name: str = "x", y_name: str = "y", *, sample_k: int = 5
|
|
64
|
+
) -> DiffReport:
|
|
65
|
+
"""Lay out a :func:`diff_table` result as a bounded text report.
|
|
66
|
+
|
|
67
|
+
The identity and mode; the columns compared, dropped, excluded, ignored and
|
|
68
|
+
truncated; a picture of how the rows of the two tables align; per measure that
|
|
69
|
+
changed, its match rate and largest differences; and up to ``sample_k`` keys
|
|
70
|
+
found in only one table. ``x_name`` and ``y_name`` label the two tables.
|
|
71
|
+
"""
|
|
72
|
+
if not isinstance(d, pd.DataFrame) or "status" not in d:
|
|
73
|
+
raise DiffError("`d` must be a result of diff_table().")
|
|
74
|
+
for arg, v in (("x_name", x_name), ("y_name", y_name)):
|
|
75
|
+
if not isinstance(v, str) or not v:
|
|
76
|
+
raise DiffError(f"`{arg}` must be a single non-empty string.")
|
|
77
|
+
if isinstance(sample_k, bool) or not isinstance(sample_k, (int, np.integer)) or sample_k < 0:
|
|
78
|
+
raise DiffError("`sample_k` must be a single whole number >= 0.")
|
|
79
|
+
|
|
80
|
+
attrs = d.attrs
|
|
81
|
+
shape = _result_shape(d)
|
|
82
|
+
name_map = attrs.get("name_map", {})
|
|
83
|
+
|
|
84
|
+
def label(c):
|
|
85
|
+
return (name_map.get(c) or {}).get("x") or c
|
|
86
|
+
|
|
87
|
+
ids = _result_ids(d)
|
|
88
|
+
measures: list[str] = []
|
|
89
|
+
if "metric" in d:
|
|
90
|
+
present = list(dict.fromkeys(d["metric"].astype(str)))
|
|
91
|
+
order = [m for m in attrs.get("measures", []) if m in present]
|
|
92
|
+
measures = order + [m for m in present if m not in order]
|
|
93
|
+
|
|
94
|
+
# Rows: one per source row (per key for "groups"), changed if any measure is.
|
|
95
|
+
# Membership comes from row_id or n_rows, never from the value.
|
|
96
|
+
units = _explain_units(d, ids, shape)
|
|
97
|
+
status = units["status"].to_numpy()
|
|
98
|
+
rows = {
|
|
99
|
+
"n_x": int((status != "only_y").sum()),
|
|
100
|
+
"n_y": int((status != "only_x").sum()),
|
|
101
|
+
"n_only_x": int((status == "only_x").sum()),
|
|
102
|
+
"n_same": int((status == "same").sum()),
|
|
103
|
+
"n_changed": int((status == "changed").sum()),
|
|
104
|
+
"n_only_y": int((status == "only_y").sum()),
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
by_measure = None
|
|
108
|
+
if measures:
|
|
109
|
+
s = diff_summary(d).set_index("metric").reindex(measures)
|
|
110
|
+
joined = s["n_same"] + s["n_changed"]
|
|
111
|
+
by_measure = pd.DataFrame(
|
|
112
|
+
{
|
|
113
|
+
"measure": [label(m) for m in measures],
|
|
114
|
+
"n_same": s["n_same"].to_numpy(),
|
|
115
|
+
"n_changed": s["n_changed"].to_numpy(),
|
|
116
|
+
"match_rate": np.where(
|
|
117
|
+
joined > 0, s["n_same"] / joined.where(joined > 0, 1), np.nan
|
|
118
|
+
),
|
|
119
|
+
}
|
|
120
|
+
)
|
|
121
|
+
if shape == "cells":
|
|
122
|
+
by_measure["max_abs_diff"] = s["max_abs_diff"].to_numpy()
|
|
123
|
+
by_measure["max_rel_diff"] = s["max_rel_diff"].to_numpy()
|
|
124
|
+
|
|
125
|
+
def keys(side):
|
|
126
|
+
keep = units["status"] == f"only_{side}"
|
|
127
|
+
return units.loc[keep, ids].drop_duplicates()
|
|
128
|
+
|
|
129
|
+
def sample(side):
|
|
130
|
+
out = keys(side).head(sample_k).reset_index(drop=True)
|
|
131
|
+
return out.rename(columns={c: label(c) for c in ids})
|
|
132
|
+
|
|
133
|
+
return DiffReport(
|
|
134
|
+
x_name=x_name,
|
|
135
|
+
y_name=y_name,
|
|
136
|
+
mode=attrs.get("mode", "benchmark"),
|
|
137
|
+
tolerance=attrs.get("tolerance", {"abs": 0.0, "rel": 0.0}),
|
|
138
|
+
shape=shape,
|
|
139
|
+
unit="keys" if shape == "groups" else "rows",
|
|
140
|
+
identity=[label(c) for c in ids],
|
|
141
|
+
measures=[label(m) for m in measures],
|
|
142
|
+
dropped=list(attrs.get("dropped", [])),
|
|
143
|
+
excluded=list(attrs.get("excluded", [])),
|
|
144
|
+
ignored=list(attrs.get("ignored", [])),
|
|
145
|
+
truncated={label(c): v for c, v in attrs.get("truncated", {}).items()},
|
|
146
|
+
paired=bool(attrs.get("paired", False)),
|
|
147
|
+
equal=bool((d["status"].astype(str) == "same").all()),
|
|
148
|
+
rows=rows,
|
|
149
|
+
by_measure=by_measure,
|
|
150
|
+
only_x=sample("x"),
|
|
151
|
+
only_y=sample("y"),
|
|
152
|
+
n_keys_only_x=len(keys("x")),
|
|
153
|
+
n_keys_only_y=len(keys("y")),
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
# --- drawing --------------------------------------------------------------------
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _draw(x: DiffReport, g: dict) -> list[str]:
|
|
161
|
+
r = x.rows
|
|
162
|
+
|
|
163
|
+
def noun(n, one, many=None):
|
|
164
|
+
return f"{_count(n)} {one if n == 1 else (many or one + 's')}"
|
|
165
|
+
|
|
166
|
+
intro = [
|
|
167
|
+
f"Identity: {', '.join(x.identity)}",
|
|
168
|
+
f"Mode: {x.mode}"
|
|
169
|
+
+ (f": {x.y_name} is tested against {x.x_name}" if x.mode == "benchmark" else ""),
|
|
170
|
+
]
|
|
171
|
+
if x.paired and r["n_x"] + r["n_y"] > 0:
|
|
172
|
+
intro.append("Repeated keys were paired in arrival order (key_seq).")
|
|
173
|
+
if x.shape == "groups":
|
|
174
|
+
intro.append("Repeated keys: each key's values are compared as a multiset.")
|
|
175
|
+
tol = x.tolerance or {}
|
|
176
|
+
if tol.get("abs", 0) > 0 or tol.get("rel", 0) > 0:
|
|
177
|
+
intro.append(f"Tolerance: {_num(tol['abs'], 7)} absolute, {_num(tol['rel'], 7)} relative.")
|
|
178
|
+
|
|
179
|
+
columns = [
|
|
180
|
+
f"Compared {noun(len(x.identity) + len(x.measures), 'column')}: "
|
|
181
|
+
f"{noun(len(x.identity), 'key')}, {noun(len(x.measures), 'measure')}."
|
|
182
|
+
]
|
|
183
|
+
if x.dropped:
|
|
184
|
+
columns.append(f"Only in {x.y_name}, dropped: {', '.join(x.dropped)}.")
|
|
185
|
+
if x.excluded:
|
|
186
|
+
columns.append(f"Excluded: {', '.join(x.excluded)}.")
|
|
187
|
+
if x.ignored:
|
|
188
|
+
columns.append(f"In neither the identity nor the measures: {', '.join(x.ignored)}.")
|
|
189
|
+
if x.truncated:
|
|
190
|
+
columns.append(f"Truncated by the cast to {x.x_name}'s types:")
|
|
191
|
+
for col, pairs in x.truncated.items():
|
|
192
|
+
shown = ", ".join(f"{_num(a, 7)} -> {_num(b, 7)}" for a, b in pairs[:3])
|
|
193
|
+
columns.append(f" {col}: {noun(len(pairs), 'value')} ({shown})")
|
|
194
|
+
notes = bool(x.dropped or x.excluded or x.ignored or x.truncated)
|
|
195
|
+
|
|
196
|
+
body = _indent(intro)
|
|
197
|
+
if x.equal:
|
|
198
|
+
if r["n_x"] + r["n_y"] == 0:
|
|
199
|
+
verdict = f"--- {x.x_name} and {x.y_name} are both empty ---"
|
|
200
|
+
else:
|
|
201
|
+
verdict = (
|
|
202
|
+
f"--- {x.x_name} and {x.y_name} match exactly "
|
|
203
|
+
f"({noun(r['n_x'], x.unit[:-1], x.unit)}, {noun(len(x.measures), 'measure')}) ---"
|
|
204
|
+
)
|
|
205
|
+
body += ["", *_indent([verdict])]
|
|
206
|
+
if notes:
|
|
207
|
+
body += _section("Columns", columns, g)
|
|
208
|
+
else:
|
|
209
|
+
body += _section("Columns", columns, g)
|
|
210
|
+
diagram = _row_diagram(x, g)
|
|
211
|
+
body += _section("Rows", _row_counts(x) + (["", *diagram] if diagram else []), g)
|
|
212
|
+
if x.measures:
|
|
213
|
+
body += _section("Measures", _measure_lines(x, g), g)
|
|
214
|
+
body += _sample_lines(x, "x", g) + _sample_lines(x, "y", g)
|
|
215
|
+
|
|
216
|
+
body = [line.rstrip() for line in body]
|
|
217
|
+
width = max(40, *(_width(line) for line in body))
|
|
218
|
+
left = (width - 2 - len(_TITLE)) // 2
|
|
219
|
+
header = [
|
|
220
|
+
g["htl"] + g["hh"] * (width - 2) + g["htr"],
|
|
221
|
+
g["hv"] + " " * left + _TITLE + " " * (width - 2 - len(_TITLE) - left) + g["hv"],
|
|
222
|
+
g["hbl"] + g["hh"] * (width - 2) + g["hbr"],
|
|
223
|
+
]
|
|
224
|
+
return header + body
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _indent(lines: list[str], n: int = 3) -> list[str]:
|
|
228
|
+
return [" " * n + line if line else line for line in lines]
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def _section(title: str, lines: list[str], g: dict) -> list[str]:
|
|
232
|
+
return ["", f" {title}", " " + g["under"] * len(title), *_indent(lines)]
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _center(s: str, width: int) -> str:
|
|
236
|
+
gap = width - _width(s)
|
|
237
|
+
left = gap // 2
|
|
238
|
+
return " " * left + s + " " * (gap - left)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _row_counts(x: DiffReport) -> list[str]:
|
|
242
|
+
"""``x rows (+33.33%) y rows`` over the two counts, each column centered."""
|
|
243
|
+
r = x.rows
|
|
244
|
+
if r["n_x"] == 0:
|
|
245
|
+
change = ""
|
|
246
|
+
elif r["n_y"] == r["n_x"]:
|
|
247
|
+
change = "(no change)"
|
|
248
|
+
else:
|
|
249
|
+
sign = "+" if r["n_y"] > r["n_x"] else "-"
|
|
250
|
+
change = f"({sign}{_pct(abs(r['n_y'] / r['n_x'] - 1), clamp=False)})"
|
|
251
|
+
top = [f"{x.x_name} {x.unit}", "", f"{x.y_name} {x.unit}"]
|
|
252
|
+
bottom = [_count(r["n_x"]), change, _count(r["n_y"])]
|
|
253
|
+
widths = [
|
|
254
|
+
max(_width(a), _width(b)) + extra
|
|
255
|
+
for a, b, extra in zip(top, bottom, (0, 4, 0), strict=False)
|
|
256
|
+
]
|
|
257
|
+
return [
|
|
258
|
+
"".join(_center(s, w) for s, w in zip(top, widths, strict=False)),
|
|
259
|
+
"".join(_center(s, w) for s, w in zip(bottom, widths, strict=False)),
|
|
260
|
+
]
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _row_diagram(x: DiffReport, g: dict) -> list[str]:
|
|
264
|
+
"""Two stacks of boxes, x on the left and y on the right, offset so that the rows
|
|
265
|
+
found in both tables sit side by side: x's own rows above them, y's own below.
|
|
266
|
+
Each band is labelled with its count and share: of x's rows for the rows only in
|
|
267
|
+
x, of the joined rows for ``same`` and ``changed``, of y's rows for the rows only
|
|
268
|
+
in y."""
|
|
269
|
+
r = x.rows
|
|
270
|
+
n_joined = r["n_same"] + r["n_changed"]
|
|
271
|
+
# A filtered result can have no joined band.
|
|
272
|
+
if n_joined == 0:
|
|
273
|
+
return []
|
|
274
|
+
bands = {
|
|
275
|
+
"only_x": (r["n_only_x"], f"only in {x.x_name}", r["n_only_x"] / max(r["n_x"], 1)),
|
|
276
|
+
"same": (r["n_same"], "same", r["n_same"] / n_joined),
|
|
277
|
+
"changed": (r["n_changed"], "changed", r["n_changed"] / n_joined),
|
|
278
|
+
"only_y": (r["n_only_y"], f"only in {x.y_name}", r["n_only_y"] / max(r["n_y"], 1)),
|
|
279
|
+
}
|
|
280
|
+
bands = {k: v for k, v in bands.items() if v[0] > 0}
|
|
281
|
+
counts = {k: _count(v[0]) for k, v in bands.items()}
|
|
282
|
+
kinds = {k: v[1] for k, v in bands.items()}
|
|
283
|
+
pcts = {k: f"({_pct(v[2])})" for k, v in bands.items()}
|
|
284
|
+
wc = max(len(c) for c in counts.values())
|
|
285
|
+
wk = max(_width(k) for k in kinds.values())
|
|
286
|
+
wp = max(len(p) for p in pcts.values())
|
|
287
|
+
labels = {
|
|
288
|
+
k: f"{counts[k]:>{wc}} {kinds[k] + ' ' * (wk - _width(kinds[k]))} {pcts[k]:>{wp}}"
|
|
289
|
+
for k in bands
|
|
290
|
+
}
|
|
291
|
+
label_width = max(_width(v) for v in labels.values())
|
|
292
|
+
|
|
293
|
+
def strip(left, mid, right, fill):
|
|
294
|
+
return left + mid.join([fill] * 5) + right
|
|
295
|
+
|
|
296
|
+
top = strip(g["btl"], g["btt"], g["btr"], g["bh"])
|
|
297
|
+
sep = strip(g["bml"], g["bmx"], g["bmr"], g["bmh"])
|
|
298
|
+
bottom = strip(g["bbl"], g["bbt"], g["bbr"], g["bh"])
|
|
299
|
+
|
|
300
|
+
def cells(ch):
|
|
301
|
+
return strip(g["bv"], g["biv"], g["bv"], ch)
|
|
302
|
+
|
|
303
|
+
blank = " " * _width(top)
|
|
304
|
+
dashes = g["dash"] * 3
|
|
305
|
+
run = g["dash"] * (label_width + 4)
|
|
306
|
+
|
|
307
|
+
def label(band):
|
|
308
|
+
return " " + labels[band] + " " * (label_width - _width(labels[band])) + " "
|
|
309
|
+
|
|
310
|
+
out = []
|
|
311
|
+
if "only_x" in bands:
|
|
312
|
+
out.append(top)
|
|
313
|
+
out.append(cells("-") + " " + blank + label("only_x"))
|
|
314
|
+
out.append((sep if "only_x" in bands else top) + dashes + top + run + g["open"])
|
|
315
|
+
if "same" in bands:
|
|
316
|
+
out.append(cells(" ") + " = " + cells(" ") + label("same") + g["bar"])
|
|
317
|
+
if "same" in bands and "changed" in bands:
|
|
318
|
+
out.append(sep + dashes + sep + run + g["tee"] + " " + _count(n_joined) + " joined")
|
|
319
|
+
if "changed" in bands:
|
|
320
|
+
out.append(cells(" ") + " " + g["ne"] + " " + cells(" ") + label("changed") + g["bar"])
|
|
321
|
+
out.append(bottom + dashes + (sep if "only_y" in bands else bottom) + run + g["close"])
|
|
322
|
+
if "only_y" in bands:
|
|
323
|
+
out.append(blank + " " + cells("+") + label("only_y"))
|
|
324
|
+
out.append(blank + " " + bottom)
|
|
325
|
+
return out
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def _measure_lines(x: DiffReport, g: dict) -> list[str]:
|
|
329
|
+
m = x.by_measure
|
|
330
|
+
changed = m[m["n_changed"] > 0]
|
|
331
|
+
rest = len(m) - len(changed)
|
|
332
|
+
if not len(changed):
|
|
333
|
+
return [f"Every measure matches on every row found in both tables ({_count(len(m))})."]
|
|
334
|
+
cells = x.shape == "cells"
|
|
335
|
+
header = ["Measure", "Match rate"] + (["Max |diff|", "Max rel diff"] if cells else [])
|
|
336
|
+
rows = []
|
|
337
|
+
for _, c in changed.iterrows():
|
|
338
|
+
row = [str(c["measure"]), _pct(c["match_rate"])]
|
|
339
|
+
if cells:
|
|
340
|
+
row += [_size(c["max_abs_diff"]), _size(c["max_rel_diff"])]
|
|
341
|
+
rows.append(row)
|
|
342
|
+
align = ["left", "right"] + (["right", "right"] if cells else [])
|
|
343
|
+
out = _table(header, rows, align, g)
|
|
344
|
+
if rest:
|
|
345
|
+
what = "other measure matches" if rest == 1 else "other measures match"
|
|
346
|
+
out.append(f"{_count(rest)} {what} on every row found in both tables.")
|
|
347
|
+
return out
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def _sample_lines(x: DiffReport, side: str, g: dict) -> list[str]:
|
|
351
|
+
keys = x.only_x if side == "x" else x.only_y
|
|
352
|
+
n = x.n_keys_only_x if side == "x" else x.n_keys_only_y
|
|
353
|
+
if not len(keys) or not n:
|
|
354
|
+
return []
|
|
355
|
+
max_cols = 5
|
|
356
|
+
shown = list(keys.columns[:max_cols])
|
|
357
|
+
more_cols = len(keys.columns) > max_cols
|
|
358
|
+
header = shown + (["..."] if more_cols else [])
|
|
359
|
+
rows = [
|
|
360
|
+
[_cell(keys[c].iloc[i]) for c in shown] + (["..."] if more_cols else [])
|
|
361
|
+
for i in range(len(keys))
|
|
362
|
+
]
|
|
363
|
+
name = x.x_name if side == "x" else x.y_name
|
|
364
|
+
lines = _table(header, rows, ["left"] * len(header), g)
|
|
365
|
+
if n > len(keys):
|
|
366
|
+
lines.append(f"(...and {_count(n - len(keys))} more)")
|
|
367
|
+
return _section(f"{'Keys' if x.unit == 'keys' else 'Rows'} only in {name}", lines, g)
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def _table(header: list[str], rows: list[list[str]], align: list[str], g: dict) -> list[str]:
|
|
371
|
+
widths = [max(_width(header[j]), *(_width(r[j]) for r in rows)) for j in range(len(header))]
|
|
372
|
+
|
|
373
|
+
def pad(s, w, a):
|
|
374
|
+
gap = " " * (w - _width(s))
|
|
375
|
+
return gap + s if a == "right" else s + gap
|
|
376
|
+
|
|
377
|
+
def rule(left, mid, right):
|
|
378
|
+
return left + mid.join(g["th"] * (w + 2) for w in widths) + right
|
|
379
|
+
|
|
380
|
+
def line(cells):
|
|
381
|
+
return (
|
|
382
|
+
g["tv"]
|
|
383
|
+
+ g["tv"].join(
|
|
384
|
+
f" {pad(c, w, a)} " for c, w, a in zip(cells, widths, align, strict=False)
|
|
385
|
+
)
|
|
386
|
+
+ g["tv"]
|
|
387
|
+
)
|
|
388
|
+
|
|
389
|
+
out = [rule(g["ttl"], g["ttt"], g["ttr"]), line(header), rule(g["tml"], g["tmx"], g["tmr"])]
|
|
390
|
+
out += [line(r) for r in rows]
|
|
391
|
+
return [*out, rule(g["tbl"], g["tbt"], g["tbr"])]
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
# --- formatting -----------------------------------------------------------------
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def _width(s: str) -> int:
|
|
398
|
+
"""Display width: wide East Asian characters take two cells, combining marks
|
|
399
|
+
none."""
|
|
400
|
+
return sum(
|
|
401
|
+
0 if unicodedata.combining(ch) else 2 if unicodedata.east_asian_width(ch) in "WF" else 1
|
|
402
|
+
for ch in s
|
|
403
|
+
)
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def _count(n) -> str:
|
|
407
|
+
return f"{int(n):,}"
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
def _pct(p, clamp: bool = True) -> str:
|
|
411
|
+
"""A share as a percentage with two decimals. With ``clamp``, it reads 0% or 100%
|
|
412
|
+
only when it is exactly that: one changed row in a billion is not 0.00%."""
|
|
413
|
+
if p is None or (isinstance(p, float) and math.isnan(p)):
|
|
414
|
+
return "-"
|
|
415
|
+
p = 100 * float(p)
|
|
416
|
+
if clamp and 0 < p < 100:
|
|
417
|
+
p = min(max(p, 0.01), 99.99)
|
|
418
|
+
return f"{p:.2f}%"
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def _num(v, digits: int) -> str:
|
|
422
|
+
"""A number as R's ``format(v, digits=)`` writes it: ``digits`` significant
|
|
423
|
+
digits, trailing zeros dropped, fixed notation unless scientific is shorter."""
|
|
424
|
+
v = float(v)
|
|
425
|
+
if math.isnan(v):
|
|
426
|
+
return "NA"
|
|
427
|
+
if math.isinf(v):
|
|
428
|
+
return "Inf" if v > 0 else "-Inf"
|
|
429
|
+
if v == 0:
|
|
430
|
+
return "0"
|
|
431
|
+
mantissa, exp = f"{v:.{digits - 1}e}".split("e")
|
|
432
|
+
if "." in mantissa:
|
|
433
|
+
mantissa = mantissa.rstrip("0").rstrip(".")
|
|
434
|
+
sci = f"{mantissa}e{exp[0]}{exp[1:].zfill(2)}"
|
|
435
|
+
# The significant digits the rounded value needs set the decimals; fixed notation
|
|
436
|
+
# then writes the value itself, keeping every digit before the point.
|
|
437
|
+
sig = len(mantissa.lstrip("-").replace(".", ""))
|
|
438
|
+
decimals = max(0, sig - 1 - int(exp))
|
|
439
|
+
fixed = f"{v:.{decimals}f}"
|
|
440
|
+
return fixed if len(fixed) <= len(sci) else sci
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def _size(v) -> str:
|
|
444
|
+
return "-" if v is None or pd.isna(v) else _num(v, 4)
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
def _cell(v) -> str:
|
|
448
|
+
"""One key value: text quoted, numbers with their own digits."""
|
|
449
|
+
if v is None or (not isinstance(v, (list, tuple)) and pd.isna(v)):
|
|
450
|
+
return "NA"
|
|
451
|
+
if isinstance(v, str):
|
|
452
|
+
return f'"{v}"'
|
|
453
|
+
if isinstance(v, (float, np.floating)):
|
|
454
|
+
return _num(v, 7)
|
|
455
|
+
if isinstance(v, (np.integer, np.bool_)):
|
|
456
|
+
return str(v.item())
|
|
457
|
+
return str(v)
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
_GLYPHS = {
|
|
461
|
+
# the title: heavy
|
|
462
|
+
"htl": "┏", "hh": "━", "htr": "┓", "hv": "┃", "hbl": "┗", "hbr": "┛",
|
|
463
|
+
# the row diagram: heavy edges, light inside
|
|
464
|
+
"btl": "┏", "btt": "┯", "btr": "┓", "bh": "━", "bv": "┃", "biv": "│",
|
|
465
|
+
"bml": "┠", "bmx": "┼", "bmr": "┨", "bmh": "─",
|
|
466
|
+
"bbl": "┗", "bbt": "┷", "bbr": "┛",
|
|
467
|
+
"dash": "╌", "open": "╮", "bar": "│", "tee": "├╴", "close": "╯", "ne": "≠",
|
|
468
|
+
# tables: light
|
|
469
|
+
"ttl": "┌", "ttt": "┬", "ttr": "┐", "th": "─", "tv": "│",
|
|
470
|
+
"tml": "├", "tmx": "┼", "tmr": "┤", "tbl": "└", "tbt": "┴", "tbr": "┘",
|
|
471
|
+
"under": "▔",
|
|
472
|
+
} # fmt: skip
|
|
473
|
+
|
|
474
|
+
_ASCII = {
|
|
475
|
+
"htl": "+", "hh": "=", "htr": "+", "hv": "|", "hbl": "+", "hbr": "+",
|
|
476
|
+
"btl": "+", "btt": "+", "btr": "+", "bh": "=", "bv": "|", "biv": "|",
|
|
477
|
+
"bml": "+", "bmx": "+", "bmr": "+", "bmh": "-",
|
|
478
|
+
"bbl": "+", "bbt": "+", "bbr": "+",
|
|
479
|
+
"dash": ".", "open": "+", "bar": "|", "tee": "+-", "close": "+", "ne": "!",
|
|
480
|
+
"ttl": "+", "ttt": "+", "ttr": "+", "th": "-", "tv": "|",
|
|
481
|
+
"tml": "+", "tmx": "+", "tmr": "+", "tbl": "+", "tbt": "+", "tbr": "+",
|
|
482
|
+
"under": "-",
|
|
483
|
+
} # fmt: skip
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: dazdiff
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Compare two tables value by value, from Python or the command line.
|
|
5
|
+
Project-URL: Homepage, https://github.com/pedrobtz/py-dazdiff
|
|
6
|
+
Project-URL: Documentation, https://pedrobtz.github.io/py-dazdiff/
|
|
7
|
+
Project-URL: Issues, https://github.com/pedrobtz/py-dazdiff/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/pedrobtz/py-dazdiff/blob/main/CHANGELOG.md
|
|
9
|
+
Author-email: pedrobtz <pedrobtz@gmail.com>
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: compare,csv,dataframe,diff,pandas,testing
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
24
|
+
Classifier: Topic :: Scientific/Engineering
|
|
25
|
+
Classifier: Topic :: Software Development :: Testing
|
|
26
|
+
Classifier: Typing :: Typed
|
|
27
|
+
Requires-Python: >=3.10
|
|
28
|
+
Requires-Dist: pandas>=2.2.2
|
|
29
|
+
Description-Content-Type: text/markdown
|
|
30
|
+
|
|
31
|
+
# dazdiff
|
|
32
|
+
|
|
33
|
+
[](https://github.com/pedrobtz/py-dazdiff/actions/workflows/ci.yml)
|
|
34
|
+
[](https://pypi.org/project/dazdiff/)
|
|
35
|
+
[](https://pypi.org/project/dazdiff/)
|
|
36
|
+
|
|
37
|
+
`dazdiff` compares two tables value by value, from Python or from the command
|
|
38
|
+
line. It is a small Python take on the R package
|
|
39
|
+
[pedrobtz/dazdiff](https://github.com/pedrobtz/dazdiff). Documentation:
|
|
40
|
+
<https://pedrobtz.github.io/py-dazdiff/>.
|
|
41
|
+
|
|
42
|
+
Its main function, `diff_table()`, answers one of two questions:
|
|
43
|
+
|
|
44
|
+
- **Does this table match the benchmark?** (`mode="benchmark"`, the default.)
|
|
45
|
+
The benchmark's column types are the truth: the tested table is cast to them
|
|
46
|
+
and its extra columns are dropped.
|
|
47
|
+
- **Are these two tables equal?** (`mode="equal"`.) Both tables must have the
|
|
48
|
+
same columns with the same types; nothing is cast.
|
|
49
|
+
|
|
50
|
+
The result has one row per key and measure, with `row_id_x`, `row_id_y`,
|
|
51
|
+
`value_x`, `value_y`, `diff`, `abs_diff`, `rel_diff` and a `status` of `same`,
|
|
52
|
+
`changed`, `only_x` or `only_y`.
|
|
53
|
+
|
|
54
|
+
## Installation
|
|
55
|
+
|
|
56
|
+
```sh
|
|
57
|
+
pip install dazdiff # the library and the CLI
|
|
58
|
+
uv tool install dazdiff # the CLI on its own
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
It needs Python 3.10 or later and pandas 2.2.2 or later. The development
|
|
62
|
+
version installs from GitHub:
|
|
63
|
+
`pip install git+https://github.com/pedrobtz/py-dazdiff`.
|
|
64
|
+
|
|
65
|
+
## Command line
|
|
66
|
+
|
|
67
|
+
```sh
|
|
68
|
+
$ dazdiff examples/benchmark.csv examples/stored.csv --by id,region --only-diffs
|
|
69
|
+
dazdiff: Dropped column(s) of `y` that the benchmark `x` does not have: loaded_at.
|
|
70
|
+
ID REGION metric row_id_x row_id_y value_x value_y diff abs_diff rel_diff status
|
|
71
|
+
1 north UNITS 1 1 10.0 10.4 0.4 0.4 0.038462 changed
|
|
72
|
+
2 south AMOUNT 2 2 200.0 200.5 0.5 0.5 0.002494 changed
|
|
73
|
+
4 west AMOUNT 4 <NA> 400.0 NaN NaN NaN NaN only_x
|
|
74
|
+
...
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
| Option | Meaning |
|
|
78
|
+
| --- | --- |
|
|
79
|
+
| `-b, --by COLS` | comma-separated key columns (default: every non-measure column) |
|
|
80
|
+
| `--exclude COLS` | comma-separated columns to leave out of both files |
|
|
81
|
+
| `-m, --measures` | `numeric` (default), `numeric+integer`, or comma-separated names |
|
|
82
|
+
| `--mode` | `benchmark` (default) or `equal` |
|
|
83
|
+
| `-t, --tolerance` | absolute tolerance |
|
|
84
|
+
| `-r, --rel-tol` | relative tolerance (relative to the larger magnitude) |
|
|
85
|
+
| `--no-cast` | do not cast the second file to the first's types; they must agree |
|
|
86
|
+
| `--no-truncate` | refuse casts that would truncate values to integers |
|
|
87
|
+
| `--duplicates` | `disambiguate` repeated keys in arrival order (default), `aggregate` them as multisets, or `error` |
|
|
88
|
+
| `--row-key` | `position` or `sorted`: add a `row_number` key when every column is a measure |
|
|
89
|
+
| `--sep` | CSV separator |
|
|
90
|
+
| `-o, --output FILE` | write the full comparison to a CSV file |
|
|
91
|
+
| `-s, --summary` | print the per-measure summary instead |
|
|
92
|
+
| `-R, --report` | print a text report; `--x-name`, `--y-name` label the files, `--ascii` avoids box-drawing characters |
|
|
93
|
+
| `-e, --explain [COLUMN]` | print which key values split same from not same; with a column, per value of it |
|
|
94
|
+
| `--joined` | with `--explain`, count only rows found in both files |
|
|
95
|
+
| `-d, --only-diffs` | print only rows that are not `same` |
|
|
96
|
+
| `-q, --quiet` | print nothing |
|
|
97
|
+
|
|
98
|
+
The exit status is `0` when the tables match, `1` when they differ and `2` on
|
|
99
|
+
error, so `dazdiff` works in scripts and CI. Column names are normalized (upper
|
|
100
|
+
case, other characters turned into `_`), so `id` and `ID` pair up. Because CSV
|
|
101
|
+
files carry no types, a column read as integer in one file and as decimals in
|
|
102
|
+
the other is compared as float; one that is float only because of an empty
|
|
103
|
+
value stays an integer.
|
|
104
|
+
|
|
105
|
+
`python -m dazdiff` works too.
|
|
106
|
+
|
|
107
|
+
## Python
|
|
108
|
+
|
|
109
|
+
```python
|
|
110
|
+
import pandas as pd
|
|
111
|
+
from dazdiff import diff_report, diff_summary, diff_table, is_matching
|
|
112
|
+
|
|
113
|
+
benchmark = pd.read_csv("examples/benchmark.csv")
|
|
114
|
+
stored = pd.read_csv("examples/stored.csv")
|
|
115
|
+
|
|
116
|
+
d = diff_table(benchmark, stored, by=["id", "region"], measures="numeric+integer")
|
|
117
|
+
d[d["status"] != "same"]
|
|
118
|
+
diff_summary(d)
|
|
119
|
+
assert not is_matching(d) # rows 4 and 5 are each in one table only
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
`is_matching(d, tolerance=1)` judges the values again at another threshold,
|
|
123
|
+
without running the comparison again.
|
|
124
|
+
|
|
125
|
+
`diff_explain(d)` says which key columns separate the rows that match from
|
|
126
|
+
those that do not, and `diff_explain(d, "region")` counts each status per
|
|
127
|
+
region, most differences first.
|
|
128
|
+
|
|
129
|
+
`exclude=` leaves technical columns (load stamps, run ids) out of both tables.
|
|
130
|
+
When a number of the tested table is cast to the benchmark's text, it is
|
|
131
|
+
written with the decimals the benchmark uses, so `2` meets `"2.0"`.
|
|
132
|
+
|
|
133
|
+
With no measure columns, rows are compared as wholes, and a repeated row counts
|
|
134
|
+
once per copy. Values truncated when casting to the benchmark's integer
|
|
135
|
+
columns are recorded in `d.attrs["truncated"]`.
|
|
136
|
+
|
|
137
|
+
### Report
|
|
138
|
+
|
|
139
|
+
```python
|
|
140
|
+
print(diff_report(d, "benchmark", "stored"))
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
draws the columns compared, how the rows of the two tables align, and per
|
|
144
|
+
changed measure its match rate and largest differences.
|
|
145
|
+
`diff_report(d).format(unicode=False)` draws it in ASCII.
|
|
146
|
+
|
|
147
|
+
### In tests
|
|
148
|
+
|
|
149
|
+
```python
|
|
150
|
+
from dazdiff import assert_table_equal
|
|
151
|
+
|
|
152
|
+
expected = pd.DataFrame({"id": [1, 2], "amount": [100.0, 200.0]})
|
|
153
|
+
actual = pd.DataFrame({"id": [1, 2], "amount": [100.0, 200.0 + 1e-12]})
|
|
154
|
+
assert_table_equal(actual, expected) # within a relative 1.5e-8
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
`assert_table_equal()` tests `actual` against `expected` as the benchmark and
|
|
158
|
+
raises `AssertionError` with `diff_summary()` and the first records that
|
|
159
|
+
differ, so it works under pytest, unittest or a plain script. A repeated key
|
|
160
|
+
is an error unless `duplicates=` says how to treat it.
|
|
161
|
+
|
|
162
|
+
## Development
|
|
163
|
+
|
|
164
|
+
```sh
|
|
165
|
+
uv sync
|
|
166
|
+
uv run pytest
|
|
167
|
+
```
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
dazdiff/__init__.py,sha256=3m3q3tYAott0LeWHjqJlJRK9bR8eleSee5RdVo7DbPM,798
|
|
2
|
+
dazdiff/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
|
|
3
|
+
dazdiff/cli.py,sha256=fh3GeN19pQ07NPm9ZyaiRrQseugJLJUuke6AAK5KuUU,9172
|
|
4
|
+
dazdiff/core.py,sha256=17RQlunqEl91Iz_8CatU7ZZ3OALNM7FD4s9agjgz-r8,56075
|
|
5
|
+
dazdiff/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
6
|
+
dazdiff/report.py,sha256=31MDMsTfEwEZ5ThedZyx4jFcWNctcix7N4oxSD0WWg8,18441
|
|
7
|
+
dazdiff-0.1.0.dist-info/METADATA,sha256=ONMcJtHZqWLxUx3Ef5O5FmIak6annv_zGRZq7Dn6EQo,7138
|
|
8
|
+
dazdiff-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
9
|
+
dazdiff-0.1.0.dist-info/entry_points.txt,sha256=-47tc8LTUmlSGJm-WJ27D5OSZ0xhG_14xVggCsBBYEw,45
|
|
10
|
+
dazdiff-0.1.0.dist-info/licenses/LICENSE,sha256=5-psad4KOa9d1D5z6VU6gf1MmGtRtOI2NqRBnNr-yEc,1065
|
|
11
|
+
dazdiff-0.1.0.dist-info/RECORD,,
|