dazdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dazdiff/__init__.py +34 -0
- dazdiff/__main__.py +5 -0
- dazdiff/cli.py +248 -0
- dazdiff/core.py +1357 -0
- dazdiff/py.typed +0 -0
- dazdiff/report.py +483 -0
- dazdiff-0.1.0.dist-info/METADATA +167 -0
- dazdiff-0.1.0.dist-info/RECORD +11 -0
- dazdiff-0.1.0.dist-info/WHEEL +4 -0
- dazdiff-0.1.0.dist-info/entry_points.txt +2 -0
- dazdiff-0.1.0.dist-info/licenses/LICENSE +21 -0
dazdiff/__init__.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""dazdiff: compare two tables value by value."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
4
|
+
|
|
5
|
+
from .core import (
|
|
6
|
+
DiffError,
|
|
7
|
+
DiffNote,
|
|
8
|
+
DiffWarning,
|
|
9
|
+
assert_table_equal,
|
|
10
|
+
diff_explain,
|
|
11
|
+
diff_summary,
|
|
12
|
+
diff_table,
|
|
13
|
+
is_matching,
|
|
14
|
+
normalize_names,
|
|
15
|
+
)
|
|
16
|
+
from .report import DiffReport, diff_report
|
|
17
|
+
|
|
18
|
+
try: # one source: the version in pyproject.toml, read from the installed metadata
|
|
19
|
+
__version__ = version("dazdiff")
|
|
20
|
+
except PackageNotFoundError: # pragma: no cover - running from a source tree
|
|
21
|
+
__version__ = "0+unknown"
|
|
22
|
+
__all__ = [
|
|
23
|
+
"DiffError",
|
|
24
|
+
"DiffNote",
|
|
25
|
+
"DiffReport",
|
|
26
|
+
"DiffWarning",
|
|
27
|
+
"assert_table_equal",
|
|
28
|
+
"diff_explain",
|
|
29
|
+
"diff_report",
|
|
30
|
+
"diff_summary",
|
|
31
|
+
"diff_table",
|
|
32
|
+
"is_matching",
|
|
33
|
+
"normalize_names",
|
|
34
|
+
]
|
dazdiff/__main__.py
ADDED
dazdiff/cli.py
ADDED
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
"""Command line interface: ``dazdiff X.csv Y.csv [options]``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import re
|
|
7
|
+
import sys
|
|
8
|
+
import warnings
|
|
9
|
+
from collections.abc import Sequence
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
import numpy as np
|
|
13
|
+
import pandas as pd
|
|
14
|
+
|
|
15
|
+
from . import __version__
|
|
16
|
+
from .core import _canon, diff_explain, diff_summary, diff_table, is_matching
|
|
17
|
+
from .report import diff_report
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _align_numeric(x: pd.DataFrame, y: pd.DataFrame) -> None:
|
|
21
|
+
"""CSV carries no types: where one file reads a column as int and the other as
|
|
22
|
+
float with a fraction, read both as float so the cast does not truncate it.
|
|
23
|
+
|
|
24
|
+
A float column of whole numbers is float only because it has an empty value; it
|
|
25
|
+
casts to int exactly, so it is left alone and keeps its role: widening it would
|
|
26
|
+
turn an integer key into a "numeric" measure.
|
|
27
|
+
"""
|
|
28
|
+
ny = dict(zip([_canon(c) for c in y.columns], y.columns, strict=True))
|
|
29
|
+
for cx, nx in zip(x.columns, [_canon(c) for c in x.columns], strict=True):
|
|
30
|
+
cy = ny.get(nx)
|
|
31
|
+
if cy is None:
|
|
32
|
+
continue
|
|
33
|
+
kinds = {x[cx].dtype.kind, y[cy].dtype.kind}
|
|
34
|
+
if kinds == {"i"} and x[cx].dtype != y[cy].dtype: # int64 vs Int64: both nullable
|
|
35
|
+
x[cx] = x[cx].astype("Int64")
|
|
36
|
+
y[cy] = y[cy].astype("Int64")
|
|
37
|
+
continue
|
|
38
|
+
if kinds == {"i", "f"} and _has_fraction(x[cx] if x[cx].dtype.kind == "f" else y[cy]):
|
|
39
|
+
x[cx] = x[cx].astype(float)
|
|
40
|
+
y[cy] = y[cy].astype(float)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _printable(text: str, stream) -> bool:
|
|
44
|
+
"""Whether ``stream`` can encode ``text`` (a Windows console often cannot)."""
|
|
45
|
+
encoding = getattr(stream, "encoding", None)
|
|
46
|
+
if not encoding:
|
|
47
|
+
return False
|
|
48
|
+
try:
|
|
49
|
+
text.encode(encoding)
|
|
50
|
+
except (UnicodeEncodeError, LookupError):
|
|
51
|
+
return False
|
|
52
|
+
return True
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
_LEADING_ZERO = re.compile(r"\s*[+-]?0[0-9]")
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _read(path: str, sep: str) -> tuple[pd.DataFrame, pd.DataFrame]:
|
|
59
|
+
"""The file as pandas types it, and as text."""
|
|
60
|
+
return pd.read_csv(path, sep=sep), pd.read_csv(path, sep=sep, dtype=str)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _keep_codes(x, x_text, y, y_text) -> None:
|
|
64
|
+
"""A column written with a leading zero in either file (``007``) holds codes, not
|
|
65
|
+
numbers: read it as text in both, so ``007`` and ``7`` stay apart (#47)."""
|
|
66
|
+
y_by = {_canon(c): c for c in y.columns}
|
|
67
|
+
for cx in x.columns:
|
|
68
|
+
cy = y_by.get(_canon(cx))
|
|
69
|
+
cols = [(x, x_text, cx)] + ([(y, y_text, cy)] if cy is not None else [])
|
|
70
|
+
if not any(df[c].dtype.kind in "iuf" for df, _, c in cols):
|
|
71
|
+
continue
|
|
72
|
+
if any(_has_leading_zero(text[c]) for _, text, c in cols):
|
|
73
|
+
for df, text, c in cols:
|
|
74
|
+
df[c] = text[c]
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _has_leading_zero(s: pd.Series) -> bool:
|
|
78
|
+
return bool(s.dropna().map(lambda v: bool(_LEADING_ZERO.match(v))).any())
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _restore_ints(df: pd.DataFrame) -> None:
|
|
82
|
+
"""pandas reads an integer column with an empty cell as float; read it as a
|
|
83
|
+
nullable integer instead, so its role and type follow its values (#46)."""
|
|
84
|
+
for c in df.columns:
|
|
85
|
+
s = df[c]
|
|
86
|
+
if s.dtype.kind != "f" or not s.isna().any() or s.notna().sum() == 0:
|
|
87
|
+
continue
|
|
88
|
+
v = s.dropna().to_numpy()
|
|
89
|
+
if np.isfinite(v).all() and (v == np.trunc(v)).all() and (np.abs(v) < 2.0**63).all():
|
|
90
|
+
df[c] = s.astype("Int64")
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _has_fraction(s: pd.Series) -> bool:
|
|
94
|
+
v = s.dropna().to_numpy(dtype=float)
|
|
95
|
+
v = v[np.isfinite(v)]
|
|
96
|
+
return bool((v != np.trunc(v)).any())
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _csv_list(value: str) -> list[str]:
|
|
100
|
+
return [v.strip() for v in value.split(",") if v.strip()]
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
104
|
+
p = argparse.ArgumentParser(
|
|
105
|
+
prog="dazdiff",
|
|
106
|
+
description="Compare two CSV files value by value. "
|
|
107
|
+
"Exit status: 0 if the tables match, 1 if they differ, 2 on error.",
|
|
108
|
+
)
|
|
109
|
+
p.add_argument("x", help="first CSV file (the benchmark in benchmark mode)")
|
|
110
|
+
p.add_argument("y", help="second CSV file (the tested table in benchmark mode)")
|
|
111
|
+
p.add_argument("-b", "--by", type=_csv_list, help="comma-separated key columns")
|
|
112
|
+
p.add_argument(
|
|
113
|
+
"--exclude",
|
|
114
|
+
type=_csv_list,
|
|
115
|
+
help="comma-separated columns to leave out of both files",
|
|
116
|
+
)
|
|
117
|
+
p.add_argument(
|
|
118
|
+
"-m",
|
|
119
|
+
"--measures",
|
|
120
|
+
default="numeric",
|
|
121
|
+
help='"numeric" (default), "numeric+integer", or comma-separated column names',
|
|
122
|
+
)
|
|
123
|
+
p.add_argument("--mode", choices=["benchmark", "equal"], default="benchmark")
|
|
124
|
+
p.add_argument("-t", "--tolerance", type=float, default=0.0, help="absolute tolerance")
|
|
125
|
+
p.add_argument("-r", "--rel-tol", type=float, default=0.0, help="relative tolerance")
|
|
126
|
+
p.add_argument(
|
|
127
|
+
"--no-cast",
|
|
128
|
+
dest="cast",
|
|
129
|
+
action="store_const",
|
|
130
|
+
const=False,
|
|
131
|
+
default=None,
|
|
132
|
+
help="do not cast the second file to the first's types; they must agree",
|
|
133
|
+
)
|
|
134
|
+
p.add_argument(
|
|
135
|
+
"--no-truncate",
|
|
136
|
+
dest="truncate",
|
|
137
|
+
action="store_false",
|
|
138
|
+
help="refuse casts that would truncate values to integers",
|
|
139
|
+
)
|
|
140
|
+
p.add_argument(
|
|
141
|
+
"--duplicates",
|
|
142
|
+
choices=["disambiguate", "aggregate", "error"],
|
|
143
|
+
default="disambiguate",
|
|
144
|
+
help="repeated keys: pair in arrival order (default), compare as multisets, or refuse",
|
|
145
|
+
)
|
|
146
|
+
p.add_argument(
|
|
147
|
+
"--row-key",
|
|
148
|
+
choices=["none", "position", "sorted"],
|
|
149
|
+
default="none",
|
|
150
|
+
help="add a row_number key: rows as they come, or after sorting both files",
|
|
151
|
+
)
|
|
152
|
+
p.add_argument("--sep", default=",", help="CSV field separator (default ',')")
|
|
153
|
+
p.add_argument("-o", "--output", help="write the full comparison to this CSV file")
|
|
154
|
+
p.add_argument("-s", "--summary", action="store_true", help="print the per-measure summary")
|
|
155
|
+
p.add_argument(
|
|
156
|
+
"-R", "--report", action="store_true", help="print a text report of the comparison"
|
|
157
|
+
)
|
|
158
|
+
p.add_argument("--x-name", help="the first file's name in the report (default: its file name)")
|
|
159
|
+
p.add_argument("--y-name", help="the second file's name in the report (default: its file name)")
|
|
160
|
+
p.add_argument(
|
|
161
|
+
"--ascii",
|
|
162
|
+
action="store_true",
|
|
163
|
+
help="draw the report in ASCII (the default when the terminal cannot show Unicode)",
|
|
164
|
+
)
|
|
165
|
+
p.add_argument(
|
|
166
|
+
"-e",
|
|
167
|
+
"--explain",
|
|
168
|
+
nargs="?",
|
|
169
|
+
const="",
|
|
170
|
+
metavar="COLUMN",
|
|
171
|
+
help="print which key values split same from not same; with COLUMN, per value of it",
|
|
172
|
+
)
|
|
173
|
+
p.add_argument(
|
|
174
|
+
"--joined",
|
|
175
|
+
action="store_true",
|
|
176
|
+
help="with --explain, count only rows found in both files",
|
|
177
|
+
)
|
|
178
|
+
p.add_argument(
|
|
179
|
+
"-d", "--only-diffs", action="store_true", help="print only rows that are not 'same'"
|
|
180
|
+
)
|
|
181
|
+
p.add_argument("-q", "--quiet", action="store_true", help="print nothing; use the exit status")
|
|
182
|
+
p.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
183
|
+
return p
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
187
|
+
args = build_parser().parse_args(argv)
|
|
188
|
+
try:
|
|
189
|
+
return _run(args)
|
|
190
|
+
except Exception as e: # every failure is exit 2: 1 means "the tables differ"
|
|
191
|
+
print(f"dazdiff: error: {e}", file=sys.stderr)
|
|
192
|
+
return 2
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _run(args: argparse.Namespace) -> int:
|
|
196
|
+
def show_warning(message, category, *_args, **_kwargs):
|
|
197
|
+
if not args.quiet:
|
|
198
|
+
print(f"dazdiff: {message}", file=sys.stderr)
|
|
199
|
+
|
|
200
|
+
x, x_text = _read(args.x, args.sep)
|
|
201
|
+
y, y_text = _read(args.y, args.sep)
|
|
202
|
+
_keep_codes(x, x_text, y, y_text)
|
|
203
|
+
_restore_ints(x)
|
|
204
|
+
_restore_ints(y)
|
|
205
|
+
measures = args.measures
|
|
206
|
+
if measures not in ("numeric", "numeric+integer"):
|
|
207
|
+
measures = _csv_list(measures)
|
|
208
|
+
_align_numeric(x, y)
|
|
209
|
+
with warnings.catch_warnings():
|
|
210
|
+
warnings.simplefilter("always")
|
|
211
|
+
warnings.showwarning = show_warning
|
|
212
|
+
d = diff_table(
|
|
213
|
+
x,
|
|
214
|
+
y,
|
|
215
|
+
by=args.by,
|
|
216
|
+
exclude=args.exclude,
|
|
217
|
+
cast=args.cast,
|
|
218
|
+
row_key=args.row_key,
|
|
219
|
+
measures=measures,
|
|
220
|
+
mode=args.mode,
|
|
221
|
+
tolerance=args.tolerance,
|
|
222
|
+
rel_tol=args.rel_tol,
|
|
223
|
+
truncate=args.truncate,
|
|
224
|
+
duplicates=args.duplicates,
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
if args.output:
|
|
228
|
+
d.to_csv(args.output, index=False)
|
|
229
|
+
if not args.quiet:
|
|
230
|
+
if args.report:
|
|
231
|
+
r = diff_report(d, args.x_name or Path(args.x).name, args.y_name or Path(args.y).name)
|
|
232
|
+
text = r.format(unicode=True)
|
|
233
|
+
if args.ascii or not _printable(text, sys.stdout):
|
|
234
|
+
text = r.format(unicode=False)
|
|
235
|
+
print(text)
|
|
236
|
+
elif args.explain is not None:
|
|
237
|
+
e = diff_explain(d, args.explain or None, among="joined" if args.joined else "all")
|
|
238
|
+
print(e.to_string(index=False))
|
|
239
|
+
elif args.summary:
|
|
240
|
+
print(diff_summary(d).to_string(index=False))
|
|
241
|
+
elif not args.output:
|
|
242
|
+
shown = d[d["status"] != "same"] if args.only_diffs else d
|
|
243
|
+
print(shown.to_string(index=False) if len(shown) else "No differences.")
|
|
244
|
+
return 0 if is_matching(d) else 1
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
if __name__ == "__main__":
|
|
248
|
+
sys.exit(main())
|