tracecomparekit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- difffinder/__init__.py +21 -0
- difffinder/__main__.py +3 -0
- difffinder/ai.py +209 -0
- difffinder/comparison.py +87 -0
- difffinder/demo.py +27 -0
- difffinder/diff.py +214 -0
- difffinder/hints.py +118 -0
- difffinder/outcome.py +63 -0
- difffinder/probe.py +301 -0
- difffinder/render.py +161 -0
- difffinder/selfcheck.py +92 -0
- difffinder/sorting_visualizer.py +717 -0
- difffinder/tracer.py +192 -0
- tracecomparekit-0.1.0.dist-info/METADATA +189 -0
- tracecomparekit-0.1.0.dist-info/RECORD +17 -0
- tracecomparekit-0.1.0.dist-info/WHEEL +4 -0
- tracecomparekit-0.1.0.dist-info/licenses/LICENSE +23 -0
difffinder/outcome.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Run a function once and record what it visibly did."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import copy
|
|
6
|
+
import inspect
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from typing import Any, Callable, Dict, Optional
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def short(text: str, limit: int = 60) -> str:
|
|
12
|
+
"""Shorten long text for display."""
|
|
13
|
+
return text if len(text) <= limit else text[: limit - 3] + "..."
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass
|
|
17
|
+
class Outcome:
|
|
18
|
+
"""What one call did: its result or exception, and the final state of its arguments."""
|
|
19
|
+
|
|
20
|
+
returned: Any = None
|
|
21
|
+
error_type: Optional[str] = None # for example "ValueError"; None if the call returned normally
|
|
22
|
+
error_message: str = ""
|
|
23
|
+
inputs: Dict[str, str] = field(default_factory=dict) # argument name -> repr before the call
|
|
24
|
+
arguments: Dict[str, Any] = field(default_factory=dict) # argument name -> object after the call
|
|
25
|
+
|
|
26
|
+
@property
|
|
27
|
+
def raised(self) -> bool:
|
|
28
|
+
return self.error_type is not None
|
|
29
|
+
|
|
30
|
+
def describe(self) -> str:
|
|
31
|
+
"""The result in a few words, for example ``returned 6`` or ``raised ValueError: ...``."""
|
|
32
|
+
if self.raised:
|
|
33
|
+
detail = (": " + self.error_message) if self.error_message else ""
|
|
34
|
+
return "raised %s%s" % (self.error_type, detail)
|
|
35
|
+
return "returned %s" % short(repr(self.returned))
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _bind(func: Callable[..., Any], args: tuple, kwargs: dict) -> Dict[str, Any]:
|
|
39
|
+
"""Map parameter names to the objects passed in (the objects are shared, not copied)."""
|
|
40
|
+
try:
|
|
41
|
+
return dict(inspect.signature(func).bind(*args, **kwargs).arguments)
|
|
42
|
+
except (TypeError, ValueError): # no signature available, or the arguments do not fit it
|
|
43
|
+
named = {"arg%d" % i: a for i, a in enumerate(args)}
|
|
44
|
+
named.update(kwargs)
|
|
45
|
+
return named
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def run(func: Callable[..., Any], args: tuple = (), kwargs: Optional[dict] = None) -> Outcome:
|
|
49
|
+
"""Call ``func(*args, **kwargs)`` on a deep copy of the arguments and record what happened.
|
|
50
|
+
|
|
51
|
+
An exception is recorded, not raised. Because the call works on copies, your own data is never changed,
|
|
52
|
+
but changes the function makes to its arguments are captured in ``Outcome.arguments``.
|
|
53
|
+
"""
|
|
54
|
+
args = copy.deepcopy(tuple(args))
|
|
55
|
+
kwargs = copy.deepcopy(dict(kwargs or {}))
|
|
56
|
+
arguments = _bind(func, args, kwargs)
|
|
57
|
+
inputs = {name: short(repr(value), 500) for name, value in arguments.items()}
|
|
58
|
+
try:
|
|
59
|
+
outcome = Outcome(returned=func(*args, **kwargs))
|
|
60
|
+
except Exception as exc:
|
|
61
|
+
outcome = Outcome(error_type=type(exc).__name__, error_message=str(exc))
|
|
62
|
+
outcome.inputs, outcome.arguments = inputs, arguments
|
|
63
|
+
return outcome
|
difffinder/probe.py
ADDED
|
@@ -0,0 +1,301 @@
|
|
|
1
|
+
"""Black-box testing: find inputs where two functions disagree, shrink one, and describe the pattern.
|
|
2
|
+
|
|
3
|
+
This works however differently the two functions are written, because it only looks at what each one
|
|
4
|
+
visibly does (return value, exception, and what it did to its arguments), never at their internals.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import contextlib
|
|
10
|
+
import copy
|
|
11
|
+
import io
|
|
12
|
+
import random
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
from itertools import combinations, zip_longest
|
|
15
|
+
from typing import Any, Dict, List, Optional
|
|
16
|
+
|
|
17
|
+
from .tracer import RAISE, RETURN, Trace, record
|
|
18
|
+
|
|
19
|
+
MAX_SINGLES = 55 # inputs made by changing one argument
|
|
20
|
+
MAX_PAIRS = 25 # inputs made by changing two arguments at once
|
|
21
|
+
SHRINK_BUDGET = 150
|
|
22
|
+
STEP_LIMIT = 20_000
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _isnum(x: Any) -> bool:
|
|
26
|
+
return type(x) in (int, float) # bool is not a number here
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def outcome(trace: Trace) -> tuple:
|
|
30
|
+
"""What a call visibly did. Exceptions count by type only, so message wording is not a difference."""
|
|
31
|
+
items = []
|
|
32
|
+
for e in trace.events:
|
|
33
|
+
if e.var.startswith("<"):
|
|
34
|
+
items.append((e.var, e.value.split(":")[0] if e.var == RAISE else e.value))
|
|
35
|
+
if trace.truncated:
|
|
36
|
+
items.append(("<timeout>", ""))
|
|
37
|
+
return tuple(items)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def run_pair(candidate, reference, args, kwargs):
|
|
41
|
+
"""Run both functions on copies of ``args``. Returns (candidate trace, reference trace, differ)."""
|
|
42
|
+
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
|
43
|
+
ta = record(candidate, copy.deepcopy(args), copy.deepcopy(kwargs), max_steps=STEP_LIMIT)
|
|
44
|
+
tb = record(reference, copy.deepcopy(args), copy.deepcopy(kwargs), max_steps=STEP_LIMIT)
|
|
45
|
+
return ta, tb, outcome(ta) != outcome(tb)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _unique(seq, original) -> list:
|
|
49
|
+
seen, out = {repr(original)}, []
|
|
50
|
+
for item in seq:
|
|
51
|
+
r = repr(item)
|
|
52
|
+
if r not in seen:
|
|
53
|
+
seen.add(r)
|
|
54
|
+
out.append(item)
|
|
55
|
+
return out
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _alternatives(value) -> list:
|
|
59
|
+
"""Interesting variations of one argument: edge cases and small mutations."""
|
|
60
|
+
t, out = type(value), []
|
|
61
|
+
if t is bool:
|
|
62
|
+
out = [not value]
|
|
63
|
+
elif t is int:
|
|
64
|
+
out = [0, 1, -1, 2, -2, value + 1, value - 1, -value, value * 2, 10, 100]
|
|
65
|
+
elif t is float:
|
|
66
|
+
out = [0.0, 1.0, -1.0, -value, value + 1, value / 2]
|
|
67
|
+
elif t is str:
|
|
68
|
+
out = ["", "a", "ab", "aa", value[:1], value[:-1], value[::-1], value.upper(), " "]
|
|
69
|
+
elif t in (list, tuple):
|
|
70
|
+
items, make = list(value), t
|
|
71
|
+
out = [make()]
|
|
72
|
+
out += [make(items[:i] + items[i + 1 :]) for i in range(min(len(items), 6))]
|
|
73
|
+
if items:
|
|
74
|
+
out += [make(items[::-1]), make(items[:1]), make(items + items[:1])]
|
|
75
|
+
try:
|
|
76
|
+
out.append(make(sorted(items)))
|
|
77
|
+
except TypeError:
|
|
78
|
+
pass
|
|
79
|
+
if not items or any(_isnum(x) for x in items):
|
|
80
|
+
out += [make(items + [e]) for e in (0, -1, -3)]
|
|
81
|
+
if items and all(_isnum(x) for x in items):
|
|
82
|
+
out.append(make([-x for x in items]))
|
|
83
|
+
for i in range(min(len(items), 4)):
|
|
84
|
+
out += [make(items[:i] + [alt] + items[i + 1 :]) for alt in _alternatives(items[i])[:3]]
|
|
85
|
+
elif t is dict:
|
|
86
|
+
out = [{}] + [{k: v for k, v in value.items() if k != key} for key in list(value)[:4]]
|
|
87
|
+
return _unique(out, value)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _variants(args: tuple) -> List[tuple]:
|
|
91
|
+
rng = random.Random(0) # fixed seed: the same input always gives the same report
|
|
92
|
+
groups = [[args[:i] + (alt,) + args[i + 1 :] for alt in _alternatives(a)] for i, a in enumerate(args)]
|
|
93
|
+
seen, out = {repr(args)}, []
|
|
94
|
+
|
|
95
|
+
def add(candidate):
|
|
96
|
+
r = repr(candidate)
|
|
97
|
+
if r not in seen:
|
|
98
|
+
seen.add(r)
|
|
99
|
+
out.append(candidate)
|
|
100
|
+
|
|
101
|
+
for row in zip_longest(*groups):
|
|
102
|
+
for c in row:
|
|
103
|
+
if c is not None:
|
|
104
|
+
add(c)
|
|
105
|
+
del out[MAX_SINGLES:]
|
|
106
|
+
changeable = [(i, _alternatives(a)) for i, a in enumerate(args)]
|
|
107
|
+
changeable = [(i, alts) for i, alts in changeable if alts]
|
|
108
|
+
if len(changeable) >= 2:
|
|
109
|
+
for _ in range(MAX_PAIRS):
|
|
110
|
+
(i, ai), (j, aj) = rng.sample(changeable, 2)
|
|
111
|
+
c = list(args)
|
|
112
|
+
c[i], c[j] = rng.choice(ai), rng.choice(aj)
|
|
113
|
+
add(tuple(c))
|
|
114
|
+
return out
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _shrinks(value) -> list:
|
|
118
|
+
"""Simpler versions of one argument, simplest first."""
|
|
119
|
+
t, out = type(value), []
|
|
120
|
+
try:
|
|
121
|
+
if t is bool:
|
|
122
|
+
out = [False] if value else []
|
|
123
|
+
elif t is int:
|
|
124
|
+
out = [0, int(value / 2), value - (1 if value > 0 else -1)] if value else []
|
|
125
|
+
elif t is float:
|
|
126
|
+
out = [0.0, float(round(value)), value / 2] if value else []
|
|
127
|
+
elif t in (list, tuple, str):
|
|
128
|
+
n = len(value)
|
|
129
|
+
if n:
|
|
130
|
+
out.append(value[:0])
|
|
131
|
+
if n >= 4:
|
|
132
|
+
out += [value[: n // 2], value[n // 2 :]]
|
|
133
|
+
out += [value[:i] + value[i + 1 :] for i in range(n)]
|
|
134
|
+
if t is not str:
|
|
135
|
+
for i, x in enumerate(value):
|
|
136
|
+
out += [t(list(value[:i]) + [s] + list(value[i + 1 :])) for s in _shrinks(x)[:2]]
|
|
137
|
+
elif t is dict:
|
|
138
|
+
out = [{}] + [{k: v for k, v in value.items() if k != key} for key in value]
|
|
139
|
+
except (ValueError, OverflowError):
|
|
140
|
+
out = []
|
|
141
|
+
return _unique(out, value)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _shrink(candidate, reference, args, kwargs, ta, tb):
|
|
145
|
+
"""Greedily simplify a failing input while the two functions still disagree."""
|
|
146
|
+
best, spent, improved = args, 0, True
|
|
147
|
+
while improved and spent < SHRINK_BUDGET:
|
|
148
|
+
improved = False
|
|
149
|
+
for i, a in enumerate(best):
|
|
150
|
+
for s in _shrinks(a):
|
|
151
|
+
if spent >= SHRINK_BUDGET:
|
|
152
|
+
break
|
|
153
|
+
trial = best[:i] + (s,) + best[i + 1 :]
|
|
154
|
+
spent += 1
|
|
155
|
+
t1, t2, bad = run_pair(candidate, reference, trial, kwargs)
|
|
156
|
+
if bad:
|
|
157
|
+
best, ta, tb, improved = trial, t1, t2, True
|
|
158
|
+
break
|
|
159
|
+
if improved:
|
|
160
|
+
break
|
|
161
|
+
return best, ta, tb
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _features(args: tuple, names: List[str]) -> Dict[str, bool]:
|
|
165
|
+
"""Simple yes/no facts about an input, used to describe when the functions disagree."""
|
|
166
|
+
f: Dict[str, bool] = {}
|
|
167
|
+
scalars = [(n, a) for n, a in zip(names, args) if _isnum(a)]
|
|
168
|
+
for n, a in zip(names, args):
|
|
169
|
+
tag = "`%s`" % n
|
|
170
|
+
if type(a) in (list, tuple):
|
|
171
|
+
items = list(a)
|
|
172
|
+
nums = [x for x in items if _isnum(x)]
|
|
173
|
+
f[tag + " is empty"] = not items
|
|
174
|
+
f[tag + " has exactly one element"] = len(items) == 1
|
|
175
|
+
f[tag + " has more than one element"] = len(items) > 1
|
|
176
|
+
f[tag + " has repeated elements"] = len({repr(x) for x in items}) < len(items)
|
|
177
|
+
if nums:
|
|
178
|
+
f[tag + " contains a negative number"] = any(x < 0 for x in nums)
|
|
179
|
+
f[tag + " contains 0"] = any(x == 0 for x in nums)
|
|
180
|
+
for m, s in scalars:
|
|
181
|
+
f["%s contains a number below `%s`" % (tag, m)] = any(x < s for x in nums)
|
|
182
|
+
f["%s contains a number above `%s`" % (tag, m)] = any(x > s for x in nums)
|
|
183
|
+
f["%s contains a number equal to `%s`" % (tag, m)] = any(x == s for x in nums)
|
|
184
|
+
elif type(a) is str:
|
|
185
|
+
f[tag + " is empty"] = not a
|
|
186
|
+
f[tag + " has exactly one character"] = len(a) == 1
|
|
187
|
+
f[tag + " contains a digit"] = any(c.isdigit() for c in a)
|
|
188
|
+
f[tag + " contains an upper-case letter"] = any(c.isupper() for c in a)
|
|
189
|
+
f[tag + " contains a space"] = " " in a
|
|
190
|
+
elif _isnum(a):
|
|
191
|
+
f[tag + " is negative"] = a < 0
|
|
192
|
+
f[tag + " is 0"] = a == 0
|
|
193
|
+
f[tag + " is positive"] = a > 0
|
|
194
|
+
elif type(a) is dict:
|
|
195
|
+
f[tag + " is empty"] = not a
|
|
196
|
+
for (m, x), (n, y) in combinations(scalars, 2):
|
|
197
|
+
f["`%s` is greater than `%s`" % (m, n)] = x > y
|
|
198
|
+
f["`%s` equals `%s`" % (m, n)] = x == y
|
|
199
|
+
return f
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def find_patterns(rows, names: List[str], limit: int = 2) -> List[str]:
|
|
203
|
+
"""Describe, from the inputs tried, a simple property that goes with the functions disagreeing."""
|
|
204
|
+
failed = sum(1 for _, bad in rows if bad)
|
|
205
|
+
if not failed or failed == len(rows):
|
|
206
|
+
return []
|
|
207
|
+
table = [(_features(a, names), bad) for a, bad in rows]
|
|
208
|
+
keys: List[str] = []
|
|
209
|
+
for feats, _ in table:
|
|
210
|
+
keys += [k for k in feats if k not in keys]
|
|
211
|
+
scored = []
|
|
212
|
+
for k in keys:
|
|
213
|
+
tp = sum(1 for f, bad in table if f.get(k) and bad)
|
|
214
|
+
total = sum(1 for f, _ in table if f.get(k))
|
|
215
|
+
if tp >= 2 and tp / total >= 0.85 and tp / failed >= 0.5:
|
|
216
|
+
scored.append((tp / total, tp / failed, k, tp, total))
|
|
217
|
+
scored.sort(key=lambda s: (-s[0], -s[1], len(s[2])))
|
|
218
|
+
return [
|
|
219
|
+
"It %s %s (%d of %d such inputs I tried)." % ("fails whenever" if p == 1 else "usually fails when", k, tp, total)
|
|
220
|
+
for p, _, k, tp, total in scored[:limit]
|
|
221
|
+
]
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _returned(trace: Trace):
|
|
225
|
+
hist = trace.history(RETURN)
|
|
226
|
+
return hist[-1].value if hist else None
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _raised(trace: Trace):
|
|
230
|
+
hist = trace.history(RAISE)
|
|
231
|
+
return hist[-1].value.split(":")[0] if hist else None
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def find_systematic(pairs, failing: int) -> List[str]:
|
|
235
|
+
"""Spot differences that hold on (almost) every input, which point to a basic mistake."""
|
|
236
|
+
tried = len(pairs)
|
|
237
|
+
got = [(_returned(a), _returned(b)) for a, b in pairs]
|
|
238
|
+
both = [(x, y) for x, y in got if x is not None and y is not None]
|
|
239
|
+
out: List[str] = []
|
|
240
|
+
if len(both) >= 3 and all(x in ("True", "False") and y in ("True", "False") and x != y for x, y in both):
|
|
241
|
+
out.append(
|
|
242
|
+
"Your function returns the opposite of the reference on every input where both give True or False "
|
|
243
|
+
"(%d of %d). Check what your final return statement actually tests." % (len(both), tried)
|
|
244
|
+
)
|
|
245
|
+
elif failing == tried and tried >= 5:
|
|
246
|
+
out.append(
|
|
247
|
+
"They disagree on every input I tried, so this is not an edge case: something basic is different."
|
|
248
|
+
)
|
|
249
|
+
mine = [x for x, _ in both]
|
|
250
|
+
theirs = {y for _, y in both}
|
|
251
|
+
if len(mine) >= 5 and len(set(mine)) == 1 and len(theirs) >= 2:
|
|
252
|
+
out.append("Your function returned %s on every input I tried, while the reference's result depends on the input." % _clip(mine[0]))
|
|
253
|
+
errors = [_raised(a) for a, _ in pairs]
|
|
254
|
+
kinds = {e for e in errors if e}
|
|
255
|
+
if kinds:
|
|
256
|
+
common = max(kinds, key=errors.count)
|
|
257
|
+
n_raise = errors.count(common)
|
|
258
|
+
ref_raise = sum(1 for _, b in pairs if _raised(b))
|
|
259
|
+
if n_raise >= 3 and n_raise > ref_raise:
|
|
260
|
+
out.append("Your function raises %s on %d of the %d inputs I tried; the reference does so on %d." % (common, n_raise, tried, ref_raise))
|
|
261
|
+
return out
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _clip(text: str, limit: int = 30) -> str:
|
|
265
|
+
return text if len(text) <= limit else text[: limit - 3] + "..."
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
@dataclass
|
|
269
|
+
class Probe:
|
|
270
|
+
"""What testing many inputs revealed."""
|
|
271
|
+
|
|
272
|
+
tried: int
|
|
273
|
+
failing: int
|
|
274
|
+
minimal: Optional[tuple] = None
|
|
275
|
+
minimal_candidate: Optional[Trace] = None
|
|
276
|
+
minimal_reference: Optional[Trace] = None
|
|
277
|
+
patterns: List[str] = field(default_factory=list)
|
|
278
|
+
passing: List[tuple] = field(default_factory=list)
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def run_probe(candidate, reference, args, kwargs, names, first_failed, ta, tb) -> Probe:
|
|
282
|
+
"""Try variations of ``args``, shrink a failing one, and look for a pattern."""
|
|
283
|
+
rows = [(args, first_failed)]
|
|
284
|
+
pairs = [(ta, tb)]
|
|
285
|
+
failing_runs: Dict[str, tuple] = {repr(args): (args, ta, tb)} if first_failed else {}
|
|
286
|
+
for v in _variants(args):
|
|
287
|
+
t1, t2, bad = run_pair(candidate, reference, v, kwargs)
|
|
288
|
+
rows.append((v, bad))
|
|
289
|
+
pairs.append((t1, t2))
|
|
290
|
+
if bad:
|
|
291
|
+
failing_runs[repr(v)] = (v, t1, t2)
|
|
292
|
+
probe = Probe(tried=len(rows), failing=sum(1 for _, bad in rows if bad))
|
|
293
|
+
if not failing_runs:
|
|
294
|
+
return probe
|
|
295
|
+
systematic = find_systematic(pairs, probe.failing)
|
|
296
|
+
start = args if first_failed else min((r[0] for r in failing_runs.values()), key=lambda a: len(repr(a)))
|
|
297
|
+
_, t1, t2 = failing_runs[repr(start)]
|
|
298
|
+
probe.minimal, probe.minimal_candidate, probe.minimal_reference = _shrink(candidate, reference, start, kwargs, t1, t2)
|
|
299
|
+
probe.patterns = systematic + find_patterns(rows, names)
|
|
300
|
+
probe.passing = sorted((a for a, bad in rows if not bad), key=lambda a: len(repr(a)))[:2]
|
|
301
|
+
return probe
|
difffinder/render.py
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
"""Turn a Comparison into plain text or notebook HTML."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import html
|
|
6
|
+
import linecache
|
|
7
|
+
|
|
8
|
+
from .tracer import RAISE, RETURN, _safe_repr
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _short(text: str, limit: int = 60) -> str:
|
|
12
|
+
return text if len(text) <= limit else text[: limit - 3] + "..."
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _names(cmp, n: int):
|
|
16
|
+
return [cmp.candidate.arg_names.get("<arg%d>" % i) or "arg%d" % i for i in range(n)]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _call(cmp, args: tuple) -> str:
|
|
20
|
+
return ", ".join("%s=%s" % (n, _short(repr(a), 40)) for n, a in zip(_names(cmp, len(args)), args))
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _describe(trace, args: tuple) -> str:
|
|
24
|
+
"""What a call visibly did, in a few words."""
|
|
25
|
+
parts = []
|
|
26
|
+
raised, returned = trace.history(RAISE), trace.history(RETURN)
|
|
27
|
+
if raised:
|
|
28
|
+
parts.append("raises " + _short(raised[-1].value, 50))
|
|
29
|
+
elif returned:
|
|
30
|
+
parts.append("returns " + _short(returned[-1].value, 50))
|
|
31
|
+
elif trace.truncated:
|
|
32
|
+
parts.append("never finishes (stopped after %d steps)" % trace.steps)
|
|
33
|
+
for ev in trace.events:
|
|
34
|
+
if ev.var.startswith("<arg"):
|
|
35
|
+
idx = int(ev.var[4:-1])
|
|
36
|
+
if idx < len(args) and ev.value != _safe_repr(args[idx]):
|
|
37
|
+
parts.append("changes `%s` to %s" % (trace.arg_names.get(ev.var, ev.var), _short(ev.value, 50)))
|
|
38
|
+
return ", ".join(parts) or "does nothing visible"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _where(ev) -> str:
|
|
42
|
+
src = linecache.getline(ev.filename, ev.lineno).strip()
|
|
43
|
+
return "line %d in %s()%s" % (ev.lineno, ev.func, (": " + src) if src else "")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _label(cmp, var: str) -> str:
|
|
47
|
+
if var == RETURN:
|
|
48
|
+
return "the return value"
|
|
49
|
+
if var == RAISE:
|
|
50
|
+
return "how the call ended"
|
|
51
|
+
if var.startswith("<arg"):
|
|
52
|
+
return "the final state of argument '%s'" % (cmp.candidate.arg_names.get(var) or var)
|
|
53
|
+
return "variable '%s'" % var
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _header(cmp) -> str:
|
|
57
|
+
p, d = cmp.probe, cmp.divergence
|
|
58
|
+
if cmp.outcomes_differ and p is not None and p.failing:
|
|
59
|
+
return "DIFFERENT: yours disagrees with the reference on %d of the %d inputs I tried (including yours)." % (p.failing, p.tried)
|
|
60
|
+
if cmp.outcomes_differ:
|
|
61
|
+
return "DIFFERENT: the two functions give different results on your input."
|
|
62
|
+
if p is not None and p.failing:
|
|
63
|
+
return "DIFFERENT: they agree on your input, but disagree on %d of %d other inputs I tried." % (p.failing, p.tried - 1)
|
|
64
|
+
if d is not None:
|
|
65
|
+
return "DIFFERENT: the runs first diverge on %s (change #%d)." % (_label(cmp, d.var), d.position + 1)
|
|
66
|
+
return "DIFFERENT."
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _off_track(cmp) -> str:
|
|
70
|
+
"""One sentence on the first step where the candidate's run leaves the reference's."""
|
|
71
|
+
d = cmp.divergence
|
|
72
|
+
if d is None or d.var.startswith("<"):
|
|
73
|
+
return ""
|
|
74
|
+
c, r = d.candidate, d.reference
|
|
75
|
+
if c is not None:
|
|
76
|
+
src = linecache.getline(c.filename, c.lineno).strip()
|
|
77
|
+
tail = " (the reference has %s at that point)" % _short(r.value) if r is not None else " (the reference never changes it)"
|
|
78
|
+
return "Where your run goes off track: line %d%s sets %s = %s%s." % (
|
|
79
|
+
c.lineno, (" (`%s`)" % src) if src else "", d.var, _short(c.value), tail,
|
|
80
|
+
)
|
|
81
|
+
return "Where it goes off track: the reference reaches %s = %s (%s), but your run never does." % (d.var, _short(r.value), _where(r))
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _skipped_note(cmp) -> str:
|
|
85
|
+
return "not compared step by step: %s (same name, but the runs barely agree on it, so it probably means different things; pass watch=[...] to force it)" % ", ".join(cmp.skipped)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _blocks(cmp):
|
|
89
|
+
"""The factual parts, shared by the text and HTML views: (label, call, yours, reference) rows."""
|
|
90
|
+
rows, p = [], cmp.probe
|
|
91
|
+
if cmp.outcomes_differ:
|
|
92
|
+
rows.append(("Your input", _call(cmp, cmp.args), _describe(cmp.candidate, cmp.args), _describe(cmp.reference, cmp.args)))
|
|
93
|
+
if p is not None and p.minimal is not None and (p.minimal != cmp.args or not cmp.outcomes_differ):
|
|
94
|
+
rows.append((
|
|
95
|
+
"Smallest failing input I found", _call(cmp, p.minimal),
|
|
96
|
+
_describe(p.minimal_candidate, p.minimal), _describe(p.minimal_reference, p.minimal),
|
|
97
|
+
))
|
|
98
|
+
return rows
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def to_text(cmp, include_ai: bool = True) -> str:
|
|
102
|
+
p = cmp.probe
|
|
103
|
+
if cmp.same:
|
|
104
|
+
text = "SAME behaviour: %s and %s give the same result" % (cmp.candidate.name, cmp.reference.name)
|
|
105
|
+
if p is not None and p.tried > 1:
|
|
106
|
+
text += ", also on %d other inputs I tried" % (p.tried - 1)
|
|
107
|
+
text += "."
|
|
108
|
+
return text + ("\n note: " + _skipped_note(cmp) if cmp.skipped else "")
|
|
109
|
+
lines = [_header(cmp)]
|
|
110
|
+
for label, call, yours, ref in _blocks(cmp):
|
|
111
|
+
lines += ["", "%s (%s):" % (label, call), " yours : " + yours, " reference : " + ref]
|
|
112
|
+
extra = list(p.patterns) if p is not None else []
|
|
113
|
+
if p is not None and p.failing and p.passing:
|
|
114
|
+
extra.append("It does agree on: " + "; ".join(_call(cmp, a) for a in p.passing) + ".")
|
|
115
|
+
if extra:
|
|
116
|
+
lines += [""] + extra
|
|
117
|
+
off = _off_track(cmp)
|
|
118
|
+
if off:
|
|
119
|
+
lines += ["", off]
|
|
120
|
+
if cmp.hints:
|
|
121
|
+
lines += ["", "Hints:"] + [" - " + h for h in cmp.hints[:3]]
|
|
122
|
+
if cmp.skipped:
|
|
123
|
+
lines += ["", "note: " + _skipped_note(cmp)]
|
|
124
|
+
if include_ai and cmp.ai_hint:
|
|
125
|
+
lines += ["", "AI hint:"] + [" " + part for part in cmp.ai_hint.splitlines() if part.strip()]
|
|
126
|
+
elif include_ai and cmp.ai_error:
|
|
127
|
+
lines += ["", "AI hint unavailable: " + _short(cmp.ai_error.replace("\n", " "), 200)]
|
|
128
|
+
for who, tr in (("candidate", cmp.candidate), ("reference", cmp.reference)):
|
|
129
|
+
if tr.truncated:
|
|
130
|
+
lines.append("note: the %s run was stopped after %d steps." % (who, tr.steps))
|
|
131
|
+
return "\n".join(lines)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def to_html(cmp) -> str:
|
|
135
|
+
e = html.escape
|
|
136
|
+
if cmp.same:
|
|
137
|
+
return "<p><b>SAME behaviour</b>: both give the same result.</p>"
|
|
138
|
+
p = cmp.probe
|
|
139
|
+
out = ["<p><b>%s</b></p>" % e(_header(cmp))]
|
|
140
|
+
rows = _blocks(cmp)
|
|
141
|
+
if rows:
|
|
142
|
+
body = "".join(
|
|
143
|
+
"<tr><th style='text-align:left'>%s</th><td><code>%s</code></td><td>yours: %s</td><td>reference: %s</td></tr>"
|
|
144
|
+
% (e(label), e(call), e(yours), e(ref))
|
|
145
|
+
for label, call, yours, ref in rows
|
|
146
|
+
)
|
|
147
|
+
out.append("<table>%s</table>" % body)
|
|
148
|
+
extra = list(p.patterns) if p is not None else []
|
|
149
|
+
if p is not None and p.failing and p.passing:
|
|
150
|
+
extra.append("It does agree on: " + "; ".join(_call(cmp, a) for a in p.passing) + ".")
|
|
151
|
+
out += ["<p>%s</p>" % e(x) for x in extra]
|
|
152
|
+
off = _off_track(cmp)
|
|
153
|
+
if off:
|
|
154
|
+
out.append("<p>%s</p>" % e(off))
|
|
155
|
+
if cmp.hints:
|
|
156
|
+
out.append("<p><b>Hints</b></p><ul>%s</ul>" % "".join("<li>%s</li>" % e(h) for h in cmp.hints[:3]))
|
|
157
|
+
if cmp.ai_hint:
|
|
158
|
+
out.append("<p><b>AI hint</b></p><p>%s</p>" % e(cmp.ai_hint).replace("\n", "<br>"))
|
|
159
|
+
elif cmp.ai_error:
|
|
160
|
+
out.append("<p><i>AI hint unavailable: %s</i></p>" % e(_short(cmp.ai_error, 200)))
|
|
161
|
+
return "".join(out)
|
difffinder/selfcheck.py
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"""Built-in sanity check. Run ``python -m difffinder`` or, in a notebook, ``difffinder.selfcheck()``.
|
|
2
|
+
|
|
3
|
+
It tests the things that matter on a hosted notebook server: tracing works, functions defined in
|
|
4
|
+
notebook cells (which have no real file on disk) are traced and shown with their source lines, the
|
|
5
|
+
HTML output renders, and the package has no dependencies that could conflict with the server's.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import builtins
|
|
11
|
+
import linecache
|
|
12
|
+
import platform
|
|
13
|
+
import sys
|
|
14
|
+
|
|
15
|
+
_CELL = '''
|
|
16
|
+
def candidate(n):
|
|
17
|
+
total = 0
|
|
18
|
+
for i in range(n):
|
|
19
|
+
total += i
|
|
20
|
+
return total
|
|
21
|
+
|
|
22
|
+
def reference(n):
|
|
23
|
+
total = 0
|
|
24
|
+
for i in range(n + 1):
|
|
25
|
+
total += i
|
|
26
|
+
return total
|
|
27
|
+
|
|
28
|
+
def sort_bad(xs):
|
|
29
|
+
xs = xs.sort()
|
|
30
|
+
return xs
|
|
31
|
+
|
|
32
|
+
def sort_good(xs):
|
|
33
|
+
xs.sort()
|
|
34
|
+
return xs
|
|
35
|
+
'''
|
|
36
|
+
|
|
37
|
+
# How Jupyter names the code of a cell, depending on the IPython/ipykernel version.
|
|
38
|
+
_CELL_NAMES = (("<ipython-input-1-difffinder>", "older IPython"), ("/tmp/ipykernel_1/difffinder.py", "ipykernel"))
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _as_notebook_cell(source: str, filename: str) -> dict:
|
|
42
|
+
"""Compile like a notebook does: no file on disk, source kept only in linecache."""
|
|
43
|
+
linecache.cache[filename] = (len(source), None, source.splitlines(True), filename)
|
|
44
|
+
namespace: dict = {}
|
|
45
|
+
exec(compile(source, filename, "exec"), namespace)
|
|
46
|
+
return namespace
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def selfcheck(verbose: bool = True) -> bool:
|
|
50
|
+
"""Run the checks, print a report, and return True if every check passed."""
|
|
51
|
+
from . import __version__, compare_traces, find_difference
|
|
52
|
+
|
|
53
|
+
results = []
|
|
54
|
+
|
|
55
|
+
def check(name: str, ok: bool) -> None:
|
|
56
|
+
results.append((name, bool(ok)))
|
|
57
|
+
|
|
58
|
+
shell = getattr(builtins, "get_ipython", lambda: None)()
|
|
59
|
+
where = "Jupyter/IPython (%s)" % type(shell).__name__ if shell is not None else "plain Python"
|
|
60
|
+
try:
|
|
61
|
+
for filename, label in _CELL_NAMES:
|
|
62
|
+
ns = _as_notebook_cell(_CELL, filename)
|
|
63
|
+
cmp = compare_traces(ns["candidate"], ns["reference"], args=(4,))
|
|
64
|
+
d = cmp.divergence
|
|
65
|
+
check("finds the off-by-one loop in a notebook cell (%s)" % label, d is not None and d.var == "i" and d.position == 4)
|
|
66
|
+
check("shows the cell's source line (%s)" % label, "range(n + 1)" in str(cmp))
|
|
67
|
+
check("shrinks to the smallest failing input (%s)" % label, cmp.probe is not None and cmp.probe.minimal == (1,))
|
|
68
|
+
bad = compare_traces(ns["sort_bad"], ns["sort_good"], args=([3, 1, 2],))
|
|
69
|
+
check("explains a list that became None", any("None" in h for h in bad.hints))
|
|
70
|
+
check("renders HTML for notebooks", "<table" in cmp._repr_html_())
|
|
71
|
+
hit = find_difference(ns["candidate"], ns["reference"], [0, 1, 2])
|
|
72
|
+
check("find_difference returns the first differing input", hit is not None and hit.args == (1,)) # n=0 gives 0 in both
|
|
73
|
+
finally:
|
|
74
|
+
for filename, _ in _CELL_NAMES:
|
|
75
|
+
linecache.cache.pop(filename, None)
|
|
76
|
+
|
|
77
|
+
try:
|
|
78
|
+
from importlib import metadata
|
|
79
|
+
|
|
80
|
+
# Optional extras (dev, hermes) are only installed on request, so they do not count.
|
|
81
|
+
required = [r for r in (metadata.requires("difffinder") or []) if "extra ==" not in r]
|
|
82
|
+
check("has no required dependencies, so it cannot conflict with the server's packages", not required)
|
|
83
|
+
except Exception: # running from a source folder: no installed metadata to inspect
|
|
84
|
+
pass
|
|
85
|
+
|
|
86
|
+
passed = all(ok for _, ok in results)
|
|
87
|
+
if verbose:
|
|
88
|
+
print("difffinder %s | Python %s | %s | %s" % (__version__, platform.python_version(), platform.system(), where))
|
|
89
|
+
for name, ok in results:
|
|
90
|
+
print(" %s %s" % ("PASS" if ok else "FAIL", name))
|
|
91
|
+
print("All %d checks passed." % len(results) if passed else "Some checks FAILED. Please report the lines above.")
|
|
92
|
+
return passed
|