tracecomparekit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
difffinder/outcome.py ADDED
@@ -0,0 +1,63 @@
1
+ """Run a function once and record what it visibly did."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import copy
6
+ import inspect
7
+ from dataclasses import dataclass, field
8
+ from typing import Any, Callable, Dict, Optional
9
+
10
+
11
+ def short(text: str, limit: int = 60) -> str:
12
+ """Shorten long text for display."""
13
+ return text if len(text) <= limit else text[: limit - 3] + "..."
14
+
15
+
16
+ @dataclass
17
+ class Outcome:
18
+ """What one call did: its result or exception, and the final state of its arguments."""
19
+
20
+ returned: Any = None
21
+ error_type: Optional[str] = None # for example "ValueError"; None if the call returned normally
22
+ error_message: str = ""
23
+ inputs: Dict[str, str] = field(default_factory=dict) # argument name -> repr before the call
24
+ arguments: Dict[str, Any] = field(default_factory=dict) # argument name -> object after the call
25
+
26
+ @property
27
+ def raised(self) -> bool:
28
+ return self.error_type is not None
29
+
30
+ def describe(self) -> str:
31
+ """The result in a few words, for example ``returned 6`` or ``raised ValueError: ...``."""
32
+ if self.raised:
33
+ detail = (": " + self.error_message) if self.error_message else ""
34
+ return "raised %s%s" % (self.error_type, detail)
35
+ return "returned %s" % short(repr(self.returned))
36
+
37
+
38
+ def _bind(func: Callable[..., Any], args: tuple, kwargs: dict) -> Dict[str, Any]:
39
+ """Map parameter names to the objects passed in (the objects are shared, not copied)."""
40
+ try:
41
+ return dict(inspect.signature(func).bind(*args, **kwargs).arguments)
42
+ except (TypeError, ValueError): # no signature available, or the arguments do not fit it
43
+ named = {"arg%d" % i: a for i, a in enumerate(args)}
44
+ named.update(kwargs)
45
+ return named
46
+
47
+
48
+ def run(func: Callable[..., Any], args: tuple = (), kwargs: Optional[dict] = None) -> Outcome:
49
+ """Call ``func(*args, **kwargs)`` on a deep copy of the arguments and record what happened.
50
+
51
+ An exception is recorded, not raised. Because the call works on copies, your own data is never changed,
52
+ but changes the function makes to its arguments are captured in ``Outcome.arguments``.
53
+ """
54
+ args = copy.deepcopy(tuple(args))
55
+ kwargs = copy.deepcopy(dict(kwargs or {}))
56
+ arguments = _bind(func, args, kwargs)
57
+ inputs = {name: short(repr(value), 500) for name, value in arguments.items()}
58
+ try:
59
+ outcome = Outcome(returned=func(*args, **kwargs))
60
+ except Exception as exc:
61
+ outcome = Outcome(error_type=type(exc).__name__, error_message=str(exc))
62
+ outcome.inputs, outcome.arguments = inputs, arguments
63
+ return outcome
difffinder/probe.py ADDED
@@ -0,0 +1,301 @@
1
+ """Black-box testing: find inputs where two functions disagree, shrink one, and describe the pattern.
2
+
3
+ This works however differently the two functions are written, because it only looks at what each one
4
+ visibly does (return value, exception, and what it did to its arguments), never at their internals.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import contextlib
10
+ import copy
11
+ import io
12
+ import random
13
+ from dataclasses import dataclass, field
14
+ from itertools import combinations, zip_longest
15
+ from typing import Any, Dict, List, Optional
16
+
17
+ from .tracer import RAISE, RETURN, Trace, record
18
+
19
+ MAX_SINGLES = 55 # inputs made by changing one argument
20
+ MAX_PAIRS = 25 # inputs made by changing two arguments at once
21
+ SHRINK_BUDGET = 150
22
+ STEP_LIMIT = 20_000
23
+
24
+
25
+ def _isnum(x: Any) -> bool:
26
+ return type(x) in (int, float) # bool is not a number here
27
+
28
+
29
+ def outcome(trace: Trace) -> tuple:
30
+ """What a call visibly did. Exceptions count by type only, so message wording is not a difference."""
31
+ items = []
32
+ for e in trace.events:
33
+ if e.var.startswith("<"):
34
+ items.append((e.var, e.value.split(":")[0] if e.var == RAISE else e.value))
35
+ if trace.truncated:
36
+ items.append(("<timeout>", ""))
37
+ return tuple(items)
38
+
39
+
40
+ def run_pair(candidate, reference, args, kwargs):
41
+ """Run both functions on copies of ``args``. Returns (candidate trace, reference trace, differ)."""
42
+ with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
43
+ ta = record(candidate, copy.deepcopy(args), copy.deepcopy(kwargs), max_steps=STEP_LIMIT)
44
+ tb = record(reference, copy.deepcopy(args), copy.deepcopy(kwargs), max_steps=STEP_LIMIT)
45
+ return ta, tb, outcome(ta) != outcome(tb)
46
+
47
+
48
+ def _unique(seq, original) -> list:
49
+ seen, out = {repr(original)}, []
50
+ for item in seq:
51
+ r = repr(item)
52
+ if r not in seen:
53
+ seen.add(r)
54
+ out.append(item)
55
+ return out
56
+
57
+
58
+ def _alternatives(value) -> list:
59
+ """Interesting variations of one argument: edge cases and small mutations."""
60
+ t, out = type(value), []
61
+ if t is bool:
62
+ out = [not value]
63
+ elif t is int:
64
+ out = [0, 1, -1, 2, -2, value + 1, value - 1, -value, value * 2, 10, 100]
65
+ elif t is float:
66
+ out = [0.0, 1.0, -1.0, -value, value + 1, value / 2]
67
+ elif t is str:
68
+ out = ["", "a", "ab", "aa", value[:1], value[:-1], value[::-1], value.upper(), " "]
69
+ elif t in (list, tuple):
70
+ items, make = list(value), t
71
+ out = [make()]
72
+ out += [make(items[:i] + items[i + 1 :]) for i in range(min(len(items), 6))]
73
+ if items:
74
+ out += [make(items[::-1]), make(items[:1]), make(items + items[:1])]
75
+ try:
76
+ out.append(make(sorted(items)))
77
+ except TypeError:
78
+ pass
79
+ if not items or any(_isnum(x) for x in items):
80
+ out += [make(items + [e]) for e in (0, -1, -3)]
81
+ if items and all(_isnum(x) for x in items):
82
+ out.append(make([-x for x in items]))
83
+ for i in range(min(len(items), 4)):
84
+ out += [make(items[:i] + [alt] + items[i + 1 :]) for alt in _alternatives(items[i])[:3]]
85
+ elif t is dict:
86
+ out = [{}] + [{k: v for k, v in value.items() if k != key} for key in list(value)[:4]]
87
+ return _unique(out, value)
88
+
89
+
90
+ def _variants(args: tuple) -> List[tuple]:
91
+ rng = random.Random(0) # fixed seed: the same input always gives the same report
92
+ groups = [[args[:i] + (alt,) + args[i + 1 :] for alt in _alternatives(a)] for i, a in enumerate(args)]
93
+ seen, out = {repr(args)}, []
94
+
95
+ def add(candidate):
96
+ r = repr(candidate)
97
+ if r not in seen:
98
+ seen.add(r)
99
+ out.append(candidate)
100
+
101
+ for row in zip_longest(*groups):
102
+ for c in row:
103
+ if c is not None:
104
+ add(c)
105
+ del out[MAX_SINGLES:]
106
+ changeable = [(i, _alternatives(a)) for i, a in enumerate(args)]
107
+ changeable = [(i, alts) for i, alts in changeable if alts]
108
+ if len(changeable) >= 2:
109
+ for _ in range(MAX_PAIRS):
110
+ (i, ai), (j, aj) = rng.sample(changeable, 2)
111
+ c = list(args)
112
+ c[i], c[j] = rng.choice(ai), rng.choice(aj)
113
+ add(tuple(c))
114
+ return out
115
+
116
+
117
+ def _shrinks(value) -> list:
118
+ """Simpler versions of one argument, simplest first."""
119
+ t, out = type(value), []
120
+ try:
121
+ if t is bool:
122
+ out = [False] if value else []
123
+ elif t is int:
124
+ out = [0, int(value / 2), value - (1 if value > 0 else -1)] if value else []
125
+ elif t is float:
126
+ out = [0.0, float(round(value)), value / 2] if value else []
127
+ elif t in (list, tuple, str):
128
+ n = len(value)
129
+ if n:
130
+ out.append(value[:0])
131
+ if n >= 4:
132
+ out += [value[: n // 2], value[n // 2 :]]
133
+ out += [value[:i] + value[i + 1 :] for i in range(n)]
134
+ if t is not str:
135
+ for i, x in enumerate(value):
136
+ out += [t(list(value[:i]) + [s] + list(value[i + 1 :])) for s in _shrinks(x)[:2]]
137
+ elif t is dict:
138
+ out = [{}] + [{k: v for k, v in value.items() if k != key} for key in value]
139
+ except (ValueError, OverflowError):
140
+ out = []
141
+ return _unique(out, value)
142
+
143
+
144
+ def _shrink(candidate, reference, args, kwargs, ta, tb):
145
+ """Greedily simplify a failing input while the two functions still disagree."""
146
+ best, spent, improved = args, 0, True
147
+ while improved and spent < SHRINK_BUDGET:
148
+ improved = False
149
+ for i, a in enumerate(best):
150
+ for s in _shrinks(a):
151
+ if spent >= SHRINK_BUDGET:
152
+ break
153
+ trial = best[:i] + (s,) + best[i + 1 :]
154
+ spent += 1
155
+ t1, t2, bad = run_pair(candidate, reference, trial, kwargs)
156
+ if bad:
157
+ best, ta, tb, improved = trial, t1, t2, True
158
+ break
159
+ if improved:
160
+ break
161
+ return best, ta, tb
162
+
163
+
164
+ def _features(args: tuple, names: List[str]) -> Dict[str, bool]:
165
+ """Simple yes/no facts about an input, used to describe when the functions disagree."""
166
+ f: Dict[str, bool] = {}
167
+ scalars = [(n, a) for n, a in zip(names, args) if _isnum(a)]
168
+ for n, a in zip(names, args):
169
+ tag = "`%s`" % n
170
+ if type(a) in (list, tuple):
171
+ items = list(a)
172
+ nums = [x for x in items if _isnum(x)]
173
+ f[tag + " is empty"] = not items
174
+ f[tag + " has exactly one element"] = len(items) == 1
175
+ f[tag + " has more than one element"] = len(items) > 1
176
+ f[tag + " has repeated elements"] = len({repr(x) for x in items}) < len(items)
177
+ if nums:
178
+ f[tag + " contains a negative number"] = any(x < 0 for x in nums)
179
+ f[tag + " contains 0"] = any(x == 0 for x in nums)
180
+ for m, s in scalars:
181
+ f["%s contains a number below `%s`" % (tag, m)] = any(x < s for x in nums)
182
+ f["%s contains a number above `%s`" % (tag, m)] = any(x > s for x in nums)
183
+ f["%s contains a number equal to `%s`" % (tag, m)] = any(x == s for x in nums)
184
+ elif type(a) is str:
185
+ f[tag + " is empty"] = not a
186
+ f[tag + " has exactly one character"] = len(a) == 1
187
+ f[tag + " contains a digit"] = any(c.isdigit() for c in a)
188
+ f[tag + " contains an upper-case letter"] = any(c.isupper() for c in a)
189
+ f[tag + " contains a space"] = " " in a
190
+ elif _isnum(a):
191
+ f[tag + " is negative"] = a < 0
192
+ f[tag + " is 0"] = a == 0
193
+ f[tag + " is positive"] = a > 0
194
+ elif type(a) is dict:
195
+ f[tag + " is empty"] = not a
196
+ for (m, x), (n, y) in combinations(scalars, 2):
197
+ f["`%s` is greater than `%s`" % (m, n)] = x > y
198
+ f["`%s` equals `%s`" % (m, n)] = x == y
199
+ return f
200
+
201
+
202
+ def find_patterns(rows, names: List[str], limit: int = 2) -> List[str]:
203
+ """Describe, from the inputs tried, a simple property that goes with the functions disagreeing."""
204
+ failed = sum(1 for _, bad in rows if bad)
205
+ if not failed or failed == len(rows):
206
+ return []
207
+ table = [(_features(a, names), bad) for a, bad in rows]
208
+ keys: List[str] = []
209
+ for feats, _ in table:
210
+ keys += [k for k in feats if k not in keys]
211
+ scored = []
212
+ for k in keys:
213
+ tp = sum(1 for f, bad in table if f.get(k) and bad)
214
+ total = sum(1 for f, _ in table if f.get(k))
215
+ if tp >= 2 and tp / total >= 0.85 and tp / failed >= 0.5:
216
+ scored.append((tp / total, tp / failed, k, tp, total))
217
+ scored.sort(key=lambda s: (-s[0], -s[1], len(s[2])))
218
+ return [
219
+ "It %s %s (%d of %d such inputs I tried)." % ("fails whenever" if p == 1 else "usually fails when", k, tp, total)
220
+ for p, _, k, tp, total in scored[:limit]
221
+ ]
222
+
223
+
224
+ def _returned(trace: Trace):
225
+ hist = trace.history(RETURN)
226
+ return hist[-1].value if hist else None
227
+
228
+
229
+ def _raised(trace: Trace):
230
+ hist = trace.history(RAISE)
231
+ return hist[-1].value.split(":")[0] if hist else None
232
+
233
+
234
+ def find_systematic(pairs, failing: int) -> List[str]:
235
+ """Spot differences that hold on (almost) every input, which point to a basic mistake."""
236
+ tried = len(pairs)
237
+ got = [(_returned(a), _returned(b)) for a, b in pairs]
238
+ both = [(x, y) for x, y in got if x is not None and y is not None]
239
+ out: List[str] = []
240
+ if len(both) >= 3 and all(x in ("True", "False") and y in ("True", "False") and x != y for x, y in both):
241
+ out.append(
242
+ "Your function returns the opposite of the reference on every input where both give True or False "
243
+ "(%d of %d). Check what your final return statement actually tests." % (len(both), tried)
244
+ )
245
+ elif failing == tried and tried >= 5:
246
+ out.append(
247
+ "They disagree on every input I tried, so this is not an edge case: something basic is different."
248
+ )
249
+ mine = [x for x, _ in both]
250
+ theirs = {y for _, y in both}
251
+ if len(mine) >= 5 and len(set(mine)) == 1 and len(theirs) >= 2:
252
+ out.append("Your function returned %s on every input I tried, while the reference's result depends on the input." % _clip(mine[0]))
253
+ errors = [_raised(a) for a, _ in pairs]
254
+ kinds = {e for e in errors if e}
255
+ if kinds:
256
+ common = max(kinds, key=errors.count)
257
+ n_raise = errors.count(common)
258
+ ref_raise = sum(1 for _, b in pairs if _raised(b))
259
+ if n_raise >= 3 and n_raise > ref_raise:
260
+ out.append("Your function raises %s on %d of the %d inputs I tried; the reference does so on %d." % (common, n_raise, tried, ref_raise))
261
+ return out
262
+
263
+
264
+ def _clip(text: str, limit: int = 30) -> str:
265
+ return text if len(text) <= limit else text[: limit - 3] + "..."
266
+
267
+
268
+ @dataclass
269
+ class Probe:
270
+ """What testing many inputs revealed."""
271
+
272
+ tried: int
273
+ failing: int
274
+ minimal: Optional[tuple] = None
275
+ minimal_candidate: Optional[Trace] = None
276
+ minimal_reference: Optional[Trace] = None
277
+ patterns: List[str] = field(default_factory=list)
278
+ passing: List[tuple] = field(default_factory=list)
279
+
280
+
281
+ def run_probe(candidate, reference, args, kwargs, names, first_failed, ta, tb) -> Probe:
282
+ """Try variations of ``args``, shrink a failing one, and look for a pattern."""
283
+ rows = [(args, first_failed)]
284
+ pairs = [(ta, tb)]
285
+ failing_runs: Dict[str, tuple] = {repr(args): (args, ta, tb)} if first_failed else {}
286
+ for v in _variants(args):
287
+ t1, t2, bad = run_pair(candidate, reference, v, kwargs)
288
+ rows.append((v, bad))
289
+ pairs.append((t1, t2))
290
+ if bad:
291
+ failing_runs[repr(v)] = (v, t1, t2)
292
+ probe = Probe(tried=len(rows), failing=sum(1 for _, bad in rows if bad))
293
+ if not failing_runs:
294
+ return probe
295
+ systematic = find_systematic(pairs, probe.failing)
296
+ start = args if first_failed else min((r[0] for r in failing_runs.values()), key=lambda a: len(repr(a)))
297
+ _, t1, t2 = failing_runs[repr(start)]
298
+ probe.minimal, probe.minimal_candidate, probe.minimal_reference = _shrink(candidate, reference, start, kwargs, t1, t2)
299
+ probe.patterns = systematic + find_patterns(rows, names)
300
+ probe.passing = sorted((a for a, bad in rows if not bad), key=lambda a: len(repr(a)))[:2]
301
+ return probe
difffinder/render.py ADDED
@@ -0,0 +1,161 @@
1
+ """Turn a Comparison into plain text or notebook HTML."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import html
6
+ import linecache
7
+
8
+ from .tracer import RAISE, RETURN, _safe_repr
9
+
10
+
11
+ def _short(text: str, limit: int = 60) -> str:
12
+ return text if len(text) <= limit else text[: limit - 3] + "..."
13
+
14
+
15
+ def _names(cmp, n: int):
16
+ return [cmp.candidate.arg_names.get("<arg%d>" % i) or "arg%d" % i for i in range(n)]
17
+
18
+
19
+ def _call(cmp, args: tuple) -> str:
20
+ return ", ".join("%s=%s" % (n, _short(repr(a), 40)) for n, a in zip(_names(cmp, len(args)), args))
21
+
22
+
23
+ def _describe(trace, args: tuple) -> str:
24
+ """What a call visibly did, in a few words."""
25
+ parts = []
26
+ raised, returned = trace.history(RAISE), trace.history(RETURN)
27
+ if raised:
28
+ parts.append("raises " + _short(raised[-1].value, 50))
29
+ elif returned:
30
+ parts.append("returns " + _short(returned[-1].value, 50))
31
+ elif trace.truncated:
32
+ parts.append("never finishes (stopped after %d steps)" % trace.steps)
33
+ for ev in trace.events:
34
+ if ev.var.startswith("<arg"):
35
+ idx = int(ev.var[4:-1])
36
+ if idx < len(args) and ev.value != _safe_repr(args[idx]):
37
+ parts.append("changes `%s` to %s" % (trace.arg_names.get(ev.var, ev.var), _short(ev.value, 50)))
38
+ return ", ".join(parts) or "does nothing visible"
39
+
40
+
41
+ def _where(ev) -> str:
42
+ src = linecache.getline(ev.filename, ev.lineno).strip()
43
+ return "line %d in %s()%s" % (ev.lineno, ev.func, (": " + src) if src else "")
44
+
45
+
46
+ def _label(cmp, var: str) -> str:
47
+ if var == RETURN:
48
+ return "the return value"
49
+ if var == RAISE:
50
+ return "how the call ended"
51
+ if var.startswith("<arg"):
52
+ return "the final state of argument '%s'" % (cmp.candidate.arg_names.get(var) or var)
53
+ return "variable '%s'" % var
54
+
55
+
56
+ def _header(cmp) -> str:
57
+ p, d = cmp.probe, cmp.divergence
58
+ if cmp.outcomes_differ and p is not None and p.failing:
59
+ return "DIFFERENT: yours disagrees with the reference on %d of the %d inputs I tried (including yours)." % (p.failing, p.tried)
60
+ if cmp.outcomes_differ:
61
+ return "DIFFERENT: the two functions give different results on your input."
62
+ if p is not None and p.failing:
63
+ return "DIFFERENT: they agree on your input, but disagree on %d of %d other inputs I tried." % (p.failing, p.tried - 1)
64
+ if d is not None:
65
+ return "DIFFERENT: the runs first diverge on %s (change #%d)." % (_label(cmp, d.var), d.position + 1)
66
+ return "DIFFERENT."
67
+
68
+
69
+ def _off_track(cmp) -> str:
70
+ """One sentence on the first step where the candidate's run leaves the reference's."""
71
+ d = cmp.divergence
72
+ if d is None or d.var.startswith("<"):
73
+ return ""
74
+ c, r = d.candidate, d.reference
75
+ if c is not None:
76
+ src = linecache.getline(c.filename, c.lineno).strip()
77
+ tail = " (the reference has %s at that point)" % _short(r.value) if r is not None else " (the reference never changes it)"
78
+ return "Where your run goes off track: line %d%s sets %s = %s%s." % (
79
+ c.lineno, (" (`%s`)" % src) if src else "", d.var, _short(c.value), tail,
80
+ )
81
+ return "Where it goes off track: the reference reaches %s = %s (%s), but your run never does." % (d.var, _short(r.value), _where(r))
82
+
83
+
84
+ def _skipped_note(cmp) -> str:
85
+ return "not compared step by step: %s (same name, but the runs barely agree on it, so it probably means different things; pass watch=[...] to force it)" % ", ".join(cmp.skipped)
86
+
87
+
88
+ def _blocks(cmp):
89
+ """The factual parts, shared by the text and HTML views: (label, call, yours, reference) rows."""
90
+ rows, p = [], cmp.probe
91
+ if cmp.outcomes_differ:
92
+ rows.append(("Your input", _call(cmp, cmp.args), _describe(cmp.candidate, cmp.args), _describe(cmp.reference, cmp.args)))
93
+ if p is not None and p.minimal is not None and (p.minimal != cmp.args or not cmp.outcomes_differ):
94
+ rows.append((
95
+ "Smallest failing input I found", _call(cmp, p.minimal),
96
+ _describe(p.minimal_candidate, p.minimal), _describe(p.minimal_reference, p.minimal),
97
+ ))
98
+ return rows
99
+
100
+
101
+ def to_text(cmp, include_ai: bool = True) -> str:
102
+ p = cmp.probe
103
+ if cmp.same:
104
+ text = "SAME behaviour: %s and %s give the same result" % (cmp.candidate.name, cmp.reference.name)
105
+ if p is not None and p.tried > 1:
106
+ text += ", also on %d other inputs I tried" % (p.tried - 1)
107
+ text += "."
108
+ return text + ("\n note: " + _skipped_note(cmp) if cmp.skipped else "")
109
+ lines = [_header(cmp)]
110
+ for label, call, yours, ref in _blocks(cmp):
111
+ lines += ["", "%s (%s):" % (label, call), " yours : " + yours, " reference : " + ref]
112
+ extra = list(p.patterns) if p is not None else []
113
+ if p is not None and p.failing and p.passing:
114
+ extra.append("It does agree on: " + "; ".join(_call(cmp, a) for a in p.passing) + ".")
115
+ if extra:
116
+ lines += [""] + extra
117
+ off = _off_track(cmp)
118
+ if off:
119
+ lines += ["", off]
120
+ if cmp.hints:
121
+ lines += ["", "Hints:"] + [" - " + h for h in cmp.hints[:3]]
122
+ if cmp.skipped:
123
+ lines += ["", "note: " + _skipped_note(cmp)]
124
+ if include_ai and cmp.ai_hint:
125
+ lines += ["", "AI hint:"] + [" " + part for part in cmp.ai_hint.splitlines() if part.strip()]
126
+ elif include_ai and cmp.ai_error:
127
+ lines += ["", "AI hint unavailable: " + _short(cmp.ai_error.replace("\n", " "), 200)]
128
+ for who, tr in (("candidate", cmp.candidate), ("reference", cmp.reference)):
129
+ if tr.truncated:
130
+ lines.append("note: the %s run was stopped after %d steps." % (who, tr.steps))
131
+ return "\n".join(lines)
132
+
133
+
134
+ def to_html(cmp) -> str:
135
+ e = html.escape
136
+ if cmp.same:
137
+ return "<p><b>SAME behaviour</b>: both give the same result.</p>"
138
+ p = cmp.probe
139
+ out = ["<p><b>%s</b></p>" % e(_header(cmp))]
140
+ rows = _blocks(cmp)
141
+ if rows:
142
+ body = "".join(
143
+ "<tr><th style='text-align:left'>%s</th><td><code>%s</code></td><td>yours: %s</td><td>reference: %s</td></tr>"
144
+ % (e(label), e(call), e(yours), e(ref))
145
+ for label, call, yours, ref in rows
146
+ )
147
+ out.append("<table>%s</table>" % body)
148
+ extra = list(p.patterns) if p is not None else []
149
+ if p is not None and p.failing and p.passing:
150
+ extra.append("It does agree on: " + "; ".join(_call(cmp, a) for a in p.passing) + ".")
151
+ out += ["<p>%s</p>" % e(x) for x in extra]
152
+ off = _off_track(cmp)
153
+ if off:
154
+ out.append("<p>%s</p>" % e(off))
155
+ if cmp.hints:
156
+ out.append("<p><b>Hints</b></p><ul>%s</ul>" % "".join("<li>%s</li>" % e(h) for h in cmp.hints[:3]))
157
+ if cmp.ai_hint:
158
+ out.append("<p><b>AI hint</b></p><p>%s</p>" % e(cmp.ai_hint).replace("\n", "<br>"))
159
+ elif cmp.ai_error:
160
+ out.append("<p><i>AI hint unavailable: %s</i></p>" % e(_short(cmp.ai_error, 200)))
161
+ return "".join(out)
@@ -0,0 +1,92 @@
1
+ """Built-in sanity check. Run ``python -m difffinder`` or, in a notebook, ``difffinder.selfcheck()``.
2
+
3
+ It tests the things that matter on a hosted notebook server: tracing works, functions defined in
4
+ notebook cells (which have no real file on disk) are traced and shown with their source lines, the
5
+ HTML output renders, and the package has no dependencies that could conflict with the server's.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import builtins
11
+ import linecache
12
+ import platform
13
+ import sys
14
+
15
+ _CELL = '''
16
+ def candidate(n):
17
+ total = 0
18
+ for i in range(n):
19
+ total += i
20
+ return total
21
+
22
+ def reference(n):
23
+ total = 0
24
+ for i in range(n + 1):
25
+ total += i
26
+ return total
27
+
28
+ def sort_bad(xs):
29
+ xs = xs.sort()
30
+ return xs
31
+
32
+ def sort_good(xs):
33
+ xs.sort()
34
+ return xs
35
+ '''
36
+
37
+ # How Jupyter names the code of a cell, depending on the IPython/ipykernel version.
38
+ _CELL_NAMES = (("<ipython-input-1-difffinder>", "older IPython"), ("/tmp/ipykernel_1/difffinder.py", "ipykernel"))
39
+
40
+
41
+ def _as_notebook_cell(source: str, filename: str) -> dict:
42
+ """Compile like a notebook does: no file on disk, source kept only in linecache."""
43
+ linecache.cache[filename] = (len(source), None, source.splitlines(True), filename)
44
+ namespace: dict = {}
45
+ exec(compile(source, filename, "exec"), namespace)
46
+ return namespace
47
+
48
+
49
+ def selfcheck(verbose: bool = True) -> bool:
50
+ """Run the checks, print a report, and return True if every check passed."""
51
+ from . import __version__, compare_traces, find_difference
52
+
53
+ results = []
54
+
55
+ def check(name: str, ok: bool) -> None:
56
+ results.append((name, bool(ok)))
57
+
58
+ shell = getattr(builtins, "get_ipython", lambda: None)()
59
+ where = "Jupyter/IPython (%s)" % type(shell).__name__ if shell is not None else "plain Python"
60
+ try:
61
+ for filename, label in _CELL_NAMES:
62
+ ns = _as_notebook_cell(_CELL, filename)
63
+ cmp = compare_traces(ns["candidate"], ns["reference"], args=(4,))
64
+ d = cmp.divergence
65
+ check("finds the off-by-one loop in a notebook cell (%s)" % label, d is not None and d.var == "i" and d.position == 4)
66
+ check("shows the cell's source line (%s)" % label, "range(n + 1)" in str(cmp))
67
+ check("shrinks to the smallest failing input (%s)" % label, cmp.probe is not None and cmp.probe.minimal == (1,))
68
+ bad = compare_traces(ns["sort_bad"], ns["sort_good"], args=([3, 1, 2],))
69
+ check("explains a list that became None", any("None" in h for h in bad.hints))
70
+ check("renders HTML for notebooks", "<table" in cmp._repr_html_())
71
+ hit = find_difference(ns["candidate"], ns["reference"], [0, 1, 2])
72
+ check("find_difference returns the first differing input", hit is not None and hit.args == (1,)) # n=0 gives 0 in both
73
+ finally:
74
+ for filename, _ in _CELL_NAMES:
75
+ linecache.cache.pop(filename, None)
76
+
77
+ try:
78
+ from importlib import metadata
79
+
80
+ # Optional extras (dev, hermes) are only installed on request, so they do not count.
81
+ required = [r for r in (metadata.requires("difffinder") or []) if "extra ==" not in r]
82
+ check("has no required dependencies, so it cannot conflict with the server's packages", not required)
83
+ except Exception: # running from a source folder: no installed metadata to inspect
84
+ pass
85
+
86
+ passed = all(ok for _, ok in results)
87
+ if verbose:
88
+ print("difffinder %s | Python %s | %s | %s" % (__version__, platform.python_version(), platform.system(), where))
89
+ for name, ok in results:
90
+ print(" %s %s" % ("PASS" if ok else "FAIL", name))
91
+ print("All %d checks passed." % len(results) if passed else "Some checks FAILED. Please report the lines above.")
92
+ return passed