tracecomparekit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
difffinder/__init__.py ADDED
@@ -0,0 +1,21 @@
1
+ """DiffFinder: check a Python function against a reference solution and see how they differ.
2
+
3
+ >>> from difffinder import compare
4
+ >>> compare(my_function, reference_function, args=(4,))
5
+ """
6
+
7
+ from .comparison import Comparison, Difference, compare
8
+ from .ai import explain, hermes_llm, openai_compatible
9
+ from .diff import Divergence, TraceComparison, compare_traces, find_difference, find_divergence
10
+ from .outcome import Outcome, run
11
+ from .selfcheck import selfcheck
12
+ from .tracer import RAISE, RETURN, Event, StepLimitExceeded, Trace, record
13
+
14
+ __version__ = "0.1.0"
15
+
16
+ __all__ = [
17
+ "compare", "run", "Comparison", "Difference", "Outcome",
18
+ "compare_traces", "TraceComparison", "Divergence", "find_difference", "find_divergence",
19
+ "record", "Trace", "Event", "StepLimitExceeded", "RETURN", "RAISE",
20
+ "explain", "openai_compatible", "hermes_llm", "selfcheck", "__version__",
21
+ ]
difffinder/__main__.py ADDED
@@ -0,0 +1,3 @@
1
+ from .selfcheck import selfcheck
2
+
3
+ raise SystemExit(0 if selfcheck() else 1)
difffinder/ai.py ADDED
@@ -0,0 +1,209 @@
1
+ """Optional: ask a language model for a short hint. Off unless you ask for it.
2
+
3
+ Quickest way: set an API key, then pass ``ai=True``::
4
+
5
+ import os, getpass
6
+ os.environ["DEEPSEEK_API_KEY"] = getpass.getpass() # or OPENAI_API_KEY
7
+ compare_traces(mine, reference, args=(...), ai=True)
8
+
9
+ Ready-made models: :func:`openai_compatible` (any OpenAI-style endpoint, standard library only) and
10
+ :func:`hermes_llm` (the Hermes Agent via ``hermes-acp-sdk``). Or pass any function ``llm(prompt) -> str``.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import asyncio
16
+ import inspect
17
+ import json
18
+ import os
19
+ import tempfile
20
+ import threading
21
+ import urllib.request
22
+ from typing import Callable, Optional
23
+
24
+ SYSTEM = (
25
+ "You are a friendly programming tutor talking to one first-year student. You are given the results of "
26
+ "automatically testing their function against a correct reference function: the inputs where the two "
27
+ "differ, the smallest failing input, a pattern for when it fails, and what each function did. "
28
+ "Reply in at most 5 short sentences, in plain spoken English, not as a report, and without repeating "
29
+ "numbers the student can already see. Say what their code most likely gets wrong, pointing at one "
30
+ "specific line or variable, then ask ONE guiding question. "
31
+ "Do not write corrected code and do not reveal the reference solution (do not quote or describe it). "
32
+ "No praise, no apologies."
33
+ )
34
+
35
+ _KEY_VARS = ("DIFFFINDER_API_KEY", "SOCRATIC_LLM_API_KEY", "DEEPSEEK_API_KEY", "OPENAI_API_KEY")
36
+
37
+
38
+ def _source(fn) -> str:
39
+ try:
40
+ return inspect.getsource(fn)
41
+ except (OSError, TypeError):
42
+ return "(source not available)"
43
+
44
+
45
+ def _short(text: str, limit: int = 80) -> str:
46
+ return text if len(text) <= limit else text[: limit - 3] + "..."
47
+
48
+
49
+ def build_prompt(cmp, share_reference: bool = True) -> str:
50
+ """The text sent to the model: the code, the evidence, and what the student's run actually did."""
51
+ from .render import to_text
52
+
53
+ d = cmp.divergence
54
+ parts = ["INPUT: %r" % (cmp.args,), "STUDENT'S FUNCTION:\n" + _source(cmp.candidate_fn)]
55
+ if share_reference:
56
+ parts.append("REFERENCE FUNCTION (for your eyes only, never quote or describe it):\n" + _source(cmp.reference_fn))
57
+ parts.append("AUTOMATIC COMPARISON:\n" + to_text(cmp, include_ai=False))
58
+ if d is not None:
59
+ def values(trace):
60
+ vals = [_short(e.value, 40) for e in trace.history(d.var)][:15]
61
+ return ", ".join(vals) if vals else "(never set)"
62
+
63
+ parts.append(
64
+ "VALUES OF '%s' IN ORDER:\n student : %s\n reference: %s" % (d.var, values(cmp.candidate), values(cmp.reference))
65
+ )
66
+ probe = cmp.probe
67
+ focus = probe.minimal_candidate if probe is not None and probe.minimal_candidate is not None else cmp.candidate
68
+ recent = [e for e in focus.events if not e.var.startswith("<")][-12:]
69
+ if recent:
70
+ parts.append(
71
+ "LAST CHANGES IN THE STUDENT'S RUN%s:\n" % (" (on the smallest failing input)" if focus is not cmp.candidate else "")
72
+ + "\n".join(" line %d: %s = %s" % (e.lineno, e.var, _short(e.value, 60)) for e in recent)
73
+ )
74
+ return "\n\n".join(parts)
75
+
76
+
77
+ def openai_compatible(
78
+ base_url: Optional[str] = None,
79
+ api_key: Optional[str] = None,
80
+ model: Optional[str] = None,
81
+ timeout: float = 60.0,
82
+ ) -> Callable[[str], str]:
83
+ """Return ``llm(prompt) -> str`` for an OpenAI-style chat endpoint.
84
+
85
+ Everything can come from the environment. Key: DIFFFINDER_API_KEY, SOCRATIC_LLM_API_KEY, DEEPSEEK_API_KEY
86
+ or OPENAI_API_KEY (first one found). With a DeepSeek key, the DeepSeek endpoint and ``deepseek-chat`` are
87
+ used by default. Override with DIFFFINDER_BASE_URL / DIFFFINDER_MODEL (``SOCRATIC_LLM_*`` and
88
+ ``OPENAI_BASE_URL`` / ``OPENAI_MODEL`` are read too).
89
+ """
90
+ env = os.environ.get
91
+ own_key = next((env(v) for v in _KEY_VARS[:3] if env(v)), None)
92
+ openai_key = env("OPENAI_API_KEY")
93
+ key = api_key if api_key is not None else (own_key or openai_key or "")
94
+ use_openai = api_key is None and not own_key and bool(openai_key)
95
+ base = (
96
+ base_url
97
+ or env("DIFFFINDER_BASE_URL")
98
+ or env("SOCRATIC_LLM_BASE_URL")
99
+ or env("OPENAI_BASE_URL")
100
+ or ("https://api.openai.com/v1" if use_openai else "https://api.deepseek.com")
101
+ )
102
+ model = model or env("DIFFFINDER_MODEL") or env("SOCRATIC_LLM_MODEL") or env("OPENAI_MODEL") or (None if use_openai else "deepseek-chat")
103
+ if not key and not (base_url or env("DIFFFINDER_BASE_URL")):
104
+ raise RuntimeError(
105
+ "No API key found. Set DEEPSEEK_API_KEY (or OPENAI_API_KEY) first, for example:\n"
106
+ " import os, getpass; os.environ['DEEPSEEK_API_KEY'] = getpass.getpass()"
107
+ )
108
+ if not model:
109
+ raise RuntimeError("Set DIFFFINDER_MODEL to the model name you want to use (for example your OpenAI model).")
110
+ url = base.rstrip("/") + "/chat/completions"
111
+
112
+ def ask(prompt: str) -> str:
113
+ body = json.dumps(
114
+ {
115
+ "model": model,
116
+ "temperature": 0.2,
117
+ "messages": [{"role": "system", "content": SYSTEM}, {"role": "user", "content": prompt}],
118
+ }
119
+ ).encode()
120
+ headers = {"Content-Type": "application/json", "Authorization": "Bearer " + key}
121
+ with urllib.request.urlopen(urllib.request.Request(url, data=body, headers=headers), timeout=timeout) as r:
122
+ return json.load(r)["choices"][0]["message"]["content"].strip()
123
+
124
+ ask.url, ask.model = url, model # handy when debugging which endpoint is used
125
+ return ask
126
+
127
+
128
+ def _run(make_coro):
129
+ """Run a coroutine from sync code, also when a loop is already running (Jupyter)."""
130
+ try:
131
+ asyncio.get_running_loop()
132
+ except RuntimeError:
133
+ return asyncio.run(make_coro())
134
+ box = {}
135
+
136
+ def worker():
137
+ try:
138
+ box["value"] = asyncio.run(make_coro())
139
+ except BaseException as exc: # re-raised in the caller's thread
140
+ box["error"] = exc
141
+
142
+ thread = threading.Thread(target=worker)
143
+ thread.start()
144
+ thread.join()
145
+ if "error" in box:
146
+ raise box["error"]
147
+ return box["value"]
148
+
149
+
150
+ def hermes_llm(timeout: float = 120.0, **client_kwargs) -> Callable[[str], str]:
151
+ """Return ``llm(prompt) -> str`` that asks the Hermes Agent through ``hermes-acp-sdk``.
152
+
153
+ Needs ``pip install hermes-acp-sdk "hermes-agent[acp]"`` (Hermes requires Python 3.11 to 3.13) and a
154
+ provider already configured for Hermes. Hermes has its own file and terminal tools that the SDK cannot
155
+ sandbox, so by default it is started in an **empty temporary folder**. Extra keyword arguments
156
+ (``profile=...``, ``clone_provider=True``, ...) go to ``HermesClient``.
157
+ """
158
+ try:
159
+ import hermes_acp_sdk # noqa: F401
160
+ except ImportError as exc:
161
+ raise RuntimeError(
162
+ 'hermes_llm needs the Hermes SDK: pip install hermes-acp-sdk "hermes-agent[acp]" '
163
+ "(Hermes requires Python 3.11 to 3.13). Or set DEEPSEEK_API_KEY / OPENAI_API_KEY to use an online model."
164
+ ) from exc
165
+
166
+ async def ask_async(prompt: str) -> str:
167
+ from hermes_acp_sdk import AgentText, HermesClient
168
+
169
+ with tempfile.TemporaryDirectory(prefix="difffinder-") as empty:
170
+ kwargs = dict(client_kwargs)
171
+ kwargs.setdefault("cwd", empty)
172
+ parts = []
173
+
174
+ async def turn():
175
+ async with HermesClient(**kwargs) as hermes:
176
+ async with hermes.session() as session:
177
+ async for event in session.prompt(SYSTEM + "\n\n" + prompt):
178
+ if isinstance(event, AgentText):
179
+ parts.append(event.text)
180
+
181
+ await asyncio.wait_for(turn(), timeout)
182
+ return "".join(parts).strip()
183
+
184
+ return lambda prompt: _run(lambda: ask_async(prompt))
185
+
186
+
187
+ def _resolve(llm):
188
+ if callable(llm):
189
+ return llm
190
+ if llm == "hermes":
191
+ return hermes_llm()
192
+ if llm == "openai":
193
+ return openai_compatible()
194
+ if llm in (None, "auto"):
195
+ configured = any(os.environ.get(v) for v in _KEY_VARS) or os.environ.get("DIFFFINDER_BASE_URL")
196
+ return openai_compatible() if configured else hermes_llm()
197
+ raise ValueError("llm must be a function, 'hermes', 'openai' or 'auto'")
198
+
199
+
200
+ def explain(cmp, llm=None, share_reference: bool = True) -> str:
201
+ """Return a short model hint for a :class:`~difffinder.TraceComparison`.
202
+
203
+ ``llm`` is a function ``prompt -> str``, or ``"hermes"`` / ``"openai"`` / ``"auto"`` (the default: an
204
+ online model if an API key is set, otherwise Hermes). With ``share_reference=False`` the reference
205
+ function's source is not sent, which hides the solution but gives the model less to work with.
206
+ """
207
+ if cmp.same:
208
+ return "Nothing to explain: the two functions behave the same."
209
+ return _resolve(llm)(build_prompt(cmp, share_reference))
@@ -0,0 +1,87 @@
1
+ """Compare two functions on the same input."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass, field
6
+ from typing import Any, Callable, List, Optional
7
+
8
+ from .outcome import Outcome, run, short
9
+
10
+
11
+ @dataclass
12
+ class Difference:
13
+ """One way the two functions behave differently."""
14
+
15
+ what: str # "result" or "argument 'nums' afterwards"
16
+ yours: str
17
+ reference: str
18
+
19
+
20
+ @dataclass(repr=False)
21
+ class Comparison:
22
+ """The result of :func:`compare`."""
23
+
24
+ candidate_name: str
25
+ reference_name: str
26
+ candidate: Outcome
27
+ reference: Outcome
28
+ differences: List[Difference] = field(default_factory=list)
29
+
30
+ @property
31
+ def same(self) -> bool:
32
+ """True if the two functions did the same thing on this input."""
33
+ return not self.differences
34
+
35
+ def __str__(self) -> str:
36
+ call = ", ".join("%s=%s" % (name, short(value, 40)) for name, value in self.candidate.inputs.items())
37
+ names = "%s and %s" % (self.candidate_name, self.reference_name)
38
+ if self.same:
39
+ return "SAME: %s behave the same on (%s)." % (names, call)
40
+ lines = ["DIFFERENT: %s behave differently on (%s)." % (names, call)]
41
+ for d in self.differences:
42
+ lines.append(" %s: yours %s, the reference %s." % (d.what, d.yours, d.reference))
43
+ return "\n".join(lines)
44
+
45
+ __repr__ = __str__ # so a Comparison displays as text when it is the last line of a notebook cell
46
+
47
+
48
+ def _same_value(a: Any, b: Any) -> bool:
49
+ try:
50
+ return bool(a == b)
51
+ except Exception: # for example array-like objects whose == does not give a single True/False
52
+ return repr(a) == repr(b)
53
+
54
+
55
+ def _differences(mine: Outcome, ref: Outcome) -> List[Difference]:
56
+ found: List[Difference] = []
57
+ if mine.raised or ref.raised:
58
+ if mine.error_type != ref.error_type: # same exception type counts as the same
59
+ found.append(Difference("result", mine.describe(), ref.describe()))
60
+ elif not _same_value(mine.returned, ref.returned):
61
+ found.append(Difference("result", mine.describe(), ref.describe()))
62
+ # Arguments are matched by position, so the two functions may name their parameters differently.
63
+ for (name, a), (_, b) in zip(mine.arguments.items(), ref.arguments.items()):
64
+ if not _same_value(a, b):
65
+ found.append(Difference("argument '%s' afterwards" % name, short(repr(a)), short(repr(b))))
66
+ return found
67
+
68
+
69
+ def _name(func: Callable[..., Any]) -> str:
70
+ return getattr(func, "__name__", None) or repr(func)
71
+
72
+
73
+ def compare(
74
+ candidate: Callable[..., Any],
75
+ reference: Callable[..., Any],
76
+ args: tuple = (),
77
+ kwargs: Optional[dict] = None,
78
+ ) -> Comparison:
79
+ """Run both functions on the same input and report whether they behave the same.
80
+
81
+ ``candidate`` is usually your own (possibly buggy) function and ``reference`` a known-good one.
82
+ "Behave the same" means: the same return value, the same kind of exception, and the same changes
83
+ to the arguments. How the functions are written inside does not matter.
84
+ """
85
+ mine = run(candidate, args, kwargs)
86
+ ref = run(reference, args, kwargs)
87
+ return Comparison(_name(candidate), _name(reference), mine, ref, _differences(mine, ref))
difffinder/demo.py ADDED
@@ -0,0 +1,27 @@
1
+ from difffinder import compare_traces
2
+
3
+ def count_to(n):
4
+ total = 0
5
+ for i in range(n):
6
+ total += i
7
+ return total
8
+
9
+ def count_to_ref(n):
10
+ total = 0
11
+ for i in range(n + 1):
12
+ total += i
13
+ return total
14
+
15
+ if __name__ == '__main__':
16
+ import argparse
17
+
18
+ parser = argparse.ArgumentParser(description='Run the DiffFinder demo or sorting visualizer.')
19
+ mode = parser.add_mutually_exclusive_group()
20
+ mode.add_argument('--gui', action='store_true', help='Open the sorting visualizer (default)')
21
+ mode.add_argument('--console', action='store_true', help='Run the numeric console demo')
22
+ options = parser.parse_args()
23
+ if not options.console:
24
+ from difffinder.sorting_visualizer import launch_app
25
+ launch_app()
26
+ else:
27
+ print(compare_traces(count_to, count_to_ref, args=(4,)))
difffinder/diff.py ADDED
@@ -0,0 +1,214 @@
1
+ """Compare two functions: where do they differ, on which inputs, and why."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import copy
6
+ from dataclasses import dataclass, field
7
+ from typing import Any, Callable, Dict, Iterable, List, Optional
8
+
9
+ from .probe import outcome
10
+ from .tracer import RAISE, Event, Trace, record
11
+
12
+
13
+ def _special(name: str) -> bool:
14
+ """<return>, <raise> and <argN> are outcome pseudo-variables, not real variables."""
15
+ return name.startswith("<")
16
+
17
+
18
+ @dataclass
19
+ class Divergence:
20
+ """The first place two runs disagree, step by step."""
21
+
22
+ var: str # the variable (or <return> / <raise> / <argN>) that disagrees
23
+ position: int # 0-based: it is that variable's n-th new value
24
+ candidate: Optional[Event] # None means the candidate never made this change
25
+ reference: Optional[Event]
26
+ agreed: List[str] # the last few values both runs agreed on
27
+
28
+
29
+ @dataclass(repr=False)
30
+ class TraceComparison:
31
+ candidate: Trace
32
+ reference: Trace
33
+ divergence: Optional[Divergence]
34
+ compared: List[str] # variable names that were compared step by step
35
+ args: tuple = ()
36
+ kwargs: dict = field(default_factory=dict)
37
+ skipped: List[str] = field(default_factory=list) # same name, but probably a different meaning
38
+ hints: List[str] = field(default_factory=list) # plain-English clues, generated offline
39
+ candidate_fn: Any = None
40
+ reference_fn: Any = None
41
+ ai_hint: Optional[str] = None # filled in when compare_traces(..., ai=...) succeeds
42
+ ai_error: Optional[str] = None # why the AI hint is missing, if it failed
43
+ outcomes_differ: bool = False # on the input you gave: different result, exception or side effect
44
+ strict: bool = False # watch=/rename= given: step-by-step differences count too
45
+ probe: Any = None # difffinder.probe.Probe: what testing many other inputs showed
46
+
47
+ @property
48
+ def same(self) -> bool:
49
+ if self.probe is not None and self.probe.failing:
50
+ return False
51
+ return self.divergence is None if self.strict else not self.outcomes_differ
52
+
53
+ def explain(self, llm: Any = None, share_reference: bool = True) -> str:
54
+ """Ask an LLM for a short hint. See :mod:`difffinder.ai`."""
55
+ from .ai import explain
56
+
57
+ return explain(self, llm, share_reference)
58
+
59
+ def __str__(self) -> str:
60
+ from .render import to_text
61
+
62
+ return to_text(self)
63
+
64
+ __repr__ = __str__
65
+
66
+ def _repr_html_(self) -> str:
67
+ from .render import to_html
68
+
69
+ return to_html(self)
70
+
71
+
72
+ def find_divergence(
73
+ candidate: Trace,
74
+ reference: Trace,
75
+ *,
76
+ watch: Optional[Iterable[str]] = None,
77
+ rename: Optional[Dict[str, str]] = None,
78
+ ):
79
+ """Return ``(divergence_or_None, compared_names, skipped_names)`` for two traces.
80
+
81
+ Each shared variable's history is compared separately, so unrelated assignments in a different order
82
+ are not a difference. The outcome (return value, exception, final state of every argument) is always
83
+ compared. In automatic mode (no ``watch``/``rename``) a variable the two runs barely agree on is
84
+ *skipped*: different algorithms often reuse a name like ``i`` for different things.
85
+ """
86
+ rename = rename or {}
87
+
88
+ def histories(trace: Trace, mapping: Dict[str, str]) -> Dict[str, List[Event]]:
89
+ out: Dict[str, List[Event]] = {}
90
+ for e in trace.events:
91
+ out.setdefault(mapping.get(e.var, e.var), []).append(e)
92
+ return out
93
+
94
+ ha, hb = histories(candidate, rename), histories(reference, {})
95
+ auto = watch is None and not rename
96
+ specials = {n for n in set(ha) | set(hb) if _special(n)}
97
+ names = (set(watch) if watch is not None else set(ha) & set(hb)) | specials
98
+
99
+ found = []
100
+ for name in names:
101
+ a, b = ha.get(name, []), hb.get(name, [])
102
+ for i in range(max(len(a), len(b))):
103
+ ea = a[i] if i < len(a) else None
104
+ eb = b[i] if i < len(b) else None
105
+ if ea is None or eb is None or ea.value != eb.value:
106
+ when = ea.step if ea is not None else candidate.steps
107
+ if _special(name):
108
+ rank = 1 if name == RAISE else 2
109
+ elif ea is None and RAISE in ha:
110
+ rank = 1.5 # "never got here" is a consequence of the crash, so the crash goes first
111
+ else:
112
+ rank = 0 # real variables beat outcomes
113
+ agreed = [e.value for e in a[max(0, i - 3) : i]]
114
+ found.append(((when, rank, name), Divergence(name, i, ea, eb, agreed)))
115
+ break
116
+ found.sort(key=lambda item: item[0])
117
+
118
+ # A name is only trusted to mean the same thing in both functions if it is a parameter of both,
119
+ # or the two runs agreed on at least two of its values before diverging.
120
+ params = set(candidate.arg_names.values()) & set(reference.arg_names.values())
121
+ chosen, skipped = None, []
122
+ for _, div in found:
123
+ aligned = len(div.agreed) >= 2 or (div.var in params and div.position >= 1)
124
+ if auto and not _special(div.var) and not aligned:
125
+ skipped.append(div.var)
126
+ continue
127
+ chosen = div
128
+ break
129
+ compared = sorted(n for n in names if not _special(n) and (n in ha or n in hb))
130
+ return chosen, compared, skipped
131
+
132
+
133
+ def compare_traces(
134
+ candidate: Callable[..., Any],
135
+ reference: Callable[..., Any],
136
+ args: tuple = (),
137
+ kwargs: Optional[dict] = None,
138
+ *,
139
+ watch: Optional[Iterable[str]] = None,
140
+ rename: Optional[Dict[str, str]] = None,
141
+ max_steps: int = 100_000,
142
+ probe: bool = True,
143
+ ai: Any = None,
144
+ ) -> TraceComparison:
145
+ """Run both functions on the same input and report how they differ.
146
+
147
+ ``candidate`` is usually the buggy or student version, ``reference`` the known-good one. Each run gets
148
+ its own deep copy of ``args``/``kwargs``, so in-place changes cannot leak.
149
+
150
+ The verdict is based on what each function visibly does (return value, exception, changes to its
151
+ arguments), so it works however differently the two are written. With ``probe=True`` (the default) it
152
+ also tries many variations of your input, shrinks a failing one to the smallest case, and describes
153
+ when they disagree. This calls both functions many times, so use ``probe=False`` for functions with
154
+ side effects or that are slow.
155
+
156
+ ``watch`` limits the step-by-step comparison to the named variables (``watch=[]`` compares only the
157
+ outcome). ``rename`` maps variable names in ``candidate`` to the names in ``reference``. Giving either
158
+ means you vouch that the names line up, and step-by-step differences then count as differences.
159
+
160
+ ``ai=True`` adds a short model-written hint (needs an API key, see :mod:`difffinder.ai`); ``ai`` may
161
+ also be ``"hermes"``, ``"openai"`` or a function ``prompt -> str``. If the model cannot be reached the
162
+ comparison still works and the reason is shown instead.
163
+ """
164
+ kwargs = kwargs or {}
165
+ ta = record(candidate, copy.deepcopy(args), copy.deepcopy(kwargs), max_steps=max_steps)
166
+ tb = record(reference, copy.deepcopy(args), copy.deepcopy(kwargs), max_steps=max_steps)
167
+ divergence, compared, skipped = find_divergence(ta, tb, watch=watch, rename=rename)
168
+ strict = watch is not None or bool(rename)
169
+ differ = outcome(ta) != outcome(tb)
170
+ if not strict and not differ:
171
+ divergence = None # internal steps may differ between two correct solutions
172
+ result = TraceComparison(
173
+ ta, tb, divergence, compared, tuple(args), dict(kwargs), skipped, [], candidate, reference,
174
+ outcomes_differ=differ, strict=strict,
175
+ )
176
+ if probe and args:
177
+ from .probe import run_probe
178
+
179
+ names = [ta.arg_names.get("<arg%d>" % i, "arg%d" % i) for i in range(len(args))]
180
+ try:
181
+ result.probe = run_probe(candidate, reference, tuple(args), kwargs, names, differ, ta, tb)
182
+ except Exception: # probing is a bonus; never let it break the comparison
183
+ result.probe = None
184
+ if not result.same:
185
+ from .hints import generate
186
+
187
+ result.hints = generate(result)
188
+ if ai:
189
+ from .ai import explain
190
+
191
+ try:
192
+ result.ai_hint = explain(result, "auto" if ai is True else ai)
193
+ except Exception as exc: # a missing key or offline model must never break the comparison
194
+ result.ai_error = "%s: %s" % (type(exc).__name__, exc)
195
+ return result
196
+
197
+
198
+ def find_difference(
199
+ candidate: Callable[..., Any],
200
+ reference: Callable[..., Any],
201
+ cases: Iterable[Any],
202
+ **options: Any,
203
+ ) -> Optional[TraceComparison]:
204
+ """Try each input in ``cases``; return the first :class:`TraceComparison` that differs, or None.
205
+
206
+ A case that is a tuple is used as the positional arguments; anything else is wrapped.
207
+ """
208
+ options.setdefault("probe", False) # this function is itself the search
209
+ for case in cases:
210
+ args = case if isinstance(case, tuple) else (case,)
211
+ result = compare_traces(candidate, reference, args, **options)
212
+ if not result.same:
213
+ return result
214
+ return None
difffinder/hints.py ADDED
@@ -0,0 +1,118 @@
1
+ """Plain-English hints from a Comparison. Deterministic: no AI, no network."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import ast
6
+ import inspect
7
+ import linecache
8
+ import re
9
+ import textwrap
10
+ from typing import List
11
+
12
+ from .tracer import RAISE, RETURN
13
+
14
+ _MAX_HINTS = 5
15
+
16
+
17
+ def _exception_hint(text: str) -> str:
18
+ m = re.match(r"ValueError: invalid literal for int\(\) with base 10: '(.*)'", text)
19
+ if m:
20
+ bad = m.group(1)
21
+ extra = " Negative numbers contain a '-' when you loop over str(number); take abs() first." if bad == "-" else ""
22
+ return "int() was given %r, which is not a string of digits.%s" % (bad, extra)
23
+ m = re.match(r"AttributeError: 'NoneType' object has no attribute '(\w+)'", text)
24
+ if m:
25
+ return (
26
+ "A variable is None when you call .%s on it. A common cause is assigning the result of a "
27
+ "method that works in place (list.remove, append, sort, reverse), which returns None." % m.group(1)
28
+ )
29
+ if text.startswith("IndexError"):
30
+ return "An index went past the end of a list. Check your loop bounds, and whether you remove items while looping."
31
+ if text.startswith("KeyError"):
32
+ return "A dictionary key was missing. Check the keys you add, or use .get()."
33
+ if text.startswith("ZeroDivisionError"):
34
+ return "Something was divided by zero. Check the case where the divisor can be 0."
35
+ return ""
36
+
37
+
38
+ def generate(cmp) -> List[str]:
39
+ """Return up to a few short hints about the candidate's behaviour."""
40
+ out: List[str] = []
41
+ d = cmp.divergence
42
+ traces = [cmp.candidate]
43
+ probe = getattr(cmp, "probe", None)
44
+ if probe is not None and probe.minimal_candidate is not None:
45
+ traces.append(probe.minimal_candidate) # the smallest failing run is the clearest evidence
46
+
47
+ for trace in traces:
48
+ for ev in trace.history(RAISE):
49
+ hint = _exception_hint(ev.value)
50
+ if hint:
51
+ out.append(hint)
52
+ # A list that suddenly turns into None: the result of an in-place method was assigned back.
53
+ for var in sorted(trace.variables):
54
+ hist = trace.history(var)
55
+ for prev, cur in zip(hist, hist[1:]):
56
+ if cur.value == "None" and prev.value.startswith(("[", "{")):
57
+ src = linecache.getline(cur.filename, cur.lineno).strip()
58
+ out.append(
59
+ "'%s' was a collection but became None at line %d%s. Methods like remove/append/sort "
60
+ "change the object in place and return None, so do not assign their result back."
61
+ % (var, cur.lineno, (" (`%s`)" % src) if src else "")
62
+ )
63
+ break
64
+
65
+ name_of = cmp.candidate.arg_names
66
+ if d is not None and d.var.startswith("<arg"):
67
+ out.append(
68
+ "The input '%s' ends up different from the reference. If the task says to modify it in place, "
69
+ "change the object itself; rebinding the name to a new object does not affect the caller."
70
+ % name_of.get(d.var, "an argument")
71
+ )
72
+ if d is not None and d.var == RETURN and d.candidate is not None and d.candidate.value == "None":
73
+ out.append("Your function returned None. Does every path end with `return <value>`?")
74
+
75
+ out.extend(_source_hints(cmp))
76
+ seen, unique = set(), []
77
+ for h in out:
78
+ if h and h not in seen:
79
+ seen.add(h)
80
+ unique.append(h)
81
+ return unique[:_MAX_HINTS]
82
+
83
+
84
+ def _source_hints(cmp) -> List[str]:
85
+ """Hints that come from reading the candidate's source code."""
86
+ try:
87
+ lines, first = inspect.getsourcelines(cmp.candidate_fn)
88
+ tree = ast.parse(textwrap.dedent("".join(lines)))
89
+ fn = next(n for n in ast.walk(tree) if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)))
90
+ except (OSError, TypeError, SyntaxError, StopIteration):
91
+ return []
92
+ params = [a.arg for a in fn.args.posonlyargs + fn.args.args + fn.args.kwonlyargs]
93
+ out = []
94
+ for node in ast.walk(fn):
95
+ line = first + getattr(node, "lineno", 1) - 1
96
+ # for ... else without a break: the else block always runs
97
+ if isinstance(node, (ast.For, ast.While)) and node.orelse:
98
+ if not any(isinstance(n, ast.Break) for b in node.body for n in ast.walk(b)):
99
+ out.append(
100
+ "Line %d: a loop has an `else:` block but no `break`. A loop's `else` runs whenever the "
101
+ "loop finishes without break, so here it always runs; it is not the 'otherwise' of an `if`." % line
102
+ )
103
+ # alias = parameter: does not copy a list/dict/set
104
+ if (
105
+ isinstance(node, ast.Assign)
106
+ and len(node.targets) == 1
107
+ and isinstance(node.targets[0], ast.Name)
108
+ and isinstance(node.value, ast.Name)
109
+ and node.value.id in params
110
+ ):
111
+ idx = params.index(node.value.id)
112
+ if idx < len(cmp.args) and isinstance(cmp.args[idx], (list, dict, set)):
113
+ out.append(
114
+ "Line %d: `%s = %s` does not copy. Both names point to the same %s, so changing one changes "
115
+ "the other. Use a copy (for example `list(%s)`) if you need the original."
116
+ % (line, node.targets[0].id, node.value.id, type(cmp.args[idx]).__name__, node.value.id)
117
+ )
118
+ return out