tracecomparekit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- difffinder/__init__.py +21 -0
- difffinder/__main__.py +3 -0
- difffinder/ai.py +209 -0
- difffinder/comparison.py +87 -0
- difffinder/demo.py +27 -0
- difffinder/diff.py +214 -0
- difffinder/hints.py +118 -0
- difffinder/outcome.py +63 -0
- difffinder/probe.py +301 -0
- difffinder/render.py +161 -0
- difffinder/selfcheck.py +92 -0
- difffinder/sorting_visualizer.py +717 -0
- difffinder/tracer.py +192 -0
- tracecomparekit-0.1.0.dist-info/METADATA +189 -0
- tracecomparekit-0.1.0.dist-info/RECORD +17 -0
- tracecomparekit-0.1.0.dist-info/WHEEL +4 -0
- tracecomparekit-0.1.0.dist-info/licenses/LICENSE +23 -0
difffinder/__init__.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""DiffFinder: check a Python function against a reference solution and see how they differ.
|
|
2
|
+
|
|
3
|
+
>>> from difffinder import compare
|
|
4
|
+
>>> compare(my_function, reference_function, args=(4,))
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from .comparison import Comparison, Difference, compare
|
|
8
|
+
from .ai import explain, hermes_llm, openai_compatible
|
|
9
|
+
from .diff import Divergence, TraceComparison, compare_traces, find_difference, find_divergence
|
|
10
|
+
from .outcome import Outcome, run
|
|
11
|
+
from .selfcheck import selfcheck
|
|
12
|
+
from .tracer import RAISE, RETURN, Event, StepLimitExceeded, Trace, record
|
|
13
|
+
|
|
14
|
+
__version__ = "0.1.0"
|
|
15
|
+
|
|
16
|
+
__all__ = [
|
|
17
|
+
"compare", "run", "Comparison", "Difference", "Outcome",
|
|
18
|
+
"compare_traces", "TraceComparison", "Divergence", "find_difference", "find_divergence",
|
|
19
|
+
"record", "Trace", "Event", "StepLimitExceeded", "RETURN", "RAISE",
|
|
20
|
+
"explain", "openai_compatible", "hermes_llm", "selfcheck", "__version__",
|
|
21
|
+
]
|
difffinder/__main__.py
ADDED
difffinder/ai.py
ADDED
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
"""Optional: ask a language model for a short hint. Off unless you ask for it.
|
|
2
|
+
|
|
3
|
+
Quickest way: set an API key, then pass ``ai=True``::
|
|
4
|
+
|
|
5
|
+
import os, getpass
|
|
6
|
+
os.environ["DEEPSEEK_API_KEY"] = getpass.getpass() # or OPENAI_API_KEY
|
|
7
|
+
compare_traces(mine, reference, args=(...), ai=True)
|
|
8
|
+
|
|
9
|
+
Ready-made models: :func:`openai_compatible` (any OpenAI-style endpoint, standard library only) and
|
|
10
|
+
:func:`hermes_llm` (the Hermes Agent via ``hermes-acp-sdk``). Or pass any function ``llm(prompt) -> str``.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import asyncio
|
|
16
|
+
import inspect
|
|
17
|
+
import json
|
|
18
|
+
import os
|
|
19
|
+
import tempfile
|
|
20
|
+
import threading
|
|
21
|
+
import urllib.request
|
|
22
|
+
from typing import Callable, Optional
|
|
23
|
+
|
|
24
|
+
SYSTEM = (
|
|
25
|
+
"You are a friendly programming tutor talking to one first-year student. You are given the results of "
|
|
26
|
+
"automatically testing their function against a correct reference function: the inputs where the two "
|
|
27
|
+
"differ, the smallest failing input, a pattern for when it fails, and what each function did. "
|
|
28
|
+
"Reply in at most 5 short sentences, in plain spoken English, not as a report, and without repeating "
|
|
29
|
+
"numbers the student can already see. Say what their code most likely gets wrong, pointing at one "
|
|
30
|
+
"specific line or variable, then ask ONE guiding question. "
|
|
31
|
+
"Do not write corrected code and do not reveal the reference solution (do not quote or describe it). "
|
|
32
|
+
"No praise, no apologies."
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
_KEY_VARS = ("DIFFFINDER_API_KEY", "SOCRATIC_LLM_API_KEY", "DEEPSEEK_API_KEY", "OPENAI_API_KEY")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _source(fn) -> str:
|
|
39
|
+
try:
|
|
40
|
+
return inspect.getsource(fn)
|
|
41
|
+
except (OSError, TypeError):
|
|
42
|
+
return "(source not available)"
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _short(text: str, limit: int = 80) -> str:
|
|
46
|
+
return text if len(text) <= limit else text[: limit - 3] + "..."
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def build_prompt(cmp, share_reference: bool = True) -> str:
|
|
50
|
+
"""The text sent to the model: the code, the evidence, and what the student's run actually did."""
|
|
51
|
+
from .render import to_text
|
|
52
|
+
|
|
53
|
+
d = cmp.divergence
|
|
54
|
+
parts = ["INPUT: %r" % (cmp.args,), "STUDENT'S FUNCTION:\n" + _source(cmp.candidate_fn)]
|
|
55
|
+
if share_reference:
|
|
56
|
+
parts.append("REFERENCE FUNCTION (for your eyes only, never quote or describe it):\n" + _source(cmp.reference_fn))
|
|
57
|
+
parts.append("AUTOMATIC COMPARISON:\n" + to_text(cmp, include_ai=False))
|
|
58
|
+
if d is not None:
|
|
59
|
+
def values(trace):
|
|
60
|
+
vals = [_short(e.value, 40) for e in trace.history(d.var)][:15]
|
|
61
|
+
return ", ".join(vals) if vals else "(never set)"
|
|
62
|
+
|
|
63
|
+
parts.append(
|
|
64
|
+
"VALUES OF '%s' IN ORDER:\n student : %s\n reference: %s" % (d.var, values(cmp.candidate), values(cmp.reference))
|
|
65
|
+
)
|
|
66
|
+
probe = cmp.probe
|
|
67
|
+
focus = probe.minimal_candidate if probe is not None and probe.minimal_candidate is not None else cmp.candidate
|
|
68
|
+
recent = [e for e in focus.events if not e.var.startswith("<")][-12:]
|
|
69
|
+
if recent:
|
|
70
|
+
parts.append(
|
|
71
|
+
"LAST CHANGES IN THE STUDENT'S RUN%s:\n" % (" (on the smallest failing input)" if focus is not cmp.candidate else "")
|
|
72
|
+
+ "\n".join(" line %d: %s = %s" % (e.lineno, e.var, _short(e.value, 60)) for e in recent)
|
|
73
|
+
)
|
|
74
|
+
return "\n\n".join(parts)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def openai_compatible(
|
|
78
|
+
base_url: Optional[str] = None,
|
|
79
|
+
api_key: Optional[str] = None,
|
|
80
|
+
model: Optional[str] = None,
|
|
81
|
+
timeout: float = 60.0,
|
|
82
|
+
) -> Callable[[str], str]:
|
|
83
|
+
"""Return ``llm(prompt) -> str`` for an OpenAI-style chat endpoint.
|
|
84
|
+
|
|
85
|
+
Everything can come from the environment. Key: DIFFFINDER_API_KEY, SOCRATIC_LLM_API_KEY, DEEPSEEK_API_KEY
|
|
86
|
+
or OPENAI_API_KEY (first one found). With a DeepSeek key, the DeepSeek endpoint and ``deepseek-chat`` are
|
|
87
|
+
used by default. Override with DIFFFINDER_BASE_URL / DIFFFINDER_MODEL (``SOCRATIC_LLM_*`` and
|
|
88
|
+
``OPENAI_BASE_URL`` / ``OPENAI_MODEL`` are read too).
|
|
89
|
+
"""
|
|
90
|
+
env = os.environ.get
|
|
91
|
+
own_key = next((env(v) for v in _KEY_VARS[:3] if env(v)), None)
|
|
92
|
+
openai_key = env("OPENAI_API_KEY")
|
|
93
|
+
key = api_key if api_key is not None else (own_key or openai_key or "")
|
|
94
|
+
use_openai = api_key is None and not own_key and bool(openai_key)
|
|
95
|
+
base = (
|
|
96
|
+
base_url
|
|
97
|
+
or env("DIFFFINDER_BASE_URL")
|
|
98
|
+
or env("SOCRATIC_LLM_BASE_URL")
|
|
99
|
+
or env("OPENAI_BASE_URL")
|
|
100
|
+
or ("https://api.openai.com/v1" if use_openai else "https://api.deepseek.com")
|
|
101
|
+
)
|
|
102
|
+
model = model or env("DIFFFINDER_MODEL") or env("SOCRATIC_LLM_MODEL") or env("OPENAI_MODEL") or (None if use_openai else "deepseek-chat")
|
|
103
|
+
if not key and not (base_url or env("DIFFFINDER_BASE_URL")):
|
|
104
|
+
raise RuntimeError(
|
|
105
|
+
"No API key found. Set DEEPSEEK_API_KEY (or OPENAI_API_KEY) first, for example:\n"
|
|
106
|
+
" import os, getpass; os.environ['DEEPSEEK_API_KEY'] = getpass.getpass()"
|
|
107
|
+
)
|
|
108
|
+
if not model:
|
|
109
|
+
raise RuntimeError("Set DIFFFINDER_MODEL to the model name you want to use (for example your OpenAI model).")
|
|
110
|
+
url = base.rstrip("/") + "/chat/completions"
|
|
111
|
+
|
|
112
|
+
def ask(prompt: str) -> str:
|
|
113
|
+
body = json.dumps(
|
|
114
|
+
{
|
|
115
|
+
"model": model,
|
|
116
|
+
"temperature": 0.2,
|
|
117
|
+
"messages": [{"role": "system", "content": SYSTEM}, {"role": "user", "content": prompt}],
|
|
118
|
+
}
|
|
119
|
+
).encode()
|
|
120
|
+
headers = {"Content-Type": "application/json", "Authorization": "Bearer " + key}
|
|
121
|
+
with urllib.request.urlopen(urllib.request.Request(url, data=body, headers=headers), timeout=timeout) as r:
|
|
122
|
+
return json.load(r)["choices"][0]["message"]["content"].strip()
|
|
123
|
+
|
|
124
|
+
ask.url, ask.model = url, model # handy when debugging which endpoint is used
|
|
125
|
+
return ask
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _run(make_coro):
|
|
129
|
+
"""Run a coroutine from sync code, also when a loop is already running (Jupyter)."""
|
|
130
|
+
try:
|
|
131
|
+
asyncio.get_running_loop()
|
|
132
|
+
except RuntimeError:
|
|
133
|
+
return asyncio.run(make_coro())
|
|
134
|
+
box = {}
|
|
135
|
+
|
|
136
|
+
def worker():
|
|
137
|
+
try:
|
|
138
|
+
box["value"] = asyncio.run(make_coro())
|
|
139
|
+
except BaseException as exc: # re-raised in the caller's thread
|
|
140
|
+
box["error"] = exc
|
|
141
|
+
|
|
142
|
+
thread = threading.Thread(target=worker)
|
|
143
|
+
thread.start()
|
|
144
|
+
thread.join()
|
|
145
|
+
if "error" in box:
|
|
146
|
+
raise box["error"]
|
|
147
|
+
return box["value"]
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def hermes_llm(timeout: float = 120.0, **client_kwargs) -> Callable[[str], str]:
|
|
151
|
+
"""Return ``llm(prompt) -> str`` that asks the Hermes Agent through ``hermes-acp-sdk``.
|
|
152
|
+
|
|
153
|
+
Needs ``pip install hermes-acp-sdk "hermes-agent[acp]"`` (Hermes requires Python 3.11 to 3.13) and a
|
|
154
|
+
provider already configured for Hermes. Hermes has its own file and terminal tools that the SDK cannot
|
|
155
|
+
sandbox, so by default it is started in an **empty temporary folder**. Extra keyword arguments
|
|
156
|
+
(``profile=...``, ``clone_provider=True``, ...) go to ``HermesClient``.
|
|
157
|
+
"""
|
|
158
|
+
try:
|
|
159
|
+
import hermes_acp_sdk # noqa: F401
|
|
160
|
+
except ImportError as exc:
|
|
161
|
+
raise RuntimeError(
|
|
162
|
+
'hermes_llm needs the Hermes SDK: pip install hermes-acp-sdk "hermes-agent[acp]" '
|
|
163
|
+
"(Hermes requires Python 3.11 to 3.13). Or set DEEPSEEK_API_KEY / OPENAI_API_KEY to use an online model."
|
|
164
|
+
) from exc
|
|
165
|
+
|
|
166
|
+
async def ask_async(prompt: str) -> str:
|
|
167
|
+
from hermes_acp_sdk import AgentText, HermesClient
|
|
168
|
+
|
|
169
|
+
with tempfile.TemporaryDirectory(prefix="difffinder-") as empty:
|
|
170
|
+
kwargs = dict(client_kwargs)
|
|
171
|
+
kwargs.setdefault("cwd", empty)
|
|
172
|
+
parts = []
|
|
173
|
+
|
|
174
|
+
async def turn():
|
|
175
|
+
async with HermesClient(**kwargs) as hermes:
|
|
176
|
+
async with hermes.session() as session:
|
|
177
|
+
async for event in session.prompt(SYSTEM + "\n\n" + prompt):
|
|
178
|
+
if isinstance(event, AgentText):
|
|
179
|
+
parts.append(event.text)
|
|
180
|
+
|
|
181
|
+
await asyncio.wait_for(turn(), timeout)
|
|
182
|
+
return "".join(parts).strip()
|
|
183
|
+
|
|
184
|
+
return lambda prompt: _run(lambda: ask_async(prompt))
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _resolve(llm):
|
|
188
|
+
if callable(llm):
|
|
189
|
+
return llm
|
|
190
|
+
if llm == "hermes":
|
|
191
|
+
return hermes_llm()
|
|
192
|
+
if llm == "openai":
|
|
193
|
+
return openai_compatible()
|
|
194
|
+
if llm in (None, "auto"):
|
|
195
|
+
configured = any(os.environ.get(v) for v in _KEY_VARS) or os.environ.get("DIFFFINDER_BASE_URL")
|
|
196
|
+
return openai_compatible() if configured else hermes_llm()
|
|
197
|
+
raise ValueError("llm must be a function, 'hermes', 'openai' or 'auto'")
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def explain(cmp, llm=None, share_reference: bool = True) -> str:
|
|
201
|
+
"""Return a short model hint for a :class:`~difffinder.TraceComparison`.
|
|
202
|
+
|
|
203
|
+
``llm`` is a function ``prompt -> str``, or ``"hermes"`` / ``"openai"`` / ``"auto"`` (the default: an
|
|
204
|
+
online model if an API key is set, otherwise Hermes). With ``share_reference=False`` the reference
|
|
205
|
+
function's source is not sent, which hides the solution but gives the model less to work with.
|
|
206
|
+
"""
|
|
207
|
+
if cmp.same:
|
|
208
|
+
return "Nothing to explain: the two functions behave the same."
|
|
209
|
+
return _resolve(llm)(build_prompt(cmp, share_reference))
|
difffinder/comparison.py
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""Compare two functions on the same input."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from typing import Any, Callable, List, Optional
|
|
7
|
+
|
|
8
|
+
from .outcome import Outcome, run, short
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dataclass
|
|
12
|
+
class Difference:
|
|
13
|
+
"""One way the two functions behave differently."""
|
|
14
|
+
|
|
15
|
+
what: str # "result" or "argument 'nums' afterwards"
|
|
16
|
+
yours: str
|
|
17
|
+
reference: str
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass(repr=False)
|
|
21
|
+
class Comparison:
|
|
22
|
+
"""The result of :func:`compare`."""
|
|
23
|
+
|
|
24
|
+
candidate_name: str
|
|
25
|
+
reference_name: str
|
|
26
|
+
candidate: Outcome
|
|
27
|
+
reference: Outcome
|
|
28
|
+
differences: List[Difference] = field(default_factory=list)
|
|
29
|
+
|
|
30
|
+
@property
|
|
31
|
+
def same(self) -> bool:
|
|
32
|
+
"""True if the two functions did the same thing on this input."""
|
|
33
|
+
return not self.differences
|
|
34
|
+
|
|
35
|
+
def __str__(self) -> str:
|
|
36
|
+
call = ", ".join("%s=%s" % (name, short(value, 40)) for name, value in self.candidate.inputs.items())
|
|
37
|
+
names = "%s and %s" % (self.candidate_name, self.reference_name)
|
|
38
|
+
if self.same:
|
|
39
|
+
return "SAME: %s behave the same on (%s)." % (names, call)
|
|
40
|
+
lines = ["DIFFERENT: %s behave differently on (%s)." % (names, call)]
|
|
41
|
+
for d in self.differences:
|
|
42
|
+
lines.append(" %s: yours %s, the reference %s." % (d.what, d.yours, d.reference))
|
|
43
|
+
return "\n".join(lines)
|
|
44
|
+
|
|
45
|
+
__repr__ = __str__ # so a Comparison displays as text when it is the last line of a notebook cell
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _same_value(a: Any, b: Any) -> bool:
|
|
49
|
+
try:
|
|
50
|
+
return bool(a == b)
|
|
51
|
+
except Exception: # for example array-like objects whose == does not give a single True/False
|
|
52
|
+
return repr(a) == repr(b)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _differences(mine: Outcome, ref: Outcome) -> List[Difference]:
|
|
56
|
+
found: List[Difference] = []
|
|
57
|
+
if mine.raised or ref.raised:
|
|
58
|
+
if mine.error_type != ref.error_type: # same exception type counts as the same
|
|
59
|
+
found.append(Difference("result", mine.describe(), ref.describe()))
|
|
60
|
+
elif not _same_value(mine.returned, ref.returned):
|
|
61
|
+
found.append(Difference("result", mine.describe(), ref.describe()))
|
|
62
|
+
# Arguments are matched by position, so the two functions may name their parameters differently.
|
|
63
|
+
for (name, a), (_, b) in zip(mine.arguments.items(), ref.arguments.items()):
|
|
64
|
+
if not _same_value(a, b):
|
|
65
|
+
found.append(Difference("argument '%s' afterwards" % name, short(repr(a)), short(repr(b))))
|
|
66
|
+
return found
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _name(func: Callable[..., Any]) -> str:
|
|
70
|
+
return getattr(func, "__name__", None) or repr(func)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def compare(
|
|
74
|
+
candidate: Callable[..., Any],
|
|
75
|
+
reference: Callable[..., Any],
|
|
76
|
+
args: tuple = (),
|
|
77
|
+
kwargs: Optional[dict] = None,
|
|
78
|
+
) -> Comparison:
|
|
79
|
+
"""Run both functions on the same input and report whether they behave the same.
|
|
80
|
+
|
|
81
|
+
``candidate`` is usually your own (possibly buggy) function and ``reference`` a known-good one.
|
|
82
|
+
"Behave the same" means: the same return value, the same kind of exception, and the same changes
|
|
83
|
+
to the arguments. How the functions are written inside does not matter.
|
|
84
|
+
"""
|
|
85
|
+
mine = run(candidate, args, kwargs)
|
|
86
|
+
ref = run(reference, args, kwargs)
|
|
87
|
+
return Comparison(_name(candidate), _name(reference), mine, ref, _differences(mine, ref))
|
difffinder/demo.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
from difffinder import compare_traces
|
|
2
|
+
|
|
3
|
+
def count_to(n):
|
|
4
|
+
total = 0
|
|
5
|
+
for i in range(n):
|
|
6
|
+
total += i
|
|
7
|
+
return total
|
|
8
|
+
|
|
9
|
+
def count_to_ref(n):
|
|
10
|
+
total = 0
|
|
11
|
+
for i in range(n + 1):
|
|
12
|
+
total += i
|
|
13
|
+
return total
|
|
14
|
+
|
|
15
|
+
if __name__ == '__main__':
|
|
16
|
+
import argparse
|
|
17
|
+
|
|
18
|
+
parser = argparse.ArgumentParser(description='Run the DiffFinder demo or sorting visualizer.')
|
|
19
|
+
mode = parser.add_mutually_exclusive_group()
|
|
20
|
+
mode.add_argument('--gui', action='store_true', help='Open the sorting visualizer (default)')
|
|
21
|
+
mode.add_argument('--console', action='store_true', help='Run the numeric console demo')
|
|
22
|
+
options = parser.parse_args()
|
|
23
|
+
if not options.console:
|
|
24
|
+
from difffinder.sorting_visualizer import launch_app
|
|
25
|
+
launch_app()
|
|
26
|
+
else:
|
|
27
|
+
print(compare_traces(count_to, count_to_ref, args=(4,)))
|
difffinder/diff.py
ADDED
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
"""Compare two functions: where do they differ, on which inputs, and why."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import copy
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from typing import Any, Callable, Dict, Iterable, List, Optional
|
|
8
|
+
|
|
9
|
+
from .probe import outcome
|
|
10
|
+
from .tracer import RAISE, Event, Trace, record
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _special(name: str) -> bool:
|
|
14
|
+
"""<return>, <raise> and <argN> are outcome pseudo-variables, not real variables."""
|
|
15
|
+
return name.startswith("<")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class Divergence:
|
|
20
|
+
"""The first place two runs disagree, step by step."""
|
|
21
|
+
|
|
22
|
+
var: str # the variable (or <return> / <raise> / <argN>) that disagrees
|
|
23
|
+
position: int # 0-based: it is that variable's n-th new value
|
|
24
|
+
candidate: Optional[Event] # None means the candidate never made this change
|
|
25
|
+
reference: Optional[Event]
|
|
26
|
+
agreed: List[str] # the last few values both runs agreed on
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(repr=False)
|
|
30
|
+
class TraceComparison:
|
|
31
|
+
candidate: Trace
|
|
32
|
+
reference: Trace
|
|
33
|
+
divergence: Optional[Divergence]
|
|
34
|
+
compared: List[str] # variable names that were compared step by step
|
|
35
|
+
args: tuple = ()
|
|
36
|
+
kwargs: dict = field(default_factory=dict)
|
|
37
|
+
skipped: List[str] = field(default_factory=list) # same name, but probably a different meaning
|
|
38
|
+
hints: List[str] = field(default_factory=list) # plain-English clues, generated offline
|
|
39
|
+
candidate_fn: Any = None
|
|
40
|
+
reference_fn: Any = None
|
|
41
|
+
ai_hint: Optional[str] = None # filled in when compare_traces(..., ai=...) succeeds
|
|
42
|
+
ai_error: Optional[str] = None # why the AI hint is missing, if it failed
|
|
43
|
+
outcomes_differ: bool = False # on the input you gave: different result, exception or side effect
|
|
44
|
+
strict: bool = False # watch=/rename= given: step-by-step differences count too
|
|
45
|
+
probe: Any = None # difffinder.probe.Probe: what testing many other inputs showed
|
|
46
|
+
|
|
47
|
+
@property
|
|
48
|
+
def same(self) -> bool:
|
|
49
|
+
if self.probe is not None and self.probe.failing:
|
|
50
|
+
return False
|
|
51
|
+
return self.divergence is None if self.strict else not self.outcomes_differ
|
|
52
|
+
|
|
53
|
+
def explain(self, llm: Any = None, share_reference: bool = True) -> str:
|
|
54
|
+
"""Ask an LLM for a short hint. See :mod:`difffinder.ai`."""
|
|
55
|
+
from .ai import explain
|
|
56
|
+
|
|
57
|
+
return explain(self, llm, share_reference)
|
|
58
|
+
|
|
59
|
+
def __str__(self) -> str:
|
|
60
|
+
from .render import to_text
|
|
61
|
+
|
|
62
|
+
return to_text(self)
|
|
63
|
+
|
|
64
|
+
__repr__ = __str__
|
|
65
|
+
|
|
66
|
+
def _repr_html_(self) -> str:
|
|
67
|
+
from .render import to_html
|
|
68
|
+
|
|
69
|
+
return to_html(self)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def find_divergence(
|
|
73
|
+
candidate: Trace,
|
|
74
|
+
reference: Trace,
|
|
75
|
+
*,
|
|
76
|
+
watch: Optional[Iterable[str]] = None,
|
|
77
|
+
rename: Optional[Dict[str, str]] = None,
|
|
78
|
+
):
|
|
79
|
+
"""Return ``(divergence_or_None, compared_names, skipped_names)`` for two traces.
|
|
80
|
+
|
|
81
|
+
Each shared variable's history is compared separately, so unrelated assignments in a different order
|
|
82
|
+
are not a difference. The outcome (return value, exception, final state of every argument) is always
|
|
83
|
+
compared. In automatic mode (no ``watch``/``rename``) a variable the two runs barely agree on is
|
|
84
|
+
*skipped*: different algorithms often reuse a name like ``i`` for different things.
|
|
85
|
+
"""
|
|
86
|
+
rename = rename or {}
|
|
87
|
+
|
|
88
|
+
def histories(trace: Trace, mapping: Dict[str, str]) -> Dict[str, List[Event]]:
|
|
89
|
+
out: Dict[str, List[Event]] = {}
|
|
90
|
+
for e in trace.events:
|
|
91
|
+
out.setdefault(mapping.get(e.var, e.var), []).append(e)
|
|
92
|
+
return out
|
|
93
|
+
|
|
94
|
+
ha, hb = histories(candidate, rename), histories(reference, {})
|
|
95
|
+
auto = watch is None and not rename
|
|
96
|
+
specials = {n for n in set(ha) | set(hb) if _special(n)}
|
|
97
|
+
names = (set(watch) if watch is not None else set(ha) & set(hb)) | specials
|
|
98
|
+
|
|
99
|
+
found = []
|
|
100
|
+
for name in names:
|
|
101
|
+
a, b = ha.get(name, []), hb.get(name, [])
|
|
102
|
+
for i in range(max(len(a), len(b))):
|
|
103
|
+
ea = a[i] if i < len(a) else None
|
|
104
|
+
eb = b[i] if i < len(b) else None
|
|
105
|
+
if ea is None or eb is None or ea.value != eb.value:
|
|
106
|
+
when = ea.step if ea is not None else candidate.steps
|
|
107
|
+
if _special(name):
|
|
108
|
+
rank = 1 if name == RAISE else 2
|
|
109
|
+
elif ea is None and RAISE in ha:
|
|
110
|
+
rank = 1.5 # "never got here" is a consequence of the crash, so the crash goes first
|
|
111
|
+
else:
|
|
112
|
+
rank = 0 # real variables beat outcomes
|
|
113
|
+
agreed = [e.value for e in a[max(0, i - 3) : i]]
|
|
114
|
+
found.append(((when, rank, name), Divergence(name, i, ea, eb, agreed)))
|
|
115
|
+
break
|
|
116
|
+
found.sort(key=lambda item: item[0])
|
|
117
|
+
|
|
118
|
+
# A name is only trusted to mean the same thing in both functions if it is a parameter of both,
|
|
119
|
+
# or the two runs agreed on at least two of its values before diverging.
|
|
120
|
+
params = set(candidate.arg_names.values()) & set(reference.arg_names.values())
|
|
121
|
+
chosen, skipped = None, []
|
|
122
|
+
for _, div in found:
|
|
123
|
+
aligned = len(div.agreed) >= 2 or (div.var in params and div.position >= 1)
|
|
124
|
+
if auto and not _special(div.var) and not aligned:
|
|
125
|
+
skipped.append(div.var)
|
|
126
|
+
continue
|
|
127
|
+
chosen = div
|
|
128
|
+
break
|
|
129
|
+
compared = sorted(n for n in names if not _special(n) and (n in ha or n in hb))
|
|
130
|
+
return chosen, compared, skipped
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def compare_traces(
|
|
134
|
+
candidate: Callable[..., Any],
|
|
135
|
+
reference: Callable[..., Any],
|
|
136
|
+
args: tuple = (),
|
|
137
|
+
kwargs: Optional[dict] = None,
|
|
138
|
+
*,
|
|
139
|
+
watch: Optional[Iterable[str]] = None,
|
|
140
|
+
rename: Optional[Dict[str, str]] = None,
|
|
141
|
+
max_steps: int = 100_000,
|
|
142
|
+
probe: bool = True,
|
|
143
|
+
ai: Any = None,
|
|
144
|
+
) -> TraceComparison:
|
|
145
|
+
"""Run both functions on the same input and report how they differ.
|
|
146
|
+
|
|
147
|
+
``candidate`` is usually the buggy or student version, ``reference`` the known-good one. Each run gets
|
|
148
|
+
its own deep copy of ``args``/``kwargs``, so in-place changes cannot leak.
|
|
149
|
+
|
|
150
|
+
The verdict is based on what each function visibly does (return value, exception, changes to its
|
|
151
|
+
arguments), so it works however differently the two are written. With ``probe=True`` (the default) it
|
|
152
|
+
also tries many variations of your input, shrinks a failing one to the smallest case, and describes
|
|
153
|
+
when they disagree. This calls both functions many times, so use ``probe=False`` for functions with
|
|
154
|
+
side effects or that are slow.
|
|
155
|
+
|
|
156
|
+
``watch`` limits the step-by-step comparison to the named variables (``watch=[]`` compares only the
|
|
157
|
+
outcome). ``rename`` maps variable names in ``candidate`` to the names in ``reference``. Giving either
|
|
158
|
+
means you vouch that the names line up, and step-by-step differences then count as differences.
|
|
159
|
+
|
|
160
|
+
``ai=True`` adds a short model-written hint (needs an API key, see :mod:`difffinder.ai`); ``ai`` may
|
|
161
|
+
also be ``"hermes"``, ``"openai"`` or a function ``prompt -> str``. If the model cannot be reached the
|
|
162
|
+
comparison still works and the reason is shown instead.
|
|
163
|
+
"""
|
|
164
|
+
kwargs = kwargs or {}
|
|
165
|
+
ta = record(candidate, copy.deepcopy(args), copy.deepcopy(kwargs), max_steps=max_steps)
|
|
166
|
+
tb = record(reference, copy.deepcopy(args), copy.deepcopy(kwargs), max_steps=max_steps)
|
|
167
|
+
divergence, compared, skipped = find_divergence(ta, tb, watch=watch, rename=rename)
|
|
168
|
+
strict = watch is not None or bool(rename)
|
|
169
|
+
differ = outcome(ta) != outcome(tb)
|
|
170
|
+
if not strict and not differ:
|
|
171
|
+
divergence = None # internal steps may differ between two correct solutions
|
|
172
|
+
result = TraceComparison(
|
|
173
|
+
ta, tb, divergence, compared, tuple(args), dict(kwargs), skipped, [], candidate, reference,
|
|
174
|
+
outcomes_differ=differ, strict=strict,
|
|
175
|
+
)
|
|
176
|
+
if probe and args:
|
|
177
|
+
from .probe import run_probe
|
|
178
|
+
|
|
179
|
+
names = [ta.arg_names.get("<arg%d>" % i, "arg%d" % i) for i in range(len(args))]
|
|
180
|
+
try:
|
|
181
|
+
result.probe = run_probe(candidate, reference, tuple(args), kwargs, names, differ, ta, tb)
|
|
182
|
+
except Exception: # probing is a bonus; never let it break the comparison
|
|
183
|
+
result.probe = None
|
|
184
|
+
if not result.same:
|
|
185
|
+
from .hints import generate
|
|
186
|
+
|
|
187
|
+
result.hints = generate(result)
|
|
188
|
+
if ai:
|
|
189
|
+
from .ai import explain
|
|
190
|
+
|
|
191
|
+
try:
|
|
192
|
+
result.ai_hint = explain(result, "auto" if ai is True else ai)
|
|
193
|
+
except Exception as exc: # a missing key or offline model must never break the comparison
|
|
194
|
+
result.ai_error = "%s: %s" % (type(exc).__name__, exc)
|
|
195
|
+
return result
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def find_difference(
|
|
199
|
+
candidate: Callable[..., Any],
|
|
200
|
+
reference: Callable[..., Any],
|
|
201
|
+
cases: Iterable[Any],
|
|
202
|
+
**options: Any,
|
|
203
|
+
) -> Optional[TraceComparison]:
|
|
204
|
+
"""Try each input in ``cases``; return the first :class:`TraceComparison` that differs, or None.
|
|
205
|
+
|
|
206
|
+
A case that is a tuple is used as the positional arguments; anything else is wrapped.
|
|
207
|
+
"""
|
|
208
|
+
options.setdefault("probe", False) # this function is itself the search
|
|
209
|
+
for case in cases:
|
|
210
|
+
args = case if isinstance(case, tuple) else (case,)
|
|
211
|
+
result = compare_traces(candidate, reference, args, **options)
|
|
212
|
+
if not result.same:
|
|
213
|
+
return result
|
|
214
|
+
return None
|
difffinder/hints.py
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""Plain-English hints from a Comparison. Deterministic: no AI, no network."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import ast
|
|
6
|
+
import inspect
|
|
7
|
+
import linecache
|
|
8
|
+
import re
|
|
9
|
+
import textwrap
|
|
10
|
+
from typing import List
|
|
11
|
+
|
|
12
|
+
from .tracer import RAISE, RETURN
|
|
13
|
+
|
|
14
|
+
_MAX_HINTS = 5
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _exception_hint(text: str) -> str:
|
|
18
|
+
m = re.match(r"ValueError: invalid literal for int\(\) with base 10: '(.*)'", text)
|
|
19
|
+
if m:
|
|
20
|
+
bad = m.group(1)
|
|
21
|
+
extra = " Negative numbers contain a '-' when you loop over str(number); take abs() first." if bad == "-" else ""
|
|
22
|
+
return "int() was given %r, which is not a string of digits.%s" % (bad, extra)
|
|
23
|
+
m = re.match(r"AttributeError: 'NoneType' object has no attribute '(\w+)'", text)
|
|
24
|
+
if m:
|
|
25
|
+
return (
|
|
26
|
+
"A variable is None when you call .%s on it. A common cause is assigning the result of a "
|
|
27
|
+
"method that works in place (list.remove, append, sort, reverse), which returns None." % m.group(1)
|
|
28
|
+
)
|
|
29
|
+
if text.startswith("IndexError"):
|
|
30
|
+
return "An index went past the end of a list. Check your loop bounds, and whether you remove items while looping."
|
|
31
|
+
if text.startswith("KeyError"):
|
|
32
|
+
return "A dictionary key was missing. Check the keys you add, or use .get()."
|
|
33
|
+
if text.startswith("ZeroDivisionError"):
|
|
34
|
+
return "Something was divided by zero. Check the case where the divisor can be 0."
|
|
35
|
+
return ""
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def generate(cmp) -> List[str]:
|
|
39
|
+
"""Return up to a few short hints about the candidate's behaviour."""
|
|
40
|
+
out: List[str] = []
|
|
41
|
+
d = cmp.divergence
|
|
42
|
+
traces = [cmp.candidate]
|
|
43
|
+
probe = getattr(cmp, "probe", None)
|
|
44
|
+
if probe is not None and probe.minimal_candidate is not None:
|
|
45
|
+
traces.append(probe.minimal_candidate) # the smallest failing run is the clearest evidence
|
|
46
|
+
|
|
47
|
+
for trace in traces:
|
|
48
|
+
for ev in trace.history(RAISE):
|
|
49
|
+
hint = _exception_hint(ev.value)
|
|
50
|
+
if hint:
|
|
51
|
+
out.append(hint)
|
|
52
|
+
# A list that suddenly turns into None: the result of an in-place method was assigned back.
|
|
53
|
+
for var in sorted(trace.variables):
|
|
54
|
+
hist = trace.history(var)
|
|
55
|
+
for prev, cur in zip(hist, hist[1:]):
|
|
56
|
+
if cur.value == "None" and prev.value.startswith(("[", "{")):
|
|
57
|
+
src = linecache.getline(cur.filename, cur.lineno).strip()
|
|
58
|
+
out.append(
|
|
59
|
+
"'%s' was a collection but became None at line %d%s. Methods like remove/append/sort "
|
|
60
|
+
"change the object in place and return None, so do not assign their result back."
|
|
61
|
+
% (var, cur.lineno, (" (`%s`)" % src) if src else "")
|
|
62
|
+
)
|
|
63
|
+
break
|
|
64
|
+
|
|
65
|
+
name_of = cmp.candidate.arg_names
|
|
66
|
+
if d is not None and d.var.startswith("<arg"):
|
|
67
|
+
out.append(
|
|
68
|
+
"The input '%s' ends up different from the reference. If the task says to modify it in place, "
|
|
69
|
+
"change the object itself; rebinding the name to a new object does not affect the caller."
|
|
70
|
+
% name_of.get(d.var, "an argument")
|
|
71
|
+
)
|
|
72
|
+
if d is not None and d.var == RETURN and d.candidate is not None and d.candidate.value == "None":
|
|
73
|
+
out.append("Your function returned None. Does every path end with `return <value>`?")
|
|
74
|
+
|
|
75
|
+
out.extend(_source_hints(cmp))
|
|
76
|
+
seen, unique = set(), []
|
|
77
|
+
for h in out:
|
|
78
|
+
if h and h not in seen:
|
|
79
|
+
seen.add(h)
|
|
80
|
+
unique.append(h)
|
|
81
|
+
return unique[:_MAX_HINTS]
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _source_hints(cmp) -> List[str]:
|
|
85
|
+
"""Hints that come from reading the candidate's source code."""
|
|
86
|
+
try:
|
|
87
|
+
lines, first = inspect.getsourcelines(cmp.candidate_fn)
|
|
88
|
+
tree = ast.parse(textwrap.dedent("".join(lines)))
|
|
89
|
+
fn = next(n for n in ast.walk(tree) if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)))
|
|
90
|
+
except (OSError, TypeError, SyntaxError, StopIteration):
|
|
91
|
+
return []
|
|
92
|
+
params = [a.arg for a in fn.args.posonlyargs + fn.args.args + fn.args.kwonlyargs]
|
|
93
|
+
out = []
|
|
94
|
+
for node in ast.walk(fn):
|
|
95
|
+
line = first + getattr(node, "lineno", 1) - 1
|
|
96
|
+
# for ... else without a break: the else block always runs
|
|
97
|
+
if isinstance(node, (ast.For, ast.While)) and node.orelse:
|
|
98
|
+
if not any(isinstance(n, ast.Break) for b in node.body for n in ast.walk(b)):
|
|
99
|
+
out.append(
|
|
100
|
+
"Line %d: a loop has an `else:` block but no `break`. A loop's `else` runs whenever the "
|
|
101
|
+
"loop finishes without break, so here it always runs; it is not the 'otherwise' of an `if`." % line
|
|
102
|
+
)
|
|
103
|
+
# alias = parameter: does not copy a list/dict/set
|
|
104
|
+
if (
|
|
105
|
+
isinstance(node, ast.Assign)
|
|
106
|
+
and len(node.targets) == 1
|
|
107
|
+
and isinstance(node.targets[0], ast.Name)
|
|
108
|
+
and isinstance(node.value, ast.Name)
|
|
109
|
+
and node.value.id in params
|
|
110
|
+
):
|
|
111
|
+
idx = params.index(node.value.id)
|
|
112
|
+
if idx < len(cmp.args) and isinstance(cmp.args[idx], (list, dict, set)):
|
|
113
|
+
out.append(
|
|
114
|
+
"Line %d: `%s = %s` does not copy. Both names point to the same %s, so changing one changes "
|
|
115
|
+
"the other. Use a copy (for example `list(%s)`) if you need the original."
|
|
116
|
+
% (line, node.targets[0].id, node.value.id, type(cmp.args[idx]).__name__, node.value.id)
|
|
117
|
+
)
|
|
118
|
+
return out
|