stucktrace 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stucktrace/__init__.py +50 -0
- stucktrace/__main__.py +3 -0
- stucktrace/_version.py +1 -0
- stucktrace/adapters.py +501 -0
- stucktrace/cli.py +64 -0
- stucktrace/compressor.py +405 -0
- stucktrace/demo.py +130 -0
- stucktrace/events.py +38 -0
- stucktrace/reporter.py +104 -0
- stucktrace/tracker.py +440 -0
- stucktrace-0.1.0.dist-info/METADATA +305 -0
- stucktrace-0.1.0.dist-info/RECORD +16 -0
- stucktrace-0.1.0.dist-info/WHEEL +5 -0
- stucktrace-0.1.0.dist-info/entry_points.txt +2 -0
- stucktrace-0.1.0.dist-info/licenses/LICENSE +21 -0
- stucktrace-0.1.0.dist-info/top_level.txt +1 -0
stucktrace/compressor.py
ADDED
|
@@ -0,0 +1,405 @@
|
|
|
1
|
+
"""Trajectory Compressor.
|
|
2
|
+
|
|
3
|
+
Turns a raw event log (hundreds of runs, previews, timestamps) into a compact,
|
|
4
|
+
structured context that a human or an LLM can read in seconds:
|
|
5
|
+
|
|
6
|
+
1. **Error clustering** – error messages are normalised (identifiers, numbers,
|
|
7
|
+
addresses stripped) into *signatures*, so ``name 'x' is not defined`` and
|
|
8
|
+
``name 'total' is not defined`` collapse into one cluster.
|
|
9
|
+
2. **Stuck / panic detection** – five signals per task:
|
|
10
|
+
* ``rapid_fail_burst`` – >= N failed runs with short gaps between them
|
|
11
|
+
* ``repeated_error`` – the same signature N times in a row
|
|
12
|
+
* ``long_pause`` – a long silence right after a failure
|
|
13
|
+
* ``rewrite`` – a large edit right after a failure (thrashing)
|
|
14
|
+
* ``confusion`` – explicit ``mark_confusion()`` marker
|
|
15
|
+
The signals are combined into a ``panic_score`` in [0, 1].
|
|
16
|
+
3. **Key transitions** – every failure -> success step, with the error that was
|
|
17
|
+
fixed, how many attempts it took, the edit size and whether a visualization
|
|
18
|
+
step preceded it.
|
|
19
|
+
4. **Run-length digest** – a one-line symbolic timeline, e.g.
|
|
20
|
+
``F:NameError x3 | P:140s | ?"loops" | V | S``.
|
|
21
|
+
"""
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import json
|
|
25
|
+
import math
|
|
26
|
+
import re
|
|
27
|
+
from collections import Counter
|
|
28
|
+
from dataclasses import asdict, dataclass, field
|
|
29
|
+
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
|
|
30
|
+
|
|
31
|
+
from .events import CONFUSION, RUN, TASK, VIZ, Event
|
|
32
|
+
|
|
33
|
+
if TYPE_CHECKING: # pragma: no cover
|
|
34
|
+
from .tracker import Session
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass
|
|
38
|
+
class CompressorConfig:
|
|
39
|
+
panic_gap: float = 25.0 # seconds; failed runs closer than this form a burst
|
|
40
|
+
panic_min_burst: int = 3 # failed runs needed for a burst
|
|
41
|
+
repeat_streak: int = 3 # identical error signature in a row
|
|
42
|
+
stall_gap: float = 120.0 # seconds of silence after a failure = long pause
|
|
43
|
+
rewrite_ratio: float = 0.4 # edit_ratio after a failure that counts as a rewrite
|
|
44
|
+
max_examples: int = 1 # code previews kept per error cluster
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
_NORMALISERS = [
|
|
48
|
+
(re.compile(r"'[^']*'|\"[^\"]*\""), "'…'"),
|
|
49
|
+
(re.compile(r"\b0x[0-9a-fA-F]+\b"), "<addr>"),
|
|
50
|
+
(re.compile(r"\b\d+(\.\d+)?\b"), "<n>"),
|
|
51
|
+
(re.compile(r"\s+"), " "),
|
|
52
|
+
]
|
|
53
|
+
|
|
54
|
+
_WEIGHTS = {"rapid_fail_burst": 1.0, "repeated_error": 0.7, "long_pause": 0.5,
|
|
55
|
+
"rewrite": 0.5, "confusion": 0.9}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def error_signature(error_type: Optional[str], message: Optional[str]) -> str:
|
|
59
|
+
msg = message or ""
|
|
60
|
+
for rx, repl in _NORMALISERS:
|
|
61
|
+
msg = rx.sub(repl, msg)
|
|
62
|
+
msg = msg.strip()[:100]
|
|
63
|
+
return f"{error_type or 'Error'}: {msg}" if msg else (error_type or "Error")
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass
|
|
67
|
+
class ErrorCluster:
|
|
68
|
+
signature: str
|
|
69
|
+
error_type: str
|
|
70
|
+
count: int
|
|
71
|
+
first_attempt: int
|
|
72
|
+
last_attempt: int
|
|
73
|
+
max_streak: int
|
|
74
|
+
resolved: bool
|
|
75
|
+
example: Optional[str] = None
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@dataclass
|
|
79
|
+
class StuckPoint:
|
|
80
|
+
kind: str
|
|
81
|
+
t: float # seconds since session start
|
|
82
|
+
attempt: Optional[int] = None # 1-based attempt number inside the task
|
|
83
|
+
detail: str = "" # English, used in the LLM prompt
|
|
84
|
+
data: Dict[str, Any] = field(default_factory=dict) # structured params for localized rendering
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
@dataclass
|
|
88
|
+
class Transition:
|
|
89
|
+
fixed_error: str
|
|
90
|
+
after_attempts: int
|
|
91
|
+
edit_ratio: Optional[float]
|
|
92
|
+
seconds: float
|
|
93
|
+
viz_before: bool
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@dataclass
|
|
97
|
+
class TaskSummary:
|
|
98
|
+
task: str
|
|
99
|
+
attempts: int
|
|
100
|
+
failures: int
|
|
101
|
+
solved: bool
|
|
102
|
+
final_ok: bool
|
|
103
|
+
first_success_attempt: Optional[int]
|
|
104
|
+
time_on_task: float
|
|
105
|
+
time_to_first_success: Optional[float]
|
|
106
|
+
panic_score: float
|
|
107
|
+
panic_level: str
|
|
108
|
+
error_clusters: List[ErrorCluster] = field(default_factory=list)
|
|
109
|
+
stuck_points: List[StuckPoint] = field(default_factory=list)
|
|
110
|
+
transitions: List[Transition] = field(default_factory=list)
|
|
111
|
+
confusions: List[str] = field(default_factory=list)
|
|
112
|
+
viz_steps: int = 0
|
|
113
|
+
viz_helped: bool = False
|
|
114
|
+
digest: str = ""
|
|
115
|
+
|
|
116
|
+
@property
|
|
117
|
+
def top_error(self) -> Optional[str]:
|
|
118
|
+
return self.error_clusters[0].signature if self.error_clusters else None
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@dataclass
|
|
122
|
+
class CompressedTrajectory:
|
|
123
|
+
session_id: str
|
|
124
|
+
metadata: Dict[str, Any]
|
|
125
|
+
duration: float
|
|
126
|
+
totals: Dict[str, Any]
|
|
127
|
+
tasks: List[TaskSummary]
|
|
128
|
+
stats: Dict[str, Any] = field(default_factory=dict)
|
|
129
|
+
|
|
130
|
+
# ---------------------------------------------------------------- views
|
|
131
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
132
|
+
d = asdict(self)
|
|
133
|
+
for t, src in zip(d["tasks"], self.tasks):
|
|
134
|
+
t["top_error"] = src.top_error
|
|
135
|
+
return d
|
|
136
|
+
|
|
137
|
+
def to_json(self, indent: Optional[int] = 2) -> str:
|
|
138
|
+
return json.dumps(self.to_dict(), ensure_ascii=False, indent=indent)
|
|
139
|
+
|
|
140
|
+
def hardest(self) -> Optional[TaskSummary]:
|
|
141
|
+
return max(self.tasks, key=lambda t: (t.panic_score, t.failures), default=None)
|
|
142
|
+
|
|
143
|
+
def to_markdown(self) -> str:
|
|
144
|
+
tt = self.totals
|
|
145
|
+
out = [
|
|
146
|
+
f"### Compressed trajectory `{self.session_id}`",
|
|
147
|
+
f"{tt['tasks']} task(s) · {tt['runs']} runs · {tt['failures']} failed · "
|
|
148
|
+
f"{_fmt_dur(self.duration)} · compression {self.stats.get('ratio', '?')}×",
|
|
149
|
+
"",
|
|
150
|
+
"| Task | Runs | Solved | Panic | Top error | Digest |",
|
|
151
|
+
"|---|---|---|---|---|---|",
|
|
152
|
+
]
|
|
153
|
+
for t in self.tasks:
|
|
154
|
+
out.append(
|
|
155
|
+
f"| {t.task} | {t.attempts} | {'✅' if t.solved else '❌'} | "
|
|
156
|
+
f"{t.panic_level} ({t.panic_score:.2f}) | {t.top_error or '—'} | `{t.digest}` |"
|
|
157
|
+
)
|
|
158
|
+
return "\n".join(out)
|
|
159
|
+
|
|
160
|
+
def to_prompt(self, max_chars: int = 6000) -> str:
|
|
161
|
+
"""Compact, LLM-friendly text (no code previews unless there is room)."""
|
|
162
|
+
lines = [f"SESSION {self.session_id} duration={_fmt_dur(self.duration)} "
|
|
163
|
+
f"tasks={self.totals['tasks']} runs={self.totals['runs']} failures={self.totals['failures']}"]
|
|
164
|
+
for t in self.tasks:
|
|
165
|
+
lines.append(
|
|
166
|
+
f"\nTASK {t.task}: attempts={t.attempts} failures={t.failures} solved={t.solved} "
|
|
167
|
+
f"first_success_attempt={t.first_success_attempt} time={_fmt_dur(t.time_on_task)} "
|
|
168
|
+
f"panic={t.panic_level}({t.panic_score:.2f}) viz_steps={t.viz_steps} viz_helped={t.viz_helped}"
|
|
169
|
+
)
|
|
170
|
+
lines.append(f" digest: {t.digest}")
|
|
171
|
+
for c in t.error_clusters:
|
|
172
|
+
lines.append(f" error x{c.count} (streak {c.max_streak}, resolved={c.resolved}): {c.signature}")
|
|
173
|
+
for s in t.stuck_points:
|
|
174
|
+
lines.append(f" stuck[{s.kind}] at +{_fmt_clock(s.t)}: {s.detail}")
|
|
175
|
+
for tr in t.transitions:
|
|
176
|
+
lines.append(f" fix: {tr.fixed_error} after {tr.after_attempts} failed attempt(s), "
|
|
177
|
+
f"edit={_pct(tr.edit_ratio)}, {tr.seconds:.0f}s, viz_before={tr.viz_before}")
|
|
178
|
+
for c in t.confusions:
|
|
179
|
+
lines.append(f' student said: "{c}"')
|
|
180
|
+
text = "\n".join(lines)
|
|
181
|
+
if len(text) < max_chars * 0.7:
|
|
182
|
+
extra = []
|
|
183
|
+
for t in self.tasks:
|
|
184
|
+
for c in t.error_clusters:
|
|
185
|
+
if c.example:
|
|
186
|
+
extra.append(f" [{t.task}] code at '{c.error_type}':\n " + c.example.replace("\n", "\n "))
|
|
187
|
+
if extra:
|
|
188
|
+
text += "\n\nCODE SNAPSHOTS (for diagnosis only):\n" + "\n".join(extra)
|
|
189
|
+
return text[:max_chars]
|
|
190
|
+
|
|
191
|
+
def _repr_markdown_(self) -> str:
|
|
192
|
+
return self.to_markdown()
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
# ------------------------------------------------------------------ helpers
|
|
196
|
+
def _fmt_dur(sec: Optional[float]) -> str:
|
|
197
|
+
if sec is None:
|
|
198
|
+
return "—"
|
|
199
|
+
sec = max(0, int(round(sec)))
|
|
200
|
+
return f"{sec // 60}m{sec % 60:02d}s" if sec >= 60 else f"{sec}s"
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _fmt_clock(sec: float) -> str:
|
|
204
|
+
sec = max(0, int(sec))
|
|
205
|
+
return f"{sec // 60:02d}:{sec % 60:02d}"
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _pct(r: Optional[float]) -> str:
|
|
209
|
+
return "—" if r is None else f"{r * 100:.0f}%"
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def _level(score: float) -> str:
|
|
213
|
+
return "high" if score >= 0.66 else "medium" if score >= 0.33 else "low"
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _digest(events: List[Event], t0: float, cfg: CompressorConfig) -> str:
|
|
217
|
+
tokens: List[str] = []
|
|
218
|
+
prev: Optional[Event] = None
|
|
219
|
+
for e in events:
|
|
220
|
+
if prev is not None and prev.kind == RUN and not prev.ok and e.ts - prev.ts >= cfg.stall_gap:
|
|
221
|
+
tokens.append(f"P:{e.ts - prev.ts:.0f}s")
|
|
222
|
+
if e.kind == RUN:
|
|
223
|
+
tokens.append("S" if e.ok else f"F:{e.error_type or 'Error'}")
|
|
224
|
+
elif e.kind == CONFUSION:
|
|
225
|
+
tokens.append(f'?"{(e.text or "")[:24]}"')
|
|
226
|
+
elif e.kind == VIZ:
|
|
227
|
+
tokens.append("V")
|
|
228
|
+
if e.kind in (RUN, CONFUSION, VIZ):
|
|
229
|
+
prev = e
|
|
230
|
+
# run-length encode
|
|
231
|
+
out: List[str] = []
|
|
232
|
+
i = 0
|
|
233
|
+
while i < len(tokens):
|
|
234
|
+
j = i
|
|
235
|
+
while j + 1 < len(tokens) and tokens[j + 1] == tokens[i]:
|
|
236
|
+
j += 1
|
|
237
|
+
n = j - i + 1
|
|
238
|
+
out.append(tokens[i] if n == 1 else f"{tokens[i]} x{n}")
|
|
239
|
+
i = j + 1
|
|
240
|
+
return " | ".join(out)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _analyze_task(name: str, events: List[Event], t0: float, cfg: CompressorConfig) -> TaskSummary:
|
|
244
|
+
runs = [e for e in events if e.kind == RUN]
|
|
245
|
+
fails = [e for e in runs if not e.ok]
|
|
246
|
+
first_ok = next((i for i, e in enumerate(runs) if e.ok), None)
|
|
247
|
+
t_start = events[0].ts
|
|
248
|
+
t_end = events[-1].ts
|
|
249
|
+
|
|
250
|
+
# ---- error clusters
|
|
251
|
+
sigs = [error_signature(e.error_type, e.error_msg) if not e.ok else None for e in runs]
|
|
252
|
+
clusters: Dict[str, ErrorCluster] = {}
|
|
253
|
+
streak_sig, streak = None, 0
|
|
254
|
+
for idx, (e, sig) in enumerate(zip(runs, sigs)):
|
|
255
|
+
streak = streak + 1 if (sig is not None and sig == streak_sig) else (1 if sig else 0)
|
|
256
|
+
streak_sig = sig
|
|
257
|
+
if sig is None:
|
|
258
|
+
continue
|
|
259
|
+
c = clusters.get(sig)
|
|
260
|
+
if c is None:
|
|
261
|
+
c = clusters[sig] = ErrorCluster(sig, e.error_type or "Error", 0, idx + 1, idx + 1, 0, False,
|
|
262
|
+
e.code_preview if cfg.max_examples else None)
|
|
263
|
+
c.count += 1
|
|
264
|
+
c.last_attempt = idx + 1
|
|
265
|
+
c.max_streak = max(c.max_streak, streak)
|
|
266
|
+
for c in clusters.values():
|
|
267
|
+
c.resolved = any(r.ok for r in runs[c.last_attempt:])
|
|
268
|
+
ordered = sorted(clusters.values(), key=lambda c: (-c.count, c.first_attempt))
|
|
269
|
+
|
|
270
|
+
# ---- stuck points
|
|
271
|
+
stuck: List[StuckPoint] = []
|
|
272
|
+
# rapid-fail bursts
|
|
273
|
+
burst: List[int] = []
|
|
274
|
+
for idx, e in enumerate(runs + [None]): # sentinel flushes the last burst
|
|
275
|
+
cont = (e is not None and not e.ok and burst and e.ts - runs[burst[-1]].ts <= cfg.panic_gap)
|
|
276
|
+
if e is not None and not e.ok and (not burst or cont):
|
|
277
|
+
burst.append(idx)
|
|
278
|
+
continue
|
|
279
|
+
if len(burst) >= cfg.panic_min_burst:
|
|
280
|
+
a, b = runs[burst[0]], runs[burst[-1]]
|
|
281
|
+
span = round(b.ts - a.ts)
|
|
282
|
+
stuck.append(StuckPoint("rapid_fail_burst", a.ts - t0, burst[0] + 1,
|
|
283
|
+
f"{len(burst)} failed runs in {span}s", {"n": len(burst), "span": span}))
|
|
284
|
+
burst = [idx] if (e is not None and not e.ok) else []
|
|
285
|
+
# repeated identical errors
|
|
286
|
+
for c in ordered:
|
|
287
|
+
if c.max_streak >= cfg.repeat_streak:
|
|
288
|
+
first = runs[c.first_attempt - 1]
|
|
289
|
+
stuck.append(StuckPoint("repeated_error", first.ts - t0, c.first_attempt,
|
|
290
|
+
f"same {c.error_type} {c.max_streak}x in a row",
|
|
291
|
+
{"etype": c.error_type, "n": c.max_streak}))
|
|
292
|
+
# long pauses after failure / rewrites after failure
|
|
293
|
+
timeline = [e for e in events if e.kind in (RUN, CONFUSION, VIZ)]
|
|
294
|
+
for a, b in zip(timeline, timeline[1:]):
|
|
295
|
+
if a.kind == RUN and not a.ok and b.ts - a.ts >= cfg.stall_gap:
|
|
296
|
+
secs = round(b.ts - a.ts)
|
|
297
|
+
stuck.append(StuckPoint("long_pause", a.ts - t0, runs.index(a) + 1,
|
|
298
|
+
f"{secs}s silence after {a.error_type}",
|
|
299
|
+
{"secs": secs, "etype": a.error_type or "Error"}))
|
|
300
|
+
for i in range(1, len(runs)):
|
|
301
|
+
r, p = runs[i], runs[i - 1]
|
|
302
|
+
if not p.ok and r.edit_ratio is not None and r.edit_ratio >= cfg.rewrite_ratio:
|
|
303
|
+
stuck.append(StuckPoint("rewrite", r.ts - t0, i + 1,
|
|
304
|
+
f"{_pct(r.edit_ratio)} of code changed after {p.error_type}",
|
|
305
|
+
{"pct": _pct(r.edit_ratio), "etype": p.error_type or "Error"}))
|
|
306
|
+
confusions = [e for e in events if e.kind == CONFUSION]
|
|
307
|
+
for e in confusions:
|
|
308
|
+
att = sum(1 for r in runs if r.ts <= e.ts)
|
|
309
|
+
stuck.append(StuckPoint("confusion", e.ts - t0, att or None, f'"{e.text}"' if e.text else "marked",
|
|
310
|
+
{"text": e.text or "?"}))
|
|
311
|
+
stuck.sort(key=lambda s: s.t)
|
|
312
|
+
|
|
313
|
+
# ---- panic score
|
|
314
|
+
kinds = Counter(s.kind for s in stuck)
|
|
315
|
+
raw = sum(_WEIGHTS[k] * n for k, n in kinds.items()) + 0.1 * max(0, len(fails) - 2)
|
|
316
|
+
score = round(1 - math.exp(-raw / 2.0), 3)
|
|
317
|
+
|
|
318
|
+
# ---- transitions + viz effect
|
|
319
|
+
viz = [e for e in events if e.kind == VIZ]
|
|
320
|
+
transitions: List[Transition] = []
|
|
321
|
+
fail_streak_start: Optional[int] = None
|
|
322
|
+
for i, r in enumerate(runs):
|
|
323
|
+
if not r.ok:
|
|
324
|
+
if fail_streak_start is None:
|
|
325
|
+
fail_streak_start = i
|
|
326
|
+
continue
|
|
327
|
+
if fail_streak_start is not None:
|
|
328
|
+
last_fail = runs[i - 1]
|
|
329
|
+
transitions.append(Transition(
|
|
330
|
+
fixed_error=error_signature(last_fail.error_type, last_fail.error_msg),
|
|
331
|
+
after_attempts=i - fail_streak_start, # failed attempts before the fix
|
|
332
|
+
edit_ratio=r.edit_ratio,
|
|
333
|
+
seconds=round(r.ts - runs[fail_streak_start].ts, 1),
|
|
334
|
+
viz_before=any(runs[fail_streak_start].ts <= v.ts <= r.ts for v in viz),
|
|
335
|
+
))
|
|
336
|
+
fail_streak_start = None
|
|
337
|
+
viz_helped = False
|
|
338
|
+
for v in viz:
|
|
339
|
+
if any(not r.ok for r in runs if r.ts <= v.ts):
|
|
340
|
+
after = [r for r in runs if r.ts > v.ts][:3]
|
|
341
|
+
if any(r.ok for r in after):
|
|
342
|
+
viz_helped = True
|
|
343
|
+
break
|
|
344
|
+
|
|
345
|
+
return TaskSummary(
|
|
346
|
+
task=name,
|
|
347
|
+
attempts=len(runs),
|
|
348
|
+
failures=len(fails),
|
|
349
|
+
solved=first_ok is not None,
|
|
350
|
+
final_ok=bool(runs and runs[-1].ok),
|
|
351
|
+
first_success_attempt=None if first_ok is None else first_ok + 1,
|
|
352
|
+
time_on_task=round(t_end - t_start, 1),
|
|
353
|
+
time_to_first_success=None if first_ok is None else round(runs[first_ok].ts - t_start, 1),
|
|
354
|
+
panic_score=score,
|
|
355
|
+
panic_level=_level(score),
|
|
356
|
+
error_clusters=ordered,
|
|
357
|
+
stuck_points=stuck,
|
|
358
|
+
transitions=transitions,
|
|
359
|
+
confusions=[e.text or "" for e in confusions],
|
|
360
|
+
viz_steps=len(viz),
|
|
361
|
+
viz_helped=viz_helped,
|
|
362
|
+
digest=_digest(events, t0, cfg),
|
|
363
|
+
)
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def compress(source: Union["Session", Iterable[Event], Iterable[Dict[str, Any]]],
|
|
367
|
+
config: Optional[CompressorConfig] = None) -> CompressedTrajectory:
|
|
368
|
+
"""Compress a Session (or a list of events / event dicts) into a CompressedTrajectory."""
|
|
369
|
+
cfg = config or CompressorConfig()
|
|
370
|
+
session_id, metadata, started = "adhoc", {}, None
|
|
371
|
+
if hasattr(source, "events"):
|
|
372
|
+
session_id = source.session_id
|
|
373
|
+
metadata = dict(source.metadata)
|
|
374
|
+
started = source.started_at
|
|
375
|
+
events = list(source.events)
|
|
376
|
+
else:
|
|
377
|
+
events = [e if isinstance(e, Event) else Event.from_dict(e) for e in source]
|
|
378
|
+
events.sort(key=lambda e: e.ts)
|
|
379
|
+
t0 = started if started is not None else (events[0].ts if events else 0.0)
|
|
380
|
+
|
|
381
|
+
by_task: Dict[str, List[Event]] = {}
|
|
382
|
+
for e in events:
|
|
383
|
+
if e.kind == TASK:
|
|
384
|
+
continue
|
|
385
|
+
by_task.setdefault(e.task or "default", []).append(e)
|
|
386
|
+
tasks = [_analyze_task(name, evs, t0, cfg) for name, evs in by_task.items() if evs]
|
|
387
|
+
|
|
388
|
+
runs = [e for e in events if e.kind == RUN]
|
|
389
|
+
all_errors = Counter(error_signature(e.error_type, e.error_msg) for e in runs if not e.ok)
|
|
390
|
+
totals = {
|
|
391
|
+
"tasks": len(tasks),
|
|
392
|
+
"runs": len(runs),
|
|
393
|
+
"failures": sum(1 for e in runs if not e.ok),
|
|
394
|
+
"solved": sum(1 for t in tasks if t.solved),
|
|
395
|
+
"confusions": sum(len(t.confusions) for t in tasks),
|
|
396
|
+
"viz_steps": sum(t.viz_steps for t in tasks),
|
|
397
|
+
"top_errors": all_errors.most_common(3),
|
|
398
|
+
}
|
|
399
|
+
duration = (events[-1].ts - t0) if events else 0.0
|
|
400
|
+
ct = CompressedTrajectory(session_id, metadata, round(duration, 1), totals, tasks)
|
|
401
|
+
raw_bytes = len(json.dumps([e.to_dict() for e in events], ensure_ascii=False).encode())
|
|
402
|
+
comp_bytes = len(ct.to_prompt().encode())
|
|
403
|
+
ct.stats = {"raw_events": len(events), "raw_bytes": raw_bytes, "compressed_bytes": comp_bytes,
|
|
404
|
+
"ratio": round(raw_bytes / comp_bytes, 1) if comp_bytes else None}
|
|
405
|
+
return ct
|
stucktrace/demo.py
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"""45-second end-to-end demo: a scripted student struggles, stucktrace explains it.
|
|
2
|
+
|
|
3
|
+
Run with ``python -m stucktrace demo`` (or ``stucktrace demo --lang ru``) or,
|
|
4
|
+
in a notebook, ``from stucktrace.demo import run_demo; run_demo()``.
|
|
5
|
+
|
|
6
|
+
The student's code attempts are REALLY executed (real exceptions are
|
|
7
|
+
captured); only the clock is simulated so that pauses of several minutes
|
|
8
|
+
fit into a short live demo.
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import time
|
|
13
|
+
from typing import Any, List, Optional, Tuple
|
|
14
|
+
|
|
15
|
+
from .adapters import MoodleAdapter, OPTMentorAdapter
|
|
16
|
+
from .reporter import generate_report
|
|
17
|
+
from .tracker import Session
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class SimClock:
|
|
21
|
+
def __init__(self, start: float = 1_700_000_000.0) -> None:
|
|
22
|
+
self.t0 = start
|
|
23
|
+
self.now = start
|
|
24
|
+
|
|
25
|
+
def __call__(self) -> float:
|
|
26
|
+
return self.now
|
|
27
|
+
|
|
28
|
+
def at(self, seconds: float) -> None:
|
|
29
|
+
self.now = self.t0 + seconds
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
DATA = 'data = "2 4 5 8 11 6".split()\n'
|
|
33
|
+
|
|
34
|
+
# (t_seconds, action, task, payload)
|
|
35
|
+
SCRIPT: List[Tuple[float, str, str, Any]] = [
|
|
36
|
+
(0, "task", "sum_evens", None),
|
|
37
|
+
(5, "run", "sum_evens", DATA + "total = 0\nfor n in dat:\n if n % 2 == 0:\n total += n\n"),
|
|
38
|
+
(12, "run", "sum_evens", DATA + "total = 0\nfor n in data:\n if n % 2 == 0:\n total += n\n"),
|
|
39
|
+
(17, "run", "sum_evens", DATA + "total = 0\nfor n in data:\n if (n % 2) == 0:\n total += n\n"),
|
|
40
|
+
(21, "run", "sum_evens", DATA + "total = 0\nfor n in data:\n if n % 2 == 0 :\n total = total + n\n"),
|
|
41
|
+
(170, "confusion", "sum_evens", "why does % not work on my numbers?"),
|
|
42
|
+
(185, "viz", "sum_evens", {"trace": [
|
|
43
|
+
{"event": "step_line", "line": 1, "func_name": "<module>", "stack_to_render": []},
|
|
44
|
+
{"event": "step_line", "line": 3, "func_name": "<module>", "stack_to_render": []},
|
|
45
|
+
{"event": "exception", "line": 4, "func_name": "<module>", "stack_to_render": [],
|
|
46
|
+
"exception_msg": "TypeError: not all arguments converted during string formatting"}]}),
|
|
47
|
+
(205, "run", "sum_evens", DATA + "total = 0\nfor n in data:\n if int(n) % 2 == 0:\n total += int(n)\n"),
|
|
48
|
+
(240, "task", "count_vowels", None),
|
|
49
|
+
(250, "run", "count_vowels", 's = "recursion"\ncount = 0\nfor i in range(len(s) + 1):\n if s[i] in "aeiou":\n count += 1\n'),
|
|
50
|
+
(262, "run", "count_vowels", 's = "recursion"\ncount = sum(\n 1 for ch in s if ch in "aeiou"\n'),
|
|
51
|
+
(281, "run", "count_vowels", 's = "recursion"\ncount = sum(1 for ch in s if ch in "aeiou")\n'),
|
|
52
|
+
(300, "task", "fizzbuzz", None),
|
|
53
|
+
(330, "run", "fizzbuzz", 'out = ["FizzBuzz" if i % 15 == 0 else "Fizz" if i % 3 == 0 else '
|
|
54
|
+
'"Buzz" if i % 5 == 0 else str(i) for i in range(1, 16)]\n'),
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def build_demo_session(verbose: bool = True) -> Session:
|
|
59
|
+
clock = SimClock()
|
|
60
|
+
s = Session(student_id="s1024", course="CS1302 Intro to Programming", clock=clock)
|
|
61
|
+
MoodleAdapter(course_id=1302, quiz_id=7, user_id="s1024", attempt=1).apply_to(s)
|
|
62
|
+
opt = OPTMentorAdapter()
|
|
63
|
+
for t, action, task, payload in SCRIPT:
|
|
64
|
+
clock.at(t)
|
|
65
|
+
if action == "task":
|
|
66
|
+
s.set_task(task)
|
|
67
|
+
if verbose:
|
|
68
|
+
print(f"\n▶ task: {task}")
|
|
69
|
+
elif action == "run":
|
|
70
|
+
ns: dict = {}
|
|
71
|
+
try:
|
|
72
|
+
with s.track(task, code=payload):
|
|
73
|
+
exec(compile(payload, f"<{task}>", "exec"), ns) # noqa: S102 - demo of student code
|
|
74
|
+
except Exception:
|
|
75
|
+
pass # the student's error is recorded by track(); the demo goes on
|
|
76
|
+
ev = s.events[-1]
|
|
77
|
+
if verbose:
|
|
78
|
+
mark = "✓ ok" if ev.ok else f"✗ {ev.error_type}: {ev.error_msg}"
|
|
79
|
+
print(f" +{int(t):>3}s run {mark}")
|
|
80
|
+
elif action == "confusion":
|
|
81
|
+
s.mark_confusion(payload, task=task)
|
|
82
|
+
if verbose:
|
|
83
|
+
print(f" +{int(t):>3}s 🆘 mark_confusion({payload!r})")
|
|
84
|
+
elif action == "viz":
|
|
85
|
+
opt.import_trace(payload, session=s, task=task)
|
|
86
|
+
if verbose:
|
|
87
|
+
print(f" +{int(t):>3}s 👁 OPT_Mentor trace imported")
|
|
88
|
+
return s
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def run_demo(lang: str = "en", backend: str = "offline", save: Optional[str] = "demo_session.jsonl",
|
|
92
|
+
pause: float = 0.0) -> Session:
|
|
93
|
+
"""Full cycle: errors -> compressed context -> student & instructor reports -> Moodle export."""
|
|
94
|
+
try:
|
|
95
|
+
from IPython import get_ipython # type: ignore
|
|
96
|
+
in_nb = get_ipython() is not None and "IPKernelApp" in get_ipython().config
|
|
97
|
+
except Exception:
|
|
98
|
+
in_nb = False
|
|
99
|
+
|
|
100
|
+
def show(md: str) -> None:
|
|
101
|
+
if in_nb:
|
|
102
|
+
from IPython.display import Markdown, display # type: ignore
|
|
103
|
+
display(Markdown(md))
|
|
104
|
+
else:
|
|
105
|
+
print(md)
|
|
106
|
+
|
|
107
|
+
def step(title: str) -> None:
|
|
108
|
+
print("\n" + "=" * 72 + f"\n{title}\n" + "=" * 72)
|
|
109
|
+
if pause:
|
|
110
|
+
time.sleep(pause)
|
|
111
|
+
|
|
112
|
+
step("1/4 Student works (real code, real exceptions, simulated clock)")
|
|
113
|
+
s = build_demo_session(verbose=True)
|
|
114
|
+
if save:
|
|
115
|
+
s.save(save)
|
|
116
|
+
|
|
117
|
+
step("2/4 Trajectory Compressor")
|
|
118
|
+
ctx = s.compress()
|
|
119
|
+
show(ctx.to_markdown())
|
|
120
|
+
|
|
121
|
+
step("3/4 Reports (Hermes if configured, otherwise offline engine)")
|
|
122
|
+
for mode in ("student", "instructor"):
|
|
123
|
+
rep = generate_report(mode, session=s, backend=backend, lang=lang)
|
|
124
|
+
show(rep._repr_markdown_() + "\n")
|
|
125
|
+
|
|
126
|
+
step("4/4 Moodle export (CSV rows per question)")
|
|
127
|
+
print(MoodleAdapter(course_id=1302, quiz_id=7, user_id="s1024", attempt=1).export_csv(ctx))
|
|
128
|
+
if save:
|
|
129
|
+
print(f"raw trajectory saved to {save} — try: stucktrace report {save} --mode instructor")
|
|
130
|
+
return s
|
stucktrace/events.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Event model: the raw, append-only trajectory record."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from dataclasses import asdict, dataclass, field, fields
|
|
5
|
+
from typing import Any, Dict, Optional
|
|
6
|
+
|
|
7
|
+
RUN = "run" # one execution attempt (cell / script / decorated function)
|
|
8
|
+
CONFUSION = "confusion" # explicit "I don't understand" marker from the student
|
|
9
|
+
VIZ = "viz" # a visualization / tracing step (OPT_Mentor, Python Tutor, ...)
|
|
10
|
+
TASK = "task" # the student switched to another task
|
|
11
|
+
NOTE = "note" # free-form note (instructor / tool)
|
|
12
|
+
|
|
13
|
+
KINDS = frozenset({RUN, CONFUSION, VIZ, TASK, NOTE})
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass
|
|
17
|
+
class Event:
|
|
18
|
+
kind: str
|
|
19
|
+
ts: float
|
|
20
|
+
task: Optional[str] = None
|
|
21
|
+
ok: Optional[bool] = None
|
|
22
|
+
error_type: Optional[str] = None
|
|
23
|
+
error_msg: Optional[str] = None
|
|
24
|
+
duration: Optional[float] = None
|
|
25
|
+
code_hash: Optional[str] = None
|
|
26
|
+
code_len: Optional[int] = None
|
|
27
|
+
edit_ratio: Optional[float] = None # 0.0 = identical to previous attempt, 1.0 = full rewrite
|
|
28
|
+
code_preview: Optional[str] = None
|
|
29
|
+
text: Optional[str] = None
|
|
30
|
+
data: Dict[str, Any] = field(default_factory=dict)
|
|
31
|
+
|
|
32
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
33
|
+
return {k: v for k, v in asdict(self).items() if v is not None and v != {} and v != ""}
|
|
34
|
+
|
|
35
|
+
@classmethod
|
|
36
|
+
def from_dict(cls, d: Dict[str, Any]) -> "Event":
|
|
37
|
+
known = {f.name for f in fields(cls)}
|
|
38
|
+
return cls(**{k: v for k, v in d.items() if k in known})
|
stucktrace/reporter.py
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""Report orchestration: compress -> Hermes (or offline fallback) -> spoiler guard -> pretty output."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import re
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from typing import Any, Optional
|
|
7
|
+
|
|
8
|
+
from .adapters import HermesClient, HermesError, LocalReportGenerator
|
|
9
|
+
from .compressor import CompressedTrajectory, CompressorConfig, compress
|
|
10
|
+
|
|
11
|
+
MODES = ("student", "instructor")
|
|
12
|
+
_CODE_BLOCK = re.compile(r"```[^\n]*\n(.*?)```", re.S)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def spoiler_guard(markdown: str, max_lines: int = 2, lang: str = "en") -> str:
|
|
16
|
+
"""Remove multi-line code blocks from student-facing text (LLMs love to 'just show the fix')."""
|
|
17
|
+
note = "_(код скрыт — попробуй сначала сам)_" if lang == "ru" else "_(code hidden — try it yourself first)_"
|
|
18
|
+
|
|
19
|
+
def repl(m: "re.Match[str]") -> str:
|
|
20
|
+
body = m.group(1).strip("\n")
|
|
21
|
+
return m.group(0) if body.count("\n") + 1 <= max_lines else note
|
|
22
|
+
return _CODE_BLOCK.sub(repl, markdown)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass
|
|
26
|
+
class Report:
|
|
27
|
+
mode: str
|
|
28
|
+
markdown: str
|
|
29
|
+
source: str # "hermes" | "offline"
|
|
30
|
+
context: CompressedTrajectory
|
|
31
|
+
warning: Optional[str] = None
|
|
32
|
+
|
|
33
|
+
def _footer(self) -> str:
|
|
34
|
+
src = "Hermes LLM" if self.source == "hermes" else "offline engine"
|
|
35
|
+
st = self.context.stats
|
|
36
|
+
f = (f"\n\n---\n<sub>stucktrace · {self.mode} report · generated by {src} · "
|
|
37
|
+
f"{st.get('raw_events')} raw events → {st.get('compressed_bytes')} B context "
|
|
38
|
+
f"({st.get('ratio')}× smaller)</sub>")
|
|
39
|
+
if self.warning:
|
|
40
|
+
f += f"\n<sub>⚠ {self.warning}</sub>"
|
|
41
|
+
return f
|
|
42
|
+
|
|
43
|
+
def _repr_markdown_(self) -> str: # auto-rendered in Jupyter
|
|
44
|
+
return self.markdown + self._footer()
|
|
45
|
+
|
|
46
|
+
def __str__(self) -> str:
|
|
47
|
+
return self.markdown
|
|
48
|
+
|
|
49
|
+
def save(self, path: str) -> str:
|
|
50
|
+
with open(path, "w", encoding="utf-8") as fh:
|
|
51
|
+
fh.write(self.markdown + self._footer() + "\n")
|
|
52
|
+
return path
|
|
53
|
+
|
|
54
|
+
def display(self) -> None:
|
|
55
|
+
try:
|
|
56
|
+
from IPython.display import Markdown, display # type: ignore
|
|
57
|
+
display(Markdown(self._repr_markdown_()))
|
|
58
|
+
except Exception:
|
|
59
|
+
print(self._repr_markdown_())
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def generate_report(
|
|
63
|
+
mode: str = "student",
|
|
64
|
+
session: Any = None,
|
|
65
|
+
backend: str = "auto",
|
|
66
|
+
lang: str = "en",
|
|
67
|
+
hermes: Optional[HermesClient] = None,
|
|
68
|
+
config: Optional[CompressorConfig] = None,
|
|
69
|
+
display: bool = False,
|
|
70
|
+
) -> Report:
|
|
71
|
+
"""Generate a Student Self-Reflection or Instructor Diagnostic report.
|
|
72
|
+
|
|
73
|
+
backend: ``"auto"`` (Hermes if configured, else offline), ``"hermes"``
|
|
74
|
+
(try Hermes, fall back with a warning) or ``"offline"``.
|
|
75
|
+
The function never raises because of the network.
|
|
76
|
+
"""
|
|
77
|
+
from .tracker import get_session
|
|
78
|
+
|
|
79
|
+
if mode not in MODES:
|
|
80
|
+
raise ValueError(f"mode must be one of {MODES}, got {mode!r}")
|
|
81
|
+
if isinstance(session, CompressedTrajectory):
|
|
82
|
+
ctx = session
|
|
83
|
+
else:
|
|
84
|
+
ctx = compress(session if session is not None else get_session(), config)
|
|
85
|
+
|
|
86
|
+
client = hermes or HermesClient()
|
|
87
|
+
text, source, warning = None, "offline", None
|
|
88
|
+
if backend in ("auto", "hermes") and (client.available or backend == "hermes"):
|
|
89
|
+
try:
|
|
90
|
+
text = client.report(ctx, mode=mode, lang=lang)
|
|
91
|
+
source = "hermes"
|
|
92
|
+
except HermesError as exc:
|
|
93
|
+
warning = f"Hermes unavailable, used offline report ({exc})"
|
|
94
|
+
except Exception as exc: # pragma: no cover - never let reporting crash a notebook
|
|
95
|
+
warning = f"Hermes error, used offline report ({exc!r})"
|
|
96
|
+
if text is None:
|
|
97
|
+
text = LocalReportGenerator().generate(ctx, mode=mode, lang=lang)
|
|
98
|
+
if mode == "student":
|
|
99
|
+
text = spoiler_guard(text, lang=lang)
|
|
100
|
+
|
|
101
|
+
rep = Report(mode=mode, markdown=text, source=source, context=ctx, warning=warning)
|
|
102
|
+
if display:
|
|
103
|
+
rep.display()
|
|
104
|
+
return rep
|