stucktrace 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stucktrace-0.1.0/LICENSE +21 -0
- stucktrace-0.1.0/PKG-INFO +305 -0
- stucktrace-0.1.0/README.md +274 -0
- stucktrace-0.1.0/pyproject.toml +51 -0
- stucktrace-0.1.0/setup.cfg +4 -0
- stucktrace-0.1.0/src/stucktrace/__init__.py +50 -0
- stucktrace-0.1.0/src/stucktrace/__main__.py +3 -0
- stucktrace-0.1.0/src/stucktrace/_version.py +1 -0
- stucktrace-0.1.0/src/stucktrace/adapters.py +501 -0
- stucktrace-0.1.0/src/stucktrace/cli.py +64 -0
- stucktrace-0.1.0/src/stucktrace/compressor.py +405 -0
- stucktrace-0.1.0/src/stucktrace/demo.py +130 -0
- stucktrace-0.1.0/src/stucktrace/events.py +38 -0
- stucktrace-0.1.0/src/stucktrace/reporter.py +104 -0
- stucktrace-0.1.0/src/stucktrace/tracker.py +440 -0
- stucktrace-0.1.0/src/stucktrace.egg-info/PKG-INFO +305 -0
- stucktrace-0.1.0/src/stucktrace.egg-info/SOURCES.txt +23 -0
- stucktrace-0.1.0/src/stucktrace.egg-info/dependency_links.txt +1 -0
- stucktrace-0.1.0/src/stucktrace.egg-info/entry_points.txt +2 -0
- stucktrace-0.1.0/src/stucktrace.egg-info/requires.txt +9 -0
- stucktrace-0.1.0/src/stucktrace.egg-info/top_level.txt +1 -0
- stucktrace-0.1.0/tests/test_compressor.py +113 -0
- stucktrace-0.1.0/tests/test_jupyter_and_cli.py +56 -0
- stucktrace-0.1.0/tests/test_reporter_adapters.py +115 -0
- stucktrace-0.1.0/tests/test_tracker.py +112 -0
stucktrace-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 tetroi
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: stucktrace
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Learning-trajectory telemetry & reflection reports for Jupyter: detects where students get stuck (Moodle + OPT_Mentor + Hermes).
|
|
5
|
+
Author: Ulykbek, Alisher, Azat, Alisher, Insar
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/ulykWyou/stucktrace
|
|
8
|
+
Project-URL: Repository, https://github.com/ulykWyou/stucktrace
|
|
9
|
+
Project-URL: Issues, https://github.com/ulykWyou/stucktrace/issues
|
|
10
|
+
Keywords: education,jupyter,jupyterhub,telemetry,learning-analytics,moodle,llm,reflection
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Framework :: Jupyter
|
|
13
|
+
Classifier: Framework :: IPython
|
|
14
|
+
Classifier: Intended Audience :: Education
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
19
|
+
Classifier: Topic :: Education
|
|
20
|
+
Requires-Python: >=3.8
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
Provides-Extra: jupyter
|
|
24
|
+
Requires-Dist: ipython>=7.0; extra == "jupyter"
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
27
|
+
Requires-Dist: ipython>=7.0; extra == "dev"
|
|
28
|
+
Requires-Dist: build; extra == "dev"
|
|
29
|
+
Requires-Dist: twine; extra == "dev"
|
|
30
|
+
Dynamic: license-file
|
|
31
|
+
|
|
32
|
+
# stucktrace
|
|
33
|
+
|
|
34
|
+
**See where students get stuck, not just whether they passed.**
|
|
35
|
+
|
|
36
|
+
`stucktrace` is a zero-dependency Python library for Jupyter/JupyterHub that records *how* a student works through programming tasks (runs, errors, edits, pauses, visualizer use, explicit "I'm lost" markers), compresses that raw trajectory into a small structured context, and turns it into two reports:
|
|
37
|
+
|
|
38
|
+
- **Student Self-Reflection Report**: where you got stuck, which error patterns you repeated, what worked, what to try next. It never gives away the solution.
|
|
39
|
+
- **Instructor Diagnostic Report**: a per-task difficulty table, timestamped panic/stuck points, error clusters, error→fix transitions and suggested interventions.
|
|
40
|
+
|
|
41
|
+
It plugs into the **Moodle + OPT_Mentor + Hermes** stack: Moodle attempt metadata in and CSV/JSON out, OPT_Mentor trace import, and Hermes LLM reports. A built-in **offline report engine** takes over whenever Hermes is unreachable.
|
|
42
|
+
|
|
43
|
+
```python
|
|
44
|
+
%load_ext stucktrace # every cell run is now recorded
|
|
45
|
+
%stucktrace task sum_evens
|
|
46
|
+
# ... student works, fails, retries ...
|
|
47
|
+
%confused why does % not work on my numbers
|
|
48
|
+
%stucktrace report student
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
---
|
|
52
|
+
|
|
53
|
+
## Motivation
|
|
54
|
+
|
|
55
|
+
A Moodle grade tells you the outcome of an attempt. Two students with the same 7/10 can have very different sessions:
|
|
56
|
+
|
|
57
|
+
- one solved everything steadily and lost points on an edge case;
|
|
58
|
+
- the other spent 6 minutes firing the same `TypeError` at the interpreter, went silent, rewrote the whole cell and got lucky.
|
|
59
|
+
|
|
60
|
+
Instructors can't watch 200 notebooks during a lab, and students rarely remember *how* they got stuck, only that they did. The process data already exists inside the kernel (every run, every traceback) but nobody keeps it. When someone does keep it, it's a raw log that is too long for a person to read and too noisy to send to an LLM.
|
|
61
|
+
|
|
62
|
+
`stucktrace` keeps the process data, compresses it, and explains it back to both sides:
|
|
63
|
+
|
|
64
|
+
| Who | Question | What stucktrace gives |
|
|
65
|
+
|---|---|---|
|
|
66
|
+
| Student | "Why did that take me so long?" | A no-spoiler reflection report: stuck points, repeated patterns, habits to change |
|
|
67
|
+
| Instructor | "Which task and which concept broke the class?" | Ranked hotspots, error clusters, panic timestamps, interventions, Moodle CSV |
|
|
68
|
+
| LLM (Hermes) | "What happened in this session?" | ~1–6 KB structured context instead of hundreds of raw events |
|
|
69
|
+
|
|
70
|
+
## Innovation
|
|
71
|
+
|
|
72
|
+
**1. Trajectory Compressor** (`compressor.py`). This is the core of the package, not just a logger:
|
|
73
|
+
|
|
74
|
+
- **Error-signature clustering.** Messages are normalized (identifiers, numbers, addresses stripped), so `name 'x' is not defined` and `name 'total' is not defined` count as *one* recurring misconception.
|
|
75
|
+
- **Five stuck/panic signals per task:**
|
|
76
|
+
| Signal | Default rule |
|
|
77
|
+
|---|---|
|
|
78
|
+
| `rapid_fail_burst` | ≥3 failed runs, each ≤25 s after the previous one (trial-and-error mode) |
|
|
79
|
+
| `repeated_error` | same error signature ≥3 times in a row |
|
|
80
|
+
| `long_pause` | ≥120 s of silence right after a failure |
|
|
81
|
+
| `rewrite` | ≥40 % of the code changed right after a failure (thrashing) |
|
|
82
|
+
| `confusion` | explicit `mark_confusion("I don't get the loop")` / `%confused` |
|
|
83
|
+
- **Panic score** in [0, 1]: a weighted combination of the signals (`1 - exp(-Σw/2)`) mapped to low / medium / high.
|
|
84
|
+
- **Key transitions.** Every failure→success step: which error was fixed, after how many failed attempts, how big the fixing edit was, and whether a visualizer step came first. This measures *whether visualization actually helped*.
|
|
85
|
+
- **Run-length digest.** The whole task in one line:
|
|
86
|
+
`F:NameError | F:TypeError x3 | P:149s | ?"why does % not work" | V | S`
|
|
87
|
+
- **Measured compression.** On a simulated 210-run lab session, 95 KB of raw events become a 6 KB context (**15.6× smaller**). The demo session (14 events) compresses about 1.6×; the gain grows with session length.
|
|
88
|
+
|
|
89
|
+
**2. Fail-safe by design.** Every telemetry entry point swallows its own errors. Student exceptions are recorded and **re-raised untouched**. No network call can block or crash a notebook, and Hermes failures fall back to the offline engine with a visible warning.
|
|
90
|
+
|
|
91
|
+
**3. Spoiler guard.** LLMs like to "just show the fix", so multi-line code blocks are removed from student-facing reports automatically. The Hermes prompt also forbids solutions.
|
|
92
|
+
|
|
93
|
+
**4. Privacy knobs.** `store_code=False` keeps only hashes and lengths, never code. Raw OPT_Mentor traces are summarized into one event instead of being stored step by step.
|
|
94
|
+
|
|
95
|
+
## Architecture
|
|
96
|
+
|
|
97
|
+
```
|
|
98
|
+
Jupyter cell hooks / @track / with track() mark_confusion · log_viz
|
|
99
|
+
│ │
|
|
100
|
+
▼ ▼
|
|
101
|
+
┌──────────────────────────────────────────────────────────────────────────┐
|
|
102
|
+
│ tracker.py Session: append-only Event log (run/confusion/viz/task) │
|
|
103
|
+
│ edit_ratio via difflib · autosave JSONL · ring buffer │
|
|
104
|
+
└───────────────────────────────┬──────────────────────────────────────────┘
|
|
105
|
+
▼
|
|
106
|
+
┌──────────────────────────────────────────────────────────────────────────┐
|
|
107
|
+
│ compressor.py error signatures → clusters → stuck signals → panic score │
|
|
108
|
+
│ transitions · viz effect · run-length digest │
|
|
109
|
+
│ → CompressedTrajectory (.to_json / .to_markdown / .to_prompt)
|
|
110
|
+
└───────────────┬───────────────────────────────┬──────────────────────────┘
|
|
111
|
+
▼ ▼
|
|
112
|
+
┌───────────────────────────────┐ ┌──────────────────────────────────────┐
|
|
113
|
+
│ adapters.py │ │ reporter.py │
|
|
114
|
+
│ MoodleAdapter (in: attempt │ │ generate_report(mode, backend, lang)│
|
|
115
|
+
│ meta, out: CSV / JSON) │──▶│ Hermes ──fail──▶ LocalReportGenerator
|
|
116
|
+
│ OPTMentorAdapter (trace in) │ │ spoiler_guard (student mode) │
|
|
117
|
+
│ HermesClient (LLM) │ │ Report → Jupyter Markdown / .md │
|
|
118
|
+
│ LocalReportGenerator (EN/RU) │ └──────────────────────────────────────┘
|
|
119
|
+
└───────────────────────────────┘
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
| Module | Responsibility |
|
|
123
|
+
|---|---|
|
|
124
|
+
| `events.py` | `Event` dataclass, the raw record format (JSONL-serializable) |
|
|
125
|
+
| `tracker.py` | `Session`, global API, IPython hooks and magics, `track` decorator/context manager |
|
|
126
|
+
| `compressor.py` | Trajectory Compressor (the innovation) |
|
|
127
|
+
| `adapters.py` | Moodle / OPT_Mentor / Hermes integrations and the offline report engine |
|
|
128
|
+
| `reporter.py` | Orchestration, fallback, spoiler guard, `Report` rendering |
|
|
129
|
+
| `cli.py`, `demo.py` | `stucktrace demo / report / compress / moodle` |
|
|
130
|
+
|
|
131
|
+
More detail: [`docs/architecture.md`](docs/architecture.md).
|
|
132
|
+
|
|
133
|
+
## Installation & Setup
|
|
134
|
+
|
|
135
|
+
```bash
|
|
136
|
+
pip install stucktrace # zero dependencies
|
|
137
|
+
pip install "stucktrace[jupyter]" # + IPython (already present on any JupyterHub)
|
|
138
|
+
pip install "stucktrace[dev]" # + pytest, build, twine
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
Requires Python ≥ 3.8 and uses only the standard library, so it can't conflict with what's already on a JupyterHub image.
|
|
142
|
+
|
|
143
|
+
**Hermes (optional).** Set environment variables, or pass a `HermesClient(...)`:
|
|
144
|
+
|
|
145
|
+
```bash
|
|
146
|
+
export HERMES_API_URL="https://<hermes-host>/v1/chat/completions" # OpenAI-compatible endpoint
|
|
147
|
+
export HERMES_API_KEY="..."
|
|
148
|
+
export HERMES_MODEL="hermes" # optional
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
If `HERMES_API_URL` is not set, reports are generated offline with no warning. If it is set but unreachable, you still get an offline report, marked with a warning.
|
|
152
|
+
|
|
153
|
+
## Quickstart
|
|
154
|
+
|
|
155
|
+
### JupyterHub
|
|
156
|
+
|
|
157
|
+
```python
|
|
158
|
+
import stucktrace as st
|
|
159
|
+
%load_ext stucktrace # or: st.start_session(student_id="s123", course="CS1302")
|
|
160
|
+
%stucktrace task loops # name the current task
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
Then just work. Every cell run is recorded with its result, error type, duration and edit size. Cells that only drive stucktrace (magics, `st.*` calls) are ignored.
|
|
164
|
+
|
|
165
|
+
```python
|
|
166
|
+
%confused I don't understand why the loop stops early # (avoid a trailing "?": IPython reads it as help)
|
|
167
|
+
st.log_viz(step=4, line=7, note="i == len(xs)") # manual visualization step
|
|
168
|
+
%stucktrace status
|
|
169
|
+
%stucktrace report student # renders Markdown in the notebook
|
|
170
|
+
%stucktrace report instructor --lang ru
|
|
171
|
+
%stucktrace save my_session.jsonl
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
### Plain Python scripts
|
|
175
|
+
|
|
176
|
+
```python
|
|
177
|
+
import stucktrace as st
|
|
178
|
+
st.start_session(student_id="s42", jupyter=False)
|
|
179
|
+
|
|
180
|
+
@st.track("average") # each call = one attempt
|
|
181
|
+
def average(xs):
|
|
182
|
+
return sum(xs) / len(xs)
|
|
183
|
+
|
|
184
|
+
try:
|
|
185
|
+
with st.track("parse", code="int('4.5')"): # context-manager form
|
|
186
|
+
int("4.5")
|
|
187
|
+
except ValueError:
|
|
188
|
+
st.mark_confusion("why can't int() read 4.5")
|
|
189
|
+
|
|
190
|
+
print(st.generate_report("instructor"))
|
|
191
|
+
st.get_session().save("session.jsonl")
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
### CLI
|
|
195
|
+
|
|
196
|
+
```bash
|
|
197
|
+
stucktrace demo # 45-second end-to-end demo (add --lang ru, --pause 2)
|
|
198
|
+
stucktrace report session.jsonl --mode instructor -o report.md
|
|
199
|
+
stucktrace compress session.jsonl --format prompt
|
|
200
|
+
stucktrace moodle session.jsonl --format csv
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
## Example Use Cases
|
|
204
|
+
|
|
205
|
+
1. **Lab session triage.** The TA collects `session.jsonl` files at the end of a lab and runs `stucktrace moodle` on each. Sorting by `panic_score` shows which students need a follow-up and which task broke the class.
|
|
206
|
+
2. **Exam post-mortem.** The instructor report marks persistent clusters (e.g. `TypeError` on `str % int` in 40 % of sessions), which points to a misconception to re-teach, not just a hard question.
|
|
207
|
+
3. **Student self-study.** A student ends a practice notebook with `%stucktrace report student` and gets habits to change ("change one thing at a time") without being given the answer.
|
|
208
|
+
4. **Does the visualizer help?** `viz_helped` and `viz_before` on transitions show whether OPT_Mentor tracing actually shortens time-to-fix.
|
|
209
|
+
5. **Moodle integration.** `MoodleAdapter.from_attempt(lti_params)` tags the session with the course/quiz/user IDs, and `export_attempt_json` / `export_csv` produce per-question rows to import.
|
|
210
|
+
|
|
211
|
+
```python
|
|
212
|
+
from stucktrace import MoodleAdapter, OPTMentorAdapter
|
|
213
|
+
MoodleAdapter.from_attempt({"courseid": 1302, "quizid": 7, "userid": "s1024", "attempt": 1},
|
|
214
|
+
question_map={"sum_evens": 1, "fizzbuzz": 2}).apply_to(st.get_session())
|
|
215
|
+
OPTMentorAdapter().import_trace(trace_json, task="sum_evens") # Python-Tutor-style trace
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
## Example reports
|
|
219
|
+
|
|
220
|
+
Output of `stucktrace demo` (offline engine). Full EN/RU versions and the exact LLM context are in [`docs/report_examples.md`](docs/report_examples.md).
|
|
221
|
+
|
|
222
|
+
<details open><summary><b>Student Self-Reflection Report</b></summary>
|
|
223
|
+
|
|
224
|
+
```markdown
|
|
225
|
+
## 🪞 Self-Reflection Report
|
|
226
|
+
**Session:** 3 task(s) · 9 runs · 6 failed · 5m30s
|
|
227
|
+
|
|
228
|
+
### Where you got stuck
|
|
229
|
+
- **sum_evens** — high pressure: 4 failed runs in 16s; same TypeError 3× in a row;
|
|
230
|
+
149s of silence after TypeError; you flagged: "why does % not work on my numbers?"
|
|
231
|
+
- **count_vowels** — low pressure: 44% of the code rewritten after IndexError
|
|
232
|
+
|
|
233
|
+
### Patterns you repeated
|
|
234
|
+
- `TypeError` ×3 (sum_evens) — two values of incompatible types meet — print the type()
|
|
235
|
+
of each operand before the failing line
|
|
236
|
+
|
|
237
|
+
### What worked
|
|
238
|
+
- On **sum_evens** you fixed `TypeError` after 4 failed attempt(s) with a small edit.
|
|
239
|
+
- On **sum_evens** the visualizer helped: success came within a few runs after tracing.
|
|
240
|
+
- **fizzbuzz** worked on the first try.
|
|
241
|
+
|
|
242
|
+
### Try next time
|
|
243
|
+
1. When a run fails, stop for 30 seconds and read the *whole* traceback before re-running.
|
|
244
|
+
2. If the same error comes back, write down (or say out loud) what you *expect* that line to do.
|
|
245
|
+
3. Change one thing at a time — big rewrites after an error hide what actually fixed it.
|
|
246
|
+
```
|
|
247
|
+
</details>
|
|
248
|
+
|
|
249
|
+
<details><summary><b>Instructor Diagnostic Report</b></summary>
|
|
250
|
+
|
|
251
|
+
```markdown
|
|
252
|
+
## 🧭 Instructor Diagnostic Report
|
|
253
|
+
**Student:** s1024 · **Course:** CS1302 Intro to Programming · 5m30s
|
|
254
|
+
|
|
255
|
+
| Task | Runs | Solved | 1st success | Time | Panic | Top error |
|
|
256
|
+
|---|---|---|---|---|---|---|
|
|
257
|
+
| sum_evens | 5 | ✅ | 5 | 3m20s | high (0.81) | TypeError: not all arguments converted… |
|
|
258
|
+
| count_vowels | 3 | ✅ | 3 | 31s | low (0.22) | IndexError: string index out of range |
|
|
259
|
+
| fizzbuzz | 1 | ✅ | 1 | 0s | low (0.00) | — |
|
|
260
|
+
|
|
261
|
+
### Difficulty hotspots (ranked by panic score)
|
|
262
|
+
- **sum_evens** — high (0.81); visualizer used 1× (helped: yes)
|
|
263
|
+
- `+00:05` rapid-fail burst: 4 failed runs in 16s
|
|
264
|
+
- `+00:12` repeated error: same TypeError 3× in a row
|
|
265
|
+
- `+00:21` long pause: 149s of silence after TypeError
|
|
266
|
+
- `+02:50` confusion marker: "why does % not work on my numbers?"
|
|
267
|
+
- digest: `F:NameError | F:TypeError x3 | P:149s | ?"why does % not work on m" | V | S`
|
|
268
|
+
|
|
269
|
+
### Suggested interventions
|
|
270
|
+
1. **sum_evens**: persistent `TypeError` — likely misconception; a 2-minute targeted explanation is worth it.
|
|
271
|
+
2. **sum_evens**: the student explicitly asked for help — follow up personally.
|
|
272
|
+
3. Visualization shortened time-to-fix on **sum_evens** — recommend it for similar exercises.
|
|
273
|
+
```
|
|
274
|
+
</details>
|
|
275
|
+
|
|
276
|
+
## Configuration
|
|
277
|
+
|
|
278
|
+
```python
|
|
279
|
+
from stucktrace import CompressorConfig, generate_report
|
|
280
|
+
cfg = CompressorConfig(panic_gap=25, panic_min_burst=3, repeat_streak=3, stall_gap=120, rewrite_ratio=0.4)
|
|
281
|
+
generate_report("instructor", config=cfg, backend="auto", lang="en") # backend: auto | hermes | offline
|
|
282
|
+
st.start_session(store_code=False, autosave="live.jsonl") # privacy / crash-safe log
|
|
283
|
+
```
|
|
284
|
+
|
|
285
|
+
## Development
|
|
286
|
+
|
|
287
|
+
```bash
|
|
288
|
+
python -m venv .venv && source .venv/bin/activate
|
|
289
|
+
pip install -e ".[dev]"
|
|
290
|
+
pytest -q # 37 tests, incl. real IPython shell + fake Hermes HTTP server
|
|
291
|
+
stucktrace demo
|
|
292
|
+
```
|
|
293
|
+
|
|
294
|
+
Team workflow: [`CONTRIBUTING.md`](CONTRIBUTING.md).
|
|
295
|
+
|
|
296
|
+
## Status & limitations (v0.1.0)
|
|
297
|
+
|
|
298
|
+
- The Hermes client assumes an OpenAI-compatible `chat/completions` API; adapt `HermesClient.complete` if your deployment differs.
|
|
299
|
+
- Moodle field names follow `mod_quiz` / LTI conventions but aren't tied to a specific Moodle web-service call yet.
|
|
300
|
+
- Panic thresholds are heuristics, chosen for introductory programming labs; tune them with `CompressorConfig`.
|
|
301
|
+
- Tracking is per kernel. Collecting sessions from many students is left to the deployment (shared folder, Moodle upload).
|
|
302
|
+
|
|
303
|
+
## License
|
|
304
|
+
|
|
305
|
+
MIT
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
# stucktrace
|
|
2
|
+
|
|
3
|
+
**See where students get stuck, not just whether they passed.**
|
|
4
|
+
|
|
5
|
+
`stucktrace` is a zero-dependency Python library for Jupyter/JupyterHub that records *how* a student works through programming tasks (runs, errors, edits, pauses, visualizer use, explicit "I'm lost" markers), compresses that raw trajectory into a small structured context, and turns it into two reports:
|
|
6
|
+
|
|
7
|
+
- **Student Self-Reflection Report**: where you got stuck, which error patterns you repeated, what worked, what to try next. It never gives away the solution.
|
|
8
|
+
- **Instructor Diagnostic Report**: a per-task difficulty table, timestamped panic/stuck points, error clusters, error→fix transitions and suggested interventions.
|
|
9
|
+
|
|
10
|
+
It plugs into the **Moodle + OPT_Mentor + Hermes** stack: Moodle attempt metadata in and CSV/JSON out, OPT_Mentor trace import, and Hermes LLM reports. A built-in **offline report engine** takes over whenever Hermes is unreachable.
|
|
11
|
+
|
|
12
|
+
```python
|
|
13
|
+
%load_ext stucktrace # every cell run is now recorded
|
|
14
|
+
%stucktrace task sum_evens
|
|
15
|
+
# ... student works, fails, retries ...
|
|
16
|
+
%confused why does % not work on my numbers
|
|
17
|
+
%stucktrace report student
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
---
|
|
21
|
+
|
|
22
|
+
## Motivation
|
|
23
|
+
|
|
24
|
+
A Moodle grade tells you the outcome of an attempt. Two students with the same 7/10 can have very different sessions:
|
|
25
|
+
|
|
26
|
+
- one solved everything steadily and lost points on an edge case;
|
|
27
|
+
- the other spent 6 minutes firing the same `TypeError` at the interpreter, went silent, rewrote the whole cell and got lucky.
|
|
28
|
+
|
|
29
|
+
Instructors can't watch 200 notebooks during a lab, and students rarely remember *how* they got stuck, only that they did. The process data already exists inside the kernel (every run, every traceback) but nobody keeps it. When someone does keep it, it's a raw log that is too long for a person to read and too noisy to send to an LLM.
|
|
30
|
+
|
|
31
|
+
`stucktrace` keeps the process data, compresses it, and explains it back to both sides:
|
|
32
|
+
|
|
33
|
+
| Who | Question | What stucktrace gives |
|
|
34
|
+
|---|---|---|
|
|
35
|
+
| Student | "Why did that take me so long?" | A no-spoiler reflection report: stuck points, repeated patterns, habits to change |
|
|
36
|
+
| Instructor | "Which task and which concept broke the class?" | Ranked hotspots, error clusters, panic timestamps, interventions, Moodle CSV |
|
|
37
|
+
| LLM (Hermes) | "What happened in this session?" | ~1–6 KB structured context instead of hundreds of raw events |
|
|
38
|
+
|
|
39
|
+
## Innovation
|
|
40
|
+
|
|
41
|
+
**1. Trajectory Compressor** (`compressor.py`). This is the core of the package, not just a logger:
|
|
42
|
+
|
|
43
|
+
- **Error-signature clustering.** Messages are normalized (identifiers, numbers, addresses stripped), so `name 'x' is not defined` and `name 'total' is not defined` count as *one* recurring misconception.
|
|
44
|
+
- **Five stuck/panic signals per task:**
|
|
45
|
+
| Signal | Default rule |
|
|
46
|
+
|---|---|
|
|
47
|
+
| `rapid_fail_burst` | ≥3 failed runs, each ≤25 s after the previous one (trial-and-error mode) |
|
|
48
|
+
| `repeated_error` | same error signature ≥3 times in a row |
|
|
49
|
+
| `long_pause` | ≥120 s of silence right after a failure |
|
|
50
|
+
| `rewrite` | ≥40 % of the code changed right after a failure (thrashing) |
|
|
51
|
+
| `confusion` | explicit `mark_confusion("I don't get the loop")` / `%confused` |
|
|
52
|
+
- **Panic score** in [0, 1]: a weighted combination of the signals (`1 - exp(-Σw/2)`) mapped to low / medium / high.
|
|
53
|
+
- **Key transitions.** Every failure→success step: which error was fixed, after how many failed attempts, how big the fixing edit was, and whether a visualizer step came first. This measures *whether visualization actually helped*.
|
|
54
|
+
- **Run-length digest.** The whole task in one line:
|
|
55
|
+
`F:NameError | F:TypeError x3 | P:149s | ?"why does % not work" | V | S`
|
|
56
|
+
- **Measured compression.** On a simulated 210-run lab session, 95 KB of raw events become a 6 KB context (**15.6× smaller**). The demo session (14 events) compresses about 1.6×; the gain grows with session length.
|
|
57
|
+
|
|
58
|
+
**2. Fail-safe by design.** Every telemetry entry point swallows its own errors. Student exceptions are recorded and **re-raised untouched**. No network call can block or crash a notebook, and Hermes failures fall back to the offline engine with a visible warning.
|
|
59
|
+
|
|
60
|
+
**3. Spoiler guard.** LLMs like to "just show the fix", so multi-line code blocks are removed from student-facing reports automatically. The Hermes prompt also forbids solutions.
|
|
61
|
+
|
|
62
|
+
**4. Privacy knobs.** `store_code=False` keeps only hashes and lengths, never code. Raw OPT_Mentor traces are summarized into one event instead of being stored step by step.
|
|
63
|
+
|
|
64
|
+
## Architecture
|
|
65
|
+
|
|
66
|
+
```
|
|
67
|
+
Jupyter cell hooks / @track / with track() mark_confusion · log_viz
|
|
68
|
+
│ │
|
|
69
|
+
▼ ▼
|
|
70
|
+
┌──────────────────────────────────────────────────────────────────────────┐
|
|
71
|
+
│ tracker.py Session: append-only Event log (run/confusion/viz/task) │
|
|
72
|
+
│ edit_ratio via difflib · autosave JSONL · ring buffer │
|
|
73
|
+
└───────────────────────────────┬──────────────────────────────────────────┘
|
|
74
|
+
▼
|
|
75
|
+
┌──────────────────────────────────────────────────────────────────────────┐
|
|
76
|
+
│ compressor.py error signatures → clusters → stuck signals → panic score │
|
|
77
|
+
│ transitions · viz effect · run-length digest │
|
|
78
|
+
│ → CompressedTrajectory (.to_json / .to_markdown / .to_prompt)
|
|
79
|
+
└───────────────┬───────────────────────────────┬──────────────────────────┘
|
|
80
|
+
▼ ▼
|
|
81
|
+
┌───────────────────────────────┐ ┌──────────────────────────────────────┐
|
|
82
|
+
│ adapters.py │ │ reporter.py │
|
|
83
|
+
│ MoodleAdapter (in: attempt │ │ generate_report(mode, backend, lang)│
|
|
84
|
+
│ meta, out: CSV / JSON) │──▶│ Hermes ──fail──▶ LocalReportGenerator
|
|
85
|
+
│ OPTMentorAdapter (trace in) │ │ spoiler_guard (student mode) │
|
|
86
|
+
│ HermesClient (LLM) │ │ Report → Jupyter Markdown / .md │
|
|
87
|
+
│ LocalReportGenerator (EN/RU) │ └──────────────────────────────────────┘
|
|
88
|
+
└───────────────────────────────┘
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
| Module | Responsibility |
|
|
92
|
+
|---|---|
|
|
93
|
+
| `events.py` | `Event` dataclass, the raw record format (JSONL-serializable) |
|
|
94
|
+
| `tracker.py` | `Session`, global API, IPython hooks and magics, `track` decorator/context manager |
|
|
95
|
+
| `compressor.py` | Trajectory Compressor (the innovation) |
|
|
96
|
+
| `adapters.py` | Moodle / OPT_Mentor / Hermes integrations and the offline report engine |
|
|
97
|
+
| `reporter.py` | Orchestration, fallback, spoiler guard, `Report` rendering |
|
|
98
|
+
| `cli.py`, `demo.py` | `stucktrace demo / report / compress / moodle` |
|
|
99
|
+
|
|
100
|
+
More detail: [`docs/architecture.md`](docs/architecture.md).
|
|
101
|
+
|
|
102
|
+
## Installation & Setup
|
|
103
|
+
|
|
104
|
+
```bash
|
|
105
|
+
pip install stucktrace # zero dependencies
|
|
106
|
+
pip install "stucktrace[jupyter]" # + IPython (already present on any JupyterHub)
|
|
107
|
+
pip install "stucktrace[dev]" # + pytest, build, twine
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
Requires Python ≥ 3.8 and uses only the standard library, so it can't conflict with what's already on a JupyterHub image.
|
|
111
|
+
|
|
112
|
+
**Hermes (optional).** Set environment variables, or pass a `HermesClient(...)`:
|
|
113
|
+
|
|
114
|
+
```bash
|
|
115
|
+
export HERMES_API_URL="https://<hermes-host>/v1/chat/completions" # OpenAI-compatible endpoint
|
|
116
|
+
export HERMES_API_KEY="..."
|
|
117
|
+
export HERMES_MODEL="hermes" # optional
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
If `HERMES_API_URL` is not set, reports are generated offline with no warning. If it is set but unreachable, you still get an offline report, marked with a warning.
|
|
121
|
+
|
|
122
|
+
## Quickstart
|
|
123
|
+
|
|
124
|
+
### JupyterHub
|
|
125
|
+
|
|
126
|
+
```python
|
|
127
|
+
import stucktrace as st
|
|
128
|
+
%load_ext stucktrace # or: st.start_session(student_id="s123", course="CS1302")
|
|
129
|
+
%stucktrace task loops # name the current task
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
Then just work. Every cell run is recorded with its result, error type, duration and edit size. Cells that only drive stucktrace (magics, `st.*` calls) are ignored.
|
|
133
|
+
|
|
134
|
+
```python
|
|
135
|
+
%confused I don't understand why the loop stops early # (avoid a trailing "?": IPython reads it as help)
|
|
136
|
+
st.log_viz(step=4, line=7, note="i == len(xs)") # manual visualization step
|
|
137
|
+
%stucktrace status
|
|
138
|
+
%stucktrace report student # renders Markdown in the notebook
|
|
139
|
+
%stucktrace report instructor --lang ru
|
|
140
|
+
%stucktrace save my_session.jsonl
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
### Plain Python scripts
|
|
144
|
+
|
|
145
|
+
```python
|
|
146
|
+
import stucktrace as st
|
|
147
|
+
st.start_session(student_id="s42", jupyter=False)
|
|
148
|
+
|
|
149
|
+
@st.track("average") # each call = one attempt
|
|
150
|
+
def average(xs):
|
|
151
|
+
return sum(xs) / len(xs)
|
|
152
|
+
|
|
153
|
+
try:
|
|
154
|
+
with st.track("parse", code="int('4.5')"): # context-manager form
|
|
155
|
+
int("4.5")
|
|
156
|
+
except ValueError:
|
|
157
|
+
st.mark_confusion("why can't int() read 4.5")
|
|
158
|
+
|
|
159
|
+
print(st.generate_report("instructor"))
|
|
160
|
+
st.get_session().save("session.jsonl")
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
### CLI
|
|
164
|
+
|
|
165
|
+
```bash
|
|
166
|
+
stucktrace demo # 45-second end-to-end demo (add --lang ru, --pause 2)
|
|
167
|
+
stucktrace report session.jsonl --mode instructor -o report.md
|
|
168
|
+
stucktrace compress session.jsonl --format prompt
|
|
169
|
+
stucktrace moodle session.jsonl --format csv
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
## Example Use Cases
|
|
173
|
+
|
|
174
|
+
1. **Lab session triage.** The TA collects `session.jsonl` files at the end of a lab and runs `stucktrace moodle` on each. Sorting by `panic_score` shows which students need a follow-up and which task broke the class.
|
|
175
|
+
2. **Exam post-mortem.** The instructor report marks persistent clusters (e.g. `TypeError` on `str % int` in 40 % of sessions), which points to a misconception to re-teach, not just a hard question.
|
|
176
|
+
3. **Student self-study.** A student ends a practice notebook with `%stucktrace report student` and gets habits to change ("change one thing at a time") without being given the answer.
|
|
177
|
+
4. **Does the visualizer help?** `viz_helped` and `viz_before` on transitions show whether OPT_Mentor tracing actually shortens time-to-fix.
|
|
178
|
+
5. **Moodle integration.** `MoodleAdapter.from_attempt(lti_params)` tags the session with the course/quiz/user IDs, and `export_attempt_json` / `export_csv` produce per-question rows to import.
|
|
179
|
+
|
|
180
|
+
```python
|
|
181
|
+
from stucktrace import MoodleAdapter, OPTMentorAdapter
|
|
182
|
+
MoodleAdapter.from_attempt({"courseid": 1302, "quizid": 7, "userid": "s1024", "attempt": 1},
|
|
183
|
+
question_map={"sum_evens": 1, "fizzbuzz": 2}).apply_to(st.get_session())
|
|
184
|
+
OPTMentorAdapter().import_trace(trace_json, task="sum_evens") # Python-Tutor-style trace
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
## Example reports
|
|
188
|
+
|
|
189
|
+
Output of `stucktrace demo` (offline engine). Full EN/RU versions and the exact LLM context are in [`docs/report_examples.md`](docs/report_examples.md).
|
|
190
|
+
|
|
191
|
+
<details open><summary><b>Student Self-Reflection Report</b></summary>
|
|
192
|
+
|
|
193
|
+
```markdown
|
|
194
|
+
## 🪞 Self-Reflection Report
|
|
195
|
+
**Session:** 3 task(s) · 9 runs · 6 failed · 5m30s
|
|
196
|
+
|
|
197
|
+
### Where you got stuck
|
|
198
|
+
- **sum_evens** — high pressure: 4 failed runs in 16s; same TypeError 3× in a row;
|
|
199
|
+
149s of silence after TypeError; you flagged: "why does % not work on my numbers?"
|
|
200
|
+
- **count_vowels** — low pressure: 44% of the code rewritten after IndexError
|
|
201
|
+
|
|
202
|
+
### Patterns you repeated
|
|
203
|
+
- `TypeError` ×3 (sum_evens) — two values of incompatible types meet — print the type()
|
|
204
|
+
of each operand before the failing line
|
|
205
|
+
|
|
206
|
+
### What worked
|
|
207
|
+
- On **sum_evens** you fixed `TypeError` after 4 failed attempt(s) with a small edit.
|
|
208
|
+
- On **sum_evens** the visualizer helped: success came within a few runs after tracing.
|
|
209
|
+
- **fizzbuzz** worked on the first try.
|
|
210
|
+
|
|
211
|
+
### Try next time
|
|
212
|
+
1. When a run fails, stop for 30 seconds and read the *whole* traceback before re-running.
|
|
213
|
+
2. If the same error comes back, write down (or say out loud) what you *expect* that line to do.
|
|
214
|
+
3. Change one thing at a time — big rewrites after an error hide what actually fixed it.
|
|
215
|
+
```
|
|
216
|
+
</details>
|
|
217
|
+
|
|
218
|
+
<details><summary><b>Instructor Diagnostic Report</b></summary>
|
|
219
|
+
|
|
220
|
+
```markdown
|
|
221
|
+
## 🧭 Instructor Diagnostic Report
|
|
222
|
+
**Student:** s1024 · **Course:** CS1302 Intro to Programming · 5m30s
|
|
223
|
+
|
|
224
|
+
| Task | Runs | Solved | 1st success | Time | Panic | Top error |
|
|
225
|
+
|---|---|---|---|---|---|---|
|
|
226
|
+
| sum_evens | 5 | ✅ | 5 | 3m20s | high (0.81) | TypeError: not all arguments converted… |
|
|
227
|
+
| count_vowels | 3 | ✅ | 3 | 31s | low (0.22) | IndexError: string index out of range |
|
|
228
|
+
| fizzbuzz | 1 | ✅ | 1 | 0s | low (0.00) | — |
|
|
229
|
+
|
|
230
|
+
### Difficulty hotspots (ranked by panic score)
|
|
231
|
+
- **sum_evens** — high (0.81); visualizer used 1× (helped: yes)
|
|
232
|
+
- `+00:05` rapid-fail burst: 4 failed runs in 16s
|
|
233
|
+
- `+00:12` repeated error: same TypeError 3× in a row
|
|
234
|
+
- `+00:21` long pause: 149s of silence after TypeError
|
|
235
|
+
- `+02:50` confusion marker: "why does % not work on my numbers?"
|
|
236
|
+
- digest: `F:NameError | F:TypeError x3 | P:149s | ?"why does % not work on m" | V | S`
|
|
237
|
+
|
|
238
|
+
### Suggested interventions
|
|
239
|
+
1. **sum_evens**: persistent `TypeError` — likely misconception; a 2-minute targeted explanation is worth it.
|
|
240
|
+
2. **sum_evens**: the student explicitly asked for help — follow up personally.
|
|
241
|
+
3. Visualization shortened time-to-fix on **sum_evens** — recommend it for similar exercises.
|
|
242
|
+
```
|
|
243
|
+
</details>
|
|
244
|
+
|
|
245
|
+
## Configuration
|
|
246
|
+
|
|
247
|
+
```python
|
|
248
|
+
from stucktrace import CompressorConfig, generate_report
|
|
249
|
+
cfg = CompressorConfig(panic_gap=25, panic_min_burst=3, repeat_streak=3, stall_gap=120, rewrite_ratio=0.4)
|
|
250
|
+
generate_report("instructor", config=cfg, backend="auto", lang="en") # backend: auto | hermes | offline
|
|
251
|
+
st.start_session(store_code=False, autosave="live.jsonl") # privacy / crash-safe log
|
|
252
|
+
```
|
|
253
|
+
|
|
254
|
+
## Development
|
|
255
|
+
|
|
256
|
+
```bash
|
|
257
|
+
python -m venv .venv && source .venv/bin/activate
|
|
258
|
+
pip install -e ".[dev]"
|
|
259
|
+
pytest -q # 37 tests, incl. real IPython shell + fake Hermes HTTP server
|
|
260
|
+
stucktrace demo
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
Team workflow: [`CONTRIBUTING.md`](CONTRIBUTING.md).
|
|
264
|
+
|
|
265
|
+
## Status & limitations (v0.1.0)
|
|
266
|
+
|
|
267
|
+
- The Hermes client assumes an OpenAI-compatible `chat/completions` API; adapt `HermesClient.complete` if your deployment differs.
|
|
268
|
+
- Moodle field names follow `mod_quiz` / LTI conventions but aren't tied to a specific Moodle web-service call yet.
|
|
269
|
+
- Panic thresholds are heuristics, chosen for introductory programming labs; tune them with `CompressorConfig`.
|
|
270
|
+
- Tracking is per kernel. Collecting sessions from many students is left to the deployment (shared folder, Moodle upload).
|
|
271
|
+
|
|
272
|
+
## License
|
|
273
|
+
|
|
274
|
+
MIT
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=64", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "stucktrace"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Learning-trajectory telemetry & reflection reports for Jupyter: detects where students get stuck (Moodle + OPT_Mentor + Hermes)."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.8"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [
|
|
13
|
+
{ name = "Ulykbek" },
|
|
14
|
+
{ name = "Alisher" },
|
|
15
|
+
{ name = "Azat" },
|
|
16
|
+
{ name = "Alisher" },
|
|
17
|
+
{ name = "Insar" },
|
|
18
|
+
]
|
|
19
|
+
keywords = ["education", "jupyter", "jupyterhub", "telemetry", "learning-analytics", "moodle", "llm", "reflection"]
|
|
20
|
+
classifiers = [
|
|
21
|
+
"Development Status :: 3 - Alpha",
|
|
22
|
+
"Framework :: Jupyter",
|
|
23
|
+
"Framework :: IPython",
|
|
24
|
+
"Intended Audience :: Education",
|
|
25
|
+
"License :: OSI Approved :: MIT License",
|
|
26
|
+
"Operating System :: OS Independent",
|
|
27
|
+
"Programming Language :: Python :: 3",
|
|
28
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
29
|
+
"Topic :: Education",
|
|
30
|
+
]
|
|
31
|
+
# Zero required dependencies: installs on any JupyterHub without conflicts.
|
|
32
|
+
dependencies = []
|
|
33
|
+
|
|
34
|
+
[project.optional-dependencies]
|
|
35
|
+
jupyter = ["ipython>=7.0"]
|
|
36
|
+
dev = ["pytest>=7", "ipython>=7.0", "build", "twine"]
|
|
37
|
+
|
|
38
|
+
[project.urls]
|
|
39
|
+
Homepage = "https://github.com/ulykWyou/stucktrace"
|
|
40
|
+
Repository = "https://github.com/ulykWyou/stucktrace"
|
|
41
|
+
Issues = "https://github.com/ulykWyou/stucktrace/issues"
|
|
42
|
+
|
|
43
|
+
[project.scripts]
|
|
44
|
+
stucktrace = "stucktrace.cli:main"
|
|
45
|
+
|
|
46
|
+
[tool.setuptools.packages.find]
|
|
47
|
+
where = ["src"]
|
|
48
|
+
|
|
49
|
+
[tool.pytest.ini_options]
|
|
50
|
+
testpaths = ["tests"]
|
|
51
|
+
pythonpath = ["src"]
|