chalk-worker 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- chalk_worker-0.5.0/LICENSE +21 -0
- chalk_worker-0.5.0/PKG-INFO +173 -0
- chalk_worker-0.5.0/README.md +158 -0
- chalk_worker-0.5.0/chalk_worker/__init__.py +6 -0
- chalk_worker-0.5.0/chalk_worker/cli.py +281 -0
- chalk_worker-0.5.0/chalk_worker/client.py +103 -0
- chalk_worker-0.5.0/chalk_worker/examples.py +263 -0
- chalk_worker-0.5.0/chalk_worker/grading.py +246 -0
- chalk_worker-0.5.0/chalk_worker/job.py +70 -0
- chalk_worker-0.5.0/chalk_worker/locate.py +68 -0
- chalk_worker-0.5.0/chalk_worker/loop.py +439 -0
- chalk_worker-0.5.0/chalk_worker/report.py +158 -0
- chalk_worker-0.5.0/chalk_worker/resources/elm.json +21 -0
- chalk_worker-0.5.0/chalk_worker/status.py +155 -0
- chalk_worker-0.5.0/chalk_worker/tui.py +327 -0
- chalk_worker-0.5.0/pyproject.toml +51 -0
- chalk_worker-0.5.0/pyproject.toml.orig +46 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Fábio Macêdo Mendes
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: chalk-worker
|
|
3
|
+
Version: 0.5.0
|
|
4
|
+
Summary: Pull worker that grades code answers of a Codehood course on the teacher's machine.
|
|
5
|
+
Author: Fábio Macêdo Mendes
|
|
6
|
+
Author-email: Fábio Macêdo Mendes <fabiomacedomendes@gmail.com>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Requires-Dist: chalk-grader>=0.4.0
|
|
10
|
+
Requires-Dist: codehood-cli==0.5.0
|
|
11
|
+
Requires-Dist: textual>=8.2
|
|
12
|
+
Requires-Dist: typer>=0.27.2
|
|
13
|
+
Requires-Python: >=3.13
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
|
|
16
|
+
# chalk-worker
|
|
17
|
+
|
|
18
|
+
A pull worker for [Codehood](https://github.com/fabiommendes/codehood). It runs
|
|
19
|
+
on the teacher's machine, claims grading jobs of `code` questions (mdq) from
|
|
20
|
+
the Codehood server, runs the test sources of each question against the answer
|
|
21
|
+
with [chalk-grader](https://github.com/fabiommendes/chalk) in a bwrap sandbox,
|
|
22
|
+
and sends back an mdq runner report with the teacher's `BOT` key. The server
|
|
23
|
+
scores the report by the rubric of the question.
|
|
24
|
+
|
|
25
|
+
Use it for questions that need an environment only the teacher has. Linux
|
|
26
|
+
only, with bubblewrap 0.8.0 or later for the sandbox (Ubuntu 22.04 ships
|
|
27
|
+
0.6.1). Languages: Elm (`elm-test`; needs `node`, `elm` and `elm-test`).
|
|
28
|
+
|
|
29
|
+
```sh
|
|
30
|
+
CODEHOOD_BOT_KEY=<key> chalk-worker [--server <url>] [--worker elm-test]
|
|
31
|
+
[--exam <slug> | --discipline <slug> | --course <discipline>/<user>_<edition>]
|
|
32
|
+
[--jobs N] [--memory 512MiB] [--memory-reserve 512MiB]
|
|
33
|
+
[--poll 10] [--lease 120] [--once] [--tui/--no-tui]
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Inside a course folder, the server and the course come from `codehood.toml`.
|
|
37
|
+
`--once` grades what is queued and exits.
|
|
38
|
+
|
|
39
|
+
The worker grades `--jobs` answers at once (default: the number of cores),
|
|
40
|
+
each in a thread that runs the profile's tools. It claims only as many jobs
|
|
41
|
+
as it has free slots (at most `--batch` per claim), so no claimed job waits
|
|
42
|
+
for a slot while its lease runs out; the lease is renewed from the claim on.
|
|
43
|
+
Each question is prepared once, by the first job that needs it; the other
|
|
44
|
+
jobs of the question wait for it. A cache directory serves one worker at a
|
|
45
|
+
time: a second worker on the same `--cache-dir` refuses to start.
|
|
46
|
+
|
|
47
|
+
`--memory` is the memory of each run of a job (default 512 MiB; the build
|
|
48
|
+
and the tests are runs of their own). It is also the budget of the job when
|
|
49
|
+
starting jobs: a job starts only while the available memory of the machine
|
|
50
|
+
(`MemAvailable`), less `--memory-reserve`, holds it on top of the budgets of
|
|
51
|
+
the jobs in progress, so the slots in use shrink when memory runs short.
|
|
52
|
+
`--memory 0` leaves the profile's default (512 MiB in Elm) and does not look
|
|
53
|
+
at the available memory.
|
|
54
|
+
|
|
55
|
+
### Limits of the runs
|
|
56
|
+
|
|
57
|
+
With a delegated cgroup v2, each run gets a cgroup of its own (chalk-grader
|
|
58
|
+
0.3 or later): `memory.max` is `--memory`, without swap, with a process limit,
|
|
59
|
+
and the whole run is killed when it goes past them, without touching the
|
|
60
|
+
worker or the other runs. An answer that allocates without end gets
|
|
61
|
+
`resource-limit` in every case, with "The answer used more than N MiB of
|
|
62
|
+
memory.". In Elm, `--memory` is also node's heap, and `elm-test` runs with one
|
|
63
|
+
worker thread, since the worker already grades one answer per core.
|
|
64
|
+
|
|
65
|
+
The worker gets the cgroup by running itself again, on start, under
|
|
66
|
+
`systemd-run --user --scope -p Delegate=yes`, so it needs the systemd of the
|
|
67
|
+
user. A desktop session has it. Over ssh, it lives only while the user has a
|
|
68
|
+
session, unless lingering is on:
|
|
69
|
+
|
|
70
|
+
```sh
|
|
71
|
+
loginctl enable-linger $USER
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
The worker says which case holds when it starts (in the log, or in the title
|
|
75
|
+
of the UI: `run limits on` or `NO run limits`). Without the systemd of the
|
|
76
|
+
user (ssh without lingering, cron, a container) or without cgroup v2, runs
|
|
77
|
+
have no memory or process limit: the budget is only node's heap in Elm, so
|
|
78
|
+
an answer that allocates without end crashes node and errs every case, and
|
|
79
|
+
the other processes of a run are not limited. There is no `RLIMIT_AS`
|
|
80
|
+
fallback: node does not start with an address space below about 1.4 GiB.
|
|
81
|
+
|
|
82
|
+
When the system's OOM killer (not the run's own limit) kills a run twice,
|
|
83
|
+
or the run cannot be started in its cgroup, the answer is not graded: the job
|
|
84
|
+
goes back to the queue spending one of its retries, the log says why, and,
|
|
85
|
+
after an OOM kill, the worker waits before claiming again. Do not run the
|
|
86
|
+
worker inside a `systemd-run --scope -p MemoryMax=...`: when that scope runs
|
|
87
|
+
out of memory, systemd stops all of it, the worker included.
|
|
88
|
+
|
|
89
|
+
On a terminal the worker shows a UI ([Textual](https://textual.textualize.io/))
|
|
90
|
+
with the slots, the jobs finished and the log; `p` pauses and resumes the
|
|
91
|
+
claims, `q` stops claiming and quits after the jobs in progress (again to
|
|
92
|
+
abort). With `--once` the UI stays open when the queue is empty, until `q`.
|
|
93
|
+
`--no-tui` logs lines to stderr instead, where `SIGINT`/`SIGTERM` finish the
|
|
94
|
+
jobs in progress and exit.
|
|
95
|
+
|
|
96
|
+
How a job is graded:
|
|
97
|
+
|
|
98
|
+
1. The question arrives resolved in the job: test sources, `file` and the
|
|
99
|
+
answer key. The first job of each question builds the project (Elm: an
|
|
100
|
+
`elm.json` with the packages of the browser runner, one test module per
|
|
101
|
+
source), fills the package cache and runs the reference solution, which
|
|
102
|
+
must pass every test.
|
|
103
|
+
2. The answer is graded with the profile of its `language`. Each JUnit case
|
|
104
|
+
is located in the sources (`source`, `line`) so the server can tell which
|
|
105
|
+
rubric item owns it; a case not found is left out of the report.
|
|
106
|
+
3. A build error is a `build-error` report (compiler output without the
|
|
107
|
+
progress lines and the worker's paths), a timeout of the run times out
|
|
108
|
+
every case, and a failure of the infrastructure calls `fail` so the job
|
|
109
|
+
returns to the queue. A run that leaves no test report is graded again
|
|
110
|
+
with the reference solution: when that still passes, the answer crashed
|
|
111
|
+
the test runner (e.g. `Debug.todo` in a top-level value) and every case
|
|
112
|
+
is an `error`; otherwise it is a failure of the infrastructure. A run
|
|
113
|
+
that goes past its memory is `resource-limit` in every case (see Limits
|
|
114
|
+
of the runs).
|
|
115
|
+
4. When the project cannot be built (a package download fails, a tool is
|
|
116
|
+
missing), the jobs of the question go back to the queue without spending
|
|
117
|
+
their retries and the worker waits before claiming again; `--once` stops
|
|
118
|
+
claiming there. The same holds when the system kills the run of the
|
|
119
|
+
reference solution. A reference solution that fails a test, or that goes
|
|
120
|
+
past `--memory`, spends the retries of every job of the question, so the
|
|
121
|
+
instructor sees them as failed.
|
|
122
|
+
|
|
123
|
+
Test sources of an Elm question may be written as examples instead of a test
|
|
124
|
+
module: a fenced block (no `origin`) of format `elm` with no `module` header.
|
|
125
|
+
A linked file, or a block of another format, with no `module` header is an
|
|
126
|
+
error of the question. One line per case, in this order:
|
|
127
|
+
|
|
128
|
+
- an `mdq:` annotation line (same rule as the MDQ lexer) is a rubric item
|
|
129
|
+
with its regions, feedback and `{public}`/`{example}`;
|
|
130
|
+
- blank lines and lines with only a `--` comment are ignored;
|
|
131
|
+
- `import X` lines, only before the first case, are copied to the generated
|
|
132
|
+
module;
|
|
133
|
+
- `expr ==> value` is `Expect.equal`, and `expr ~= value` is `Expect.within`
|
|
134
|
+
with an absolute or relative tolerance (`1.0e-9`, `1.0e-6`), for scalar
|
|
135
|
+
`Float` only. An inline `-- comment` (outside string and character
|
|
136
|
+
literals) goes into the test name. Exactly one operator per line;
|
|
137
|
+
- anything else, including `{- -}` comments, is an error.
|
|
138
|
+
|
|
139
|
+
The test name is `expr ==> value (comment)`, normalized to one space around
|
|
140
|
+
the operator, and must be unique in the block. Lines are split only on `\n`,
|
|
141
|
+
`\r` and `\r\n`. Cases before the first annotation fall in a hidden implicit
|
|
142
|
+
item. The generated module is `Examples<i>` (`i` is the index of the source),
|
|
143
|
+
exposes `tests` (`Test.concat`, no `describe`) and imports the answer as
|
|
144
|
+
`exposing (..)`; the answer module is `file` without `.elm`, `/` replaced by
|
|
145
|
+
`.`. `Examples<i>` must not collide with the answer module or another source.
|
|
146
|
+
An invalid block fails the question (with the line number), never the
|
|
147
|
+
student. An answer with no `module` header gets
|
|
148
|
+
`module <Answer> exposing (..)` prepended, which shifts the line numbers of
|
|
149
|
+
compiler errors by two. The format has a twin in TypeScript in codehood
|
|
150
|
+
(`src/services/elm-examples.ts`); both run
|
|
151
|
+
`tests/resources/elm-examples-cases.json`.
|
|
152
|
+
|
|
153
|
+
Caches live in `$XDG_CACHE_HOME/chalk-worker`.
|
|
154
|
+
|
|
155
|
+
## Development
|
|
156
|
+
|
|
157
|
+
The worker lives in the codehood repository, in `workers/chalk-worker/`, and
|
|
158
|
+
depends on `codehood-cli` from `../../cli` and on `chalk-grader` from PyPI.
|
|
159
|
+
|
|
160
|
+
```sh
|
|
161
|
+
uv run ruff check && uv run pytest
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
The Elm tests download packages on the first run and are skipped without
|
|
165
|
+
`elm`, `elm-test` and `node`. The tests of the cgroups (`tests/test_cgroup.py`)
|
|
166
|
+
need a delegated scope and the pytest of the virtualenv, not `uv run` (uv
|
|
167
|
+
would share the scope):
|
|
168
|
+
|
|
169
|
+
```sh
|
|
170
|
+
systemd-run --user --scope --quiet --collect -p Delegate=yes \
|
|
171
|
+
.venv/bin/python -m pytest tests/test_cgroup.py
|
|
172
|
+
```
|
|
173
|
+
`example/` holds a course with one question.
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
# chalk-worker
|
|
2
|
+
|
|
3
|
+
A pull worker for [Codehood](https://github.com/fabiommendes/codehood). It runs
|
|
4
|
+
on the teacher's machine, claims grading jobs of `code` questions (mdq) from
|
|
5
|
+
the Codehood server, runs the test sources of each question against the answer
|
|
6
|
+
with [chalk-grader](https://github.com/fabiommendes/chalk) in a bwrap sandbox,
|
|
7
|
+
and sends back an mdq runner report with the teacher's `BOT` key. The server
|
|
8
|
+
scores the report by the rubric of the question.
|
|
9
|
+
|
|
10
|
+
Use it for questions that need an environment only the teacher has. Linux
|
|
11
|
+
only, with bubblewrap 0.8.0 or later for the sandbox (Ubuntu 22.04 ships
|
|
12
|
+
0.6.1). Languages: Elm (`elm-test`; needs `node`, `elm` and `elm-test`).
|
|
13
|
+
|
|
14
|
+
```sh
|
|
15
|
+
CODEHOOD_BOT_KEY=<key> chalk-worker [--server <url>] [--worker elm-test]
|
|
16
|
+
[--exam <slug> | --discipline <slug> | --course <discipline>/<user>_<edition>]
|
|
17
|
+
[--jobs N] [--memory 512MiB] [--memory-reserve 512MiB]
|
|
18
|
+
[--poll 10] [--lease 120] [--once] [--tui/--no-tui]
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Inside a course folder, the server and the course come from `codehood.toml`.
|
|
22
|
+
`--once` grades what is queued and exits.
|
|
23
|
+
|
|
24
|
+
The worker grades `--jobs` answers at once (default: the number of cores),
|
|
25
|
+
each in a thread that runs the profile's tools. It claims only as many jobs
|
|
26
|
+
as it has free slots (at most `--batch` per claim), so no claimed job waits
|
|
27
|
+
for a slot while its lease runs out; the lease is renewed from the claim on.
|
|
28
|
+
Each question is prepared once, by the first job that needs it; the other
|
|
29
|
+
jobs of the question wait for it. A cache directory serves one worker at a
|
|
30
|
+
time: a second worker on the same `--cache-dir` refuses to start.
|
|
31
|
+
|
|
32
|
+
`--memory` is the memory of each run of a job (default 512 MiB; the build
|
|
33
|
+
and the tests are runs of their own). It is also the budget of the job when
|
|
34
|
+
starting jobs: a job starts only while the available memory of the machine
|
|
35
|
+
(`MemAvailable`), less `--memory-reserve`, holds it on top of the budgets of
|
|
36
|
+
the jobs in progress, so the slots in use shrink when memory runs short.
|
|
37
|
+
`--memory 0` leaves the profile's default (512 MiB in Elm) and does not look
|
|
38
|
+
at the available memory.
|
|
39
|
+
|
|
40
|
+
### Limits of the runs
|
|
41
|
+
|
|
42
|
+
With a delegated cgroup v2, each run gets a cgroup of its own (chalk-grader
|
|
43
|
+
0.3 or later): `memory.max` is `--memory`, without swap, with a process limit,
|
|
44
|
+
and the whole run is killed when it goes past them, without touching the
|
|
45
|
+
worker or the other runs. An answer that allocates without end gets
|
|
46
|
+
`resource-limit` in every case, with "The answer used more than N MiB of
|
|
47
|
+
memory.". In Elm, `--memory` is also node's heap, and `elm-test` runs with one
|
|
48
|
+
worker thread, since the worker already grades one answer per core.
|
|
49
|
+
|
|
50
|
+
The worker gets the cgroup by running itself again, on start, under
|
|
51
|
+
`systemd-run --user --scope -p Delegate=yes`, so it needs the systemd of the
|
|
52
|
+
user. A desktop session has it. Over ssh, it lives only while the user has a
|
|
53
|
+
session, unless lingering is on:
|
|
54
|
+
|
|
55
|
+
```sh
|
|
56
|
+
loginctl enable-linger $USER
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
The worker says which case holds when it starts (in the log, or in the title
|
|
60
|
+
of the UI: `run limits on` or `NO run limits`). Without the systemd of the
|
|
61
|
+
user (ssh without lingering, cron, a container) or without cgroup v2, runs
|
|
62
|
+
have no memory or process limit: the budget is only node's heap in Elm, so
|
|
63
|
+
an answer that allocates without end crashes node and errs every case, and
|
|
64
|
+
the other processes of a run are not limited. There is no `RLIMIT_AS`
|
|
65
|
+
fallback: node does not start with an address space below about 1.4 GiB.
|
|
66
|
+
|
|
67
|
+
When the system's OOM killer (not the run's own limit) kills a run twice,
|
|
68
|
+
or the run cannot be started in its cgroup, the answer is not graded: the job
|
|
69
|
+
goes back to the queue spending one of its retries, the log says why, and,
|
|
70
|
+
after an OOM kill, the worker waits before claiming again. Do not run the
|
|
71
|
+
worker inside a `systemd-run --scope -p MemoryMax=...`: when that scope runs
|
|
72
|
+
out of memory, systemd stops all of it, the worker included.
|
|
73
|
+
|
|
74
|
+
On a terminal the worker shows a UI ([Textual](https://textual.textualize.io/))
|
|
75
|
+
with the slots, the jobs finished and the log; `p` pauses and resumes the
|
|
76
|
+
claims, `q` stops claiming and quits after the jobs in progress (again to
|
|
77
|
+
abort). With `--once` the UI stays open when the queue is empty, until `q`.
|
|
78
|
+
`--no-tui` logs lines to stderr instead, where `SIGINT`/`SIGTERM` finish the
|
|
79
|
+
jobs in progress and exit.
|
|
80
|
+
|
|
81
|
+
How a job is graded:
|
|
82
|
+
|
|
83
|
+
1. The question arrives resolved in the job: test sources, `file` and the
|
|
84
|
+
answer key. The first job of each question builds the project (Elm: an
|
|
85
|
+
`elm.json` with the packages of the browser runner, one test module per
|
|
86
|
+
source), fills the package cache and runs the reference solution, which
|
|
87
|
+
must pass every test.
|
|
88
|
+
2. The answer is graded with the profile of its `language`. Each JUnit case
|
|
89
|
+
is located in the sources (`source`, `line`) so the server can tell which
|
|
90
|
+
rubric item owns it; a case not found is left out of the report.
|
|
91
|
+
3. A build error is a `build-error` report (compiler output without the
|
|
92
|
+
progress lines and the worker's paths), a timeout of the run times out
|
|
93
|
+
every case, and a failure of the infrastructure calls `fail` so the job
|
|
94
|
+
returns to the queue. A run that leaves no test report is graded again
|
|
95
|
+
with the reference solution: when that still passes, the answer crashed
|
|
96
|
+
the test runner (e.g. `Debug.todo` in a top-level value) and every case
|
|
97
|
+
is an `error`; otherwise it is a failure of the infrastructure. A run
|
|
98
|
+
that goes past its memory is `resource-limit` in every case (see Limits
|
|
99
|
+
of the runs).
|
|
100
|
+
4. When the project cannot be built (a package download fails, a tool is
|
|
101
|
+
missing), the jobs of the question go back to the queue without spending
|
|
102
|
+
their retries and the worker waits before claiming again; `--once` stops
|
|
103
|
+
claiming there. The same holds when the system kills the run of the
|
|
104
|
+
reference solution. A reference solution that fails a test, or that goes
|
|
105
|
+
past `--memory`, spends the retries of every job of the question, so the
|
|
106
|
+
instructor sees them as failed.
|
|
107
|
+
|
|
108
|
+
Test sources of an Elm question may be written as examples instead of a test
|
|
109
|
+
module: a fenced block (no `origin`) of format `elm` with no `module` header.
|
|
110
|
+
A linked file, or a block of another format, with no `module` header is an
|
|
111
|
+
error of the question. One line per case, in this order:
|
|
112
|
+
|
|
113
|
+
- an `mdq:` annotation line (same rule as the MDQ lexer) is a rubric item
|
|
114
|
+
with its regions, feedback and `{public}`/`{example}`;
|
|
115
|
+
- blank lines and lines with only a `--` comment are ignored;
|
|
116
|
+
- `import X` lines, only before the first case, are copied to the generated
|
|
117
|
+
module;
|
|
118
|
+
- `expr ==> value` is `Expect.equal`, and `expr ~= value` is `Expect.within`
|
|
119
|
+
with an absolute or relative tolerance (`1.0e-9`, `1.0e-6`), for scalar
|
|
120
|
+
`Float` only. An inline `-- comment` (outside string and character
|
|
121
|
+
literals) goes into the test name. Exactly one operator per line;
|
|
122
|
+
- anything else, including `{- -}` comments, is an error.
|
|
123
|
+
|
|
124
|
+
The test name is `expr ==> value (comment)`, normalized to one space around
|
|
125
|
+
the operator, and must be unique in the block. Lines are split only on `\n`,
|
|
126
|
+
`\r` and `\r\n`. Cases before the first annotation fall in a hidden implicit
|
|
127
|
+
item. The generated module is `Examples<i>` (`i` is the index of the source),
|
|
128
|
+
exposes `tests` (`Test.concat`, no `describe`) and imports the answer as
|
|
129
|
+
`exposing (..)`; the answer module is `file` without `.elm`, `/` replaced by
|
|
130
|
+
`.`. `Examples<i>` must not collide with the answer module or another source.
|
|
131
|
+
An invalid block fails the question (with the line number), never the
|
|
132
|
+
student. An answer with no `module` header gets
|
|
133
|
+
`module <Answer> exposing (..)` prepended, which shifts the line numbers of
|
|
134
|
+
compiler errors by two. The format has a twin in TypeScript in codehood
|
|
135
|
+
(`src/services/elm-examples.ts`); both run
|
|
136
|
+
`tests/resources/elm-examples-cases.json`.
|
|
137
|
+
|
|
138
|
+
Caches live in `$XDG_CACHE_HOME/chalk-worker`.
|
|
139
|
+
|
|
140
|
+
## Development
|
|
141
|
+
|
|
142
|
+
The worker lives in the codehood repository, in `workers/chalk-worker/`, and
|
|
143
|
+
depends on `codehood-cli` from `../../cli` and on `chalk-grader` from PyPI.
|
|
144
|
+
|
|
145
|
+
```sh
|
|
146
|
+
uv run ruff check && uv run pytest
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
The Elm tests download packages on the first run and are skipped without
|
|
150
|
+
`elm`, `elm-test` and `node`. The tests of the cgroups (`tests/test_cgroup.py`)
|
|
151
|
+
need a delegated scope and the pytest of the virtualenv, not `uv run` (uv
|
|
152
|
+
would share the scope):
|
|
153
|
+
|
|
154
|
+
```sh
|
|
155
|
+
systemd-run --user --scope --quiet --collect -p Delegate=yes \
|
|
156
|
+
.venv/bin/python -m pytest tests/test_cgroup.py
|
|
157
|
+
```
|
|
158
|
+
`example/` holds a course with one question.
|
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Command line of the worker.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import fcntl
|
|
8
|
+
import os
|
|
9
|
+
import signal
|
|
10
|
+
import sys
|
|
11
|
+
import threading
|
|
12
|
+
import tomllib
|
|
13
|
+
import traceback
|
|
14
|
+
from datetime import datetime
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import IO, Annotated, Any
|
|
17
|
+
|
|
18
|
+
import pydantic
|
|
19
|
+
import typer
|
|
20
|
+
from chalk_grader import cgroup
|
|
21
|
+
from mdq.models import CodeQuestion
|
|
22
|
+
|
|
23
|
+
from .client import WorkerClient
|
|
24
|
+
from .grading import QuestionGrader, question_hash
|
|
25
|
+
from .loop import Filters, Worker
|
|
26
|
+
|
|
27
|
+
MiB = 1024 * 1024
|
|
28
|
+
|
|
29
|
+
app = typer.Typer(add_completion=False)
|
|
30
|
+
|
|
31
|
+
#: Environment variable with the key of the grading bot.
|
|
32
|
+
BOT_KEY_VAR = "CODEHOOD_BOT_KEY"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@app.command()
|
|
36
|
+
def run(
|
|
37
|
+
server: Annotated[
|
|
38
|
+
str | None, typer.Option(help="Codehood URL; default: [server] url of codehood.toml.")
|
|
39
|
+
] = None,
|
|
40
|
+
key_file: Annotated[
|
|
41
|
+
Path | None, typer.Option(help=f"File with the BOT key; default: ${BOT_KEY_VAR}.")
|
|
42
|
+
] = None,
|
|
43
|
+
worker: Annotated[
|
|
44
|
+
list[str] | None, typer.Option(help="Worker id to take jobs of (repeatable).")
|
|
45
|
+
] = None,
|
|
46
|
+
exam: Annotated[str | None, typer.Option(help="Take only jobs of this exam (slug).")] = None,
|
|
47
|
+
discipline: Annotated[
|
|
48
|
+
str | None, typer.Option(help="Take only jobs of the courses of this discipline (slug).")
|
|
49
|
+
] = None,
|
|
50
|
+
course: Annotated[
|
|
51
|
+
str | None,
|
|
52
|
+
typer.Option(
|
|
53
|
+
help="Take only jobs of this course (<discipline>/<user>_<edition>); "
|
|
54
|
+
"default: the course of codehood.toml, unless --discipline is given."
|
|
55
|
+
),
|
|
56
|
+
] = None,
|
|
57
|
+
poll: Annotated[
|
|
58
|
+
float, typer.Option(help="Seconds between claims when the queue is empty.")
|
|
59
|
+
] = 10,
|
|
60
|
+
jobs: Annotated[
|
|
61
|
+
int | None, typer.Option(help="Jobs graded at once; default: the number of cores.")
|
|
62
|
+
] = None,
|
|
63
|
+
batch: Annotated[
|
|
64
|
+
int | None, typer.Option(help="Most jobs asked for in one claim; default: --jobs.")
|
|
65
|
+
] = None,
|
|
66
|
+
lease: Annotated[float, typer.Option(help="Seconds a claimed job is reserved.")] = 120,
|
|
67
|
+
memory: Annotated[
|
|
68
|
+
str,
|
|
69
|
+
typer.Option(
|
|
70
|
+
help="Memory each run of a job may take, as in 512MiB (with a delegated cgroup, "
|
|
71
|
+
"a run that goes past it is stopped: resource-limit); jobs start only while the "
|
|
72
|
+
"available memory holds them. 0: the profile's default, and no check of the "
|
|
73
|
+
"available memory."
|
|
74
|
+
),
|
|
75
|
+
] = "512MiB",
|
|
76
|
+
memory_reserve: Annotated[
|
|
77
|
+
str, typer.Option(help="Memory left to the rest of the machine when starting jobs.")
|
|
78
|
+
] = "512MiB",
|
|
79
|
+
sandbox: Annotated[
|
|
80
|
+
str | None, typer.Option(help="Sandbox of the runs: bwrap (default) or none.")
|
|
81
|
+
] = None,
|
|
82
|
+
cache_dir: Annotated[
|
|
83
|
+
Path | None, typer.Option(help="Where graders and packages are kept.")
|
|
84
|
+
] = None,
|
|
85
|
+
once: Annotated[
|
|
86
|
+
bool, typer.Option(help="Grade what is pending and exit instead of polling.")
|
|
87
|
+
] = False,
|
|
88
|
+
tui: Annotated[
|
|
89
|
+
bool | None,
|
|
90
|
+
typer.Option(help="Show the slots and the jobs in a terminal UI; default: on a terminal."),
|
|
91
|
+
] = None,
|
|
92
|
+
) -> None:
|
|
93
|
+
"""
|
|
94
|
+
Claim grading jobs from Codehood, grade them locally and send the reports.
|
|
95
|
+
"""
|
|
96
|
+
if exam and discipline:
|
|
97
|
+
raise typer.BadParameter("use --exam or --discipline, not both")
|
|
98
|
+
on_terminal = sys.stdin.isatty() and sys.stdout.isatty()
|
|
99
|
+
if tui is None:
|
|
100
|
+
tui = on_terminal
|
|
101
|
+
elif tui and not on_terminal:
|
|
102
|
+
raise typer.BadParameter("--tui needs a terminal on stdin and stdout; use --no-tui")
|
|
103
|
+
config = _course_config(Path.cwd())
|
|
104
|
+
url = server or config.get("server", {}).get("url")
|
|
105
|
+
if not url:
|
|
106
|
+
raise typer.BadParameter("no server: use --server or set [server] url in codehood.toml")
|
|
107
|
+
if course is None and discipline is None:
|
|
108
|
+
course = _course_ref(config)
|
|
109
|
+
key = _read_key(key_file)
|
|
110
|
+
per_job = _bytes(memory, "--memory") or None
|
|
111
|
+
reserve = _bytes(memory_reserve, "--memory-reserve")
|
|
112
|
+
root = cache_dir or _default_cache()
|
|
113
|
+
_cache_lock = _lock_cache(root) # held until the process exits
|
|
114
|
+
jobs = jobs or os.process_cpu_count() or 1
|
|
115
|
+
|
|
116
|
+
def make_grader(question: CodeQuestion) -> QuestionGrader:
|
|
117
|
+
return QuestionGrader(
|
|
118
|
+
question, root / question_hash(question)[:16], sandbox=sandbox, memory=per_job
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
loop = Worker(
|
|
122
|
+
WorkerClient(url, key),
|
|
123
|
+
make_grader,
|
|
124
|
+
workers=worker or ["elm-test"],
|
|
125
|
+
jobs=jobs,
|
|
126
|
+
batch=batch,
|
|
127
|
+
poll=poll,
|
|
128
|
+
lease=lease,
|
|
129
|
+
filters=Filters(discipline=discipline, course=course, exam=exam),
|
|
130
|
+
log=(lambda _: None) if tui else _log,
|
|
131
|
+
memory=per_job,
|
|
132
|
+
reserve=reserve,
|
|
133
|
+
)
|
|
134
|
+
workers = ", ".join(worker or ["elm-test"])
|
|
135
|
+
limited, limits = _limits_note(per_job)
|
|
136
|
+
loop.status.log(limits)
|
|
137
|
+
if tui:
|
|
138
|
+
shown = "run limits on" if limited else "NO run limits"
|
|
139
|
+
_run_tui(loop, once, f"{workers} · {jobs} slots · {shown} · {url}")
|
|
140
|
+
return
|
|
141
|
+
|
|
142
|
+
_log(limits if limited else f"warning: {limits}")
|
|
143
|
+
|
|
144
|
+
def abort(signum: int, _frame: Any) -> None:
|
|
145
|
+
# The runs in progress live in cgroups of their own: kill them first.
|
|
146
|
+
if (runs := cgroup.controller()) is not None:
|
|
147
|
+
runs.kill_all()
|
|
148
|
+
signal.signal(signum, signal.SIG_DFL)
|
|
149
|
+
os.kill(os.getpid(), signum)
|
|
150
|
+
|
|
151
|
+
def stop(signum: int, _frame: Any) -> None:
|
|
152
|
+
_log("stopping after the jobs in progress (again to abort)")
|
|
153
|
+
loop.stop()
|
|
154
|
+
signal.signal(signum, abort)
|
|
155
|
+
|
|
156
|
+
signal.signal(signal.SIGINT, stop)
|
|
157
|
+
signal.signal(signal.SIGTERM, stop)
|
|
158
|
+
_log(f"claiming {workers} jobs from {url}, {jobs} at once")
|
|
159
|
+
done = loop.run(once=once)
|
|
160
|
+
_log(f"{done} jobs processed")
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _run_tui(loop: Worker, once: bool, subtitle: str) -> None:
|
|
164
|
+
from .tui import WorkerApp
|
|
165
|
+
|
|
166
|
+
app = WorkerApp(loop, once=once, subtitle=subtitle)
|
|
167
|
+
|
|
168
|
+
hung_up = threading.Event()
|
|
169
|
+
|
|
170
|
+
def hangup(_signum: int, _frame: Any) -> None:
|
|
171
|
+
# The terminal is gone, and the display may block writing to it: let
|
|
172
|
+
# the jobs in progress end, then leave without it. Again: at once.
|
|
173
|
+
if app.finished.is_set():
|
|
174
|
+
_exit(0)
|
|
175
|
+
if hung_up.is_set():
|
|
176
|
+
_exit(130)
|
|
177
|
+
hung_up.set()
|
|
178
|
+
loop.stop()
|
|
179
|
+
threading.Thread(target=lambda: (app.finished.wait(), _exit(0)), daemon=True).start()
|
|
180
|
+
|
|
181
|
+
signal.signal(signal.SIGHUP, hangup)
|
|
182
|
+
done = app.run()
|
|
183
|
+
if app.error is not None:
|
|
184
|
+
traceback.print_exception(app.error)
|
|
185
|
+
_log(f"the claim loop failed: {type(app.error).__name__}: {app.error}")
|
|
186
|
+
_exit(1)
|
|
187
|
+
if done is None:
|
|
188
|
+
# Aborted with jobs in progress: their leases run out on the server.
|
|
189
|
+
# Leave now, without waiting for the grading threads.
|
|
190
|
+
_log("aborted")
|
|
191
|
+
_exit(130)
|
|
192
|
+
_log(f"{done} jobs processed")
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _exit(code: int) -> None:
|
|
196
|
+
# os._exit skips atexit, so the runs left in their cgroups are killed here.
|
|
197
|
+
if (runs := cgroup.controller()) is not None:
|
|
198
|
+
runs.kill_all()
|
|
199
|
+
sys.stdout.flush()
|
|
200
|
+
sys.stderr.flush()
|
|
201
|
+
os._exit(code)
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _limits_note(memory: int | None) -> tuple[bool, str]:
|
|
205
|
+
"""
|
|
206
|
+
Whether runs have a cgroup of their own, with a line that says so.
|
|
207
|
+
"""
|
|
208
|
+
if cgroup.controller() is None:
|
|
209
|
+
return False, (
|
|
210
|
+
"runs have no memory or process limit: no delegated cgroup (start the worker "
|
|
211
|
+
"where the systemd of the user answers; see the README)"
|
|
212
|
+
)
|
|
213
|
+
budget = f"{memory // MiB} MiB" if memory else "the profile's default memory"
|
|
214
|
+
return True, f"each run in a cgroup of its own, with {budget}"
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _bytes(text: str, option: str) -> int:
|
|
218
|
+
try:
|
|
219
|
+
return int(pydantic.TypeAdapter(pydantic.ByteSize).validate_python(text))
|
|
220
|
+
except pydantic.ValidationError:
|
|
221
|
+
raise typer.BadParameter(f"{option}: not a size: {text!r}") from None
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _lock_cache(root: Path) -> IO[str]:
|
|
225
|
+
"""
|
|
226
|
+
Take the cache directory for this process, so that two workers do not
|
|
227
|
+
prepare the same question in it at once.
|
|
228
|
+
"""
|
|
229
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
230
|
+
lock = (root / ".lock").open("w")
|
|
231
|
+
try:
|
|
232
|
+
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
233
|
+
except BlockingIOError:
|
|
234
|
+
lock.close()
|
|
235
|
+
raise typer.BadParameter(
|
|
236
|
+
f"another chalk-worker uses {root}; stop it, or give this one its own --cache-dir"
|
|
237
|
+
) from None
|
|
238
|
+
return lock
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _course_config(start: Path) -> dict[str, Any]:
|
|
242
|
+
for folder in (start, *start.parents):
|
|
243
|
+
if (path := folder / "codehood.toml").is_file():
|
|
244
|
+
return tomllib.loads(path.read_text())
|
|
245
|
+
return {}
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _course_ref(config: dict[str, Any]) -> str | None:
|
|
249
|
+
course = config.get("course", {})
|
|
250
|
+
parts = [course.get(k) for k in ("discipline", "instructor", "edition")]
|
|
251
|
+
if not all(parts):
|
|
252
|
+
return None
|
|
253
|
+
discipline, instructor, edition = parts
|
|
254
|
+
return f"{discipline}/{instructor}_{edition}"
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def _read_key(key_file: Path | None) -> str:
|
|
258
|
+
key = key_file.read_text().strip() if key_file else os.environ.get(BOT_KEY_VAR, "").strip()
|
|
259
|
+
if not key:
|
|
260
|
+
raise typer.BadParameter(f"no BOT key: set {BOT_KEY_VAR} or use --key-file")
|
|
261
|
+
return key
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _default_cache() -> Path:
|
|
265
|
+
base = os.environ.get("XDG_CACHE_HOME") or str(Path.home() / ".cache")
|
|
266
|
+
return Path(base) / "chalk-worker"
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _log(line: str) -> None:
|
|
270
|
+
typer.echo(f"{datetime.now():%H:%M:%S} {line}", err=True)
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def main() -> None:
|
|
274
|
+
if "--help" not in sys.argv:
|
|
275
|
+
# Every command grades. Under a scope with Delegate=yes, each run
|
|
276
|
+
# gets a cgroup with its memory and process limits.
|
|
277
|
+
cgroup.reexec_delegated()
|
|
278
|
+
# Before any thread or process: the setup of the cgroups gives up
|
|
279
|
+
# when the delegated cgroup has other processes.
|
|
280
|
+
cgroup.controller()
|
|
281
|
+
app()
|