dlai-grader 2.0b1__py3-none-any.whl → 2.0b2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dlai-grader might be problematic. Click here for more details.
- dlai_grader/__init__.py +1 -1
- dlai_grader/io.py +26 -4
- dlai_grader/templates/Makefile +54 -0
- dlai_grader/templates/copy_assignment_sh/extrafile_n +12 -0
- dlai_grader/templates/copy_assignment_sh/extrafile_y +16 -0
- dlai_grader/templates/dockerfile/data_n_solution_n +20 -0
- dlai_grader/templates/dockerfile/data_n_solution_y +21 -0
- dlai_grader/templates/dockerfile/data_y_solution_n +21 -0
- dlai_grader/templates/dockerfile/data_y_solution_y +22 -0
- dlai_grader/templates/entry_py/solution_n_file_n.py +71 -0
- dlai_grader/templates/entry_py/solution_n_file_y.py +90 -0
- dlai_grader/templates/entry_py/solution_y_file_n.py +71 -0
- dlai_grader/templates/entry_py/solution_y_file_y.py +90 -0
- dlai_grader/templates/grader.py +32 -0
- dlai_grader/templates.py +216 -362
- {dlai_grader-2.0b1.dist-info → dlai_grader-2.0b2.dist-info}/METADATA +1 -1
- dlai_grader-2.0b2.dist-info/RECORD +27 -0
- {dlai_grader-2.0b1.dist-info → dlai_grader-2.0b2.dist-info}/WHEEL +1 -1
- dlai_grader/py.typed +0 -0
- dlai_grader-2.0b1.dist-info/RECORD +0 -16
- {dlai_grader-2.0b1.dist-info → dlai_grader-2.0b2.dist-info}/entry_points.txt +0 -0
- {dlai_grader-2.0b1.dist-info → dlai_grader-2.0b2.dist-info}/licenses/LICENSE +0 -0
- {dlai_grader-2.0b1.dist-info → dlai_grader-2.0b2.dist-info}/top_level.txt +0 -0
dlai_grader/__init__.py
CHANGED
dlai_grader/io.py
CHANGED
|
@@ -5,6 +5,7 @@ import subprocess
|
|
|
5
5
|
import tarfile
|
|
6
6
|
from contextlib import contextmanager, redirect_stderr, redirect_stdout
|
|
7
7
|
from os import devnull
|
|
8
|
+
from pathlib import Path
|
|
8
9
|
from textwrap import dedent
|
|
9
10
|
from zipfile import ZipFile
|
|
10
11
|
|
|
@@ -245,10 +246,10 @@ def init_grader() -> None:
|
|
|
245
246
|
write_file_from_template("./Makefile", template_dict["makefile"])
|
|
246
247
|
write_file_from_template("./.conf", template_dict["conf"])
|
|
247
248
|
write_file_from_template("./entry.py", template_dict["entry_py"])
|
|
248
|
-
write_file_from_template(
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
)
|
|
249
|
+
# write_file_from_template(
|
|
250
|
+
# "./copy_assignment_to_submission.sh",
|
|
251
|
+
# template_dict["copy_assignment_to_submission_sh"],
|
|
252
|
+
# )
|
|
252
253
|
write_file_from_template("./requirements.txt", "dlai-grader")
|
|
253
254
|
write_file_from_template("./.env", "")
|
|
254
255
|
|
|
@@ -313,3 +314,24 @@ def grade_parts(
|
|
|
313
314
|
subprocess.run(cmd, shell=True, executable="/bin/bash")
|
|
314
315
|
except Exception as e:
|
|
315
316
|
print(f"There was an error grading with coursera_autograder. Details: {e}")
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def generate_learner_file(
|
|
320
|
+
filename_source: str,
|
|
321
|
+
filename_target: str,
|
|
322
|
+
) -> None:
|
|
323
|
+
"""
|
|
324
|
+
Generates the learning facing version of any file.
|
|
325
|
+
|
|
326
|
+
Args:
|
|
327
|
+
filename_source (str): Path to original notebook.
|
|
328
|
+
filename_target (str): Path where to save reformatted notebook.
|
|
329
|
+
|
|
330
|
+
"""
|
|
331
|
+
solution_code = Path(filename_source).read_text()
|
|
332
|
+
|
|
333
|
+
# format the code to replace with placeholders
|
|
334
|
+
fmt_code = solution_to_learner_format(solution_code)
|
|
335
|
+
|
|
336
|
+
# save the learner files
|
|
337
|
+
Path(filename_target).write_text(fmt_code, encoding="utf-8")
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
.PHONY: sync learner build debug-unsafe debug-safe grade versioning tag undeletable uneditable init upgrade coursera zip
|
|
2
|
+
|
|
3
|
+
include .conf
|
|
4
|
+
|
|
5
|
+
PARTIDS = 123 456
|
|
6
|
+
OS := $(shell uname)
|
|
7
|
+
|
|
8
|
+
sync:
|
|
9
|
+
cp mount/submission.ipynb ../$(ASSIGNMENT_NAME)_Solution.ipynb
|
|
10
|
+
cp learner/$(ASSIGNMENT_NAME).ipynb ../$(ASSIGNMENT_NAME).ipynb
|
|
11
|
+
cp mount/$(UNIT_TESTS_NAME).py ../$(UNIT_TESTS_NAME).py
|
|
12
|
+
|
|
13
|
+
learner:
|
|
14
|
+
dlai_grader --learner --output_notebook=./learner/$(ASSIGNMENT_NAME).ipynb
|
|
15
|
+
rsync -a --exclude="submission.ipynb" --exclude="__pycache__" --exclude=".mypy_cache" ./mount/ ./learner/
|
|
16
|
+
|
|
17
|
+
build:
|
|
18
|
+
docker build -t $(IMAGE_NAME):$(TAG_ID) .
|
|
19
|
+
|
|
20
|
+
debug-unsafe:
|
|
21
|
+
docker run -it --rm --mount type=bind,source=$(PWD)/mount,target=/shared/submission --mount type=bind,source=$(PWD),target=/grader/ --env-file $(PWD)/.env --entrypoint ash $(IMAGE_NAME):$(TAG_ID)
|
|
22
|
+
|
|
23
|
+
debug-safe:
|
|
24
|
+
docker run -it --rm --mount type=bind,source=$(PWD)/mount,target=/shared/submission --env-file $(PWD)/.env --entrypoint ash $(IMAGE_NAME):$(TAG_ID)
|
|
25
|
+
|
|
26
|
+
grade:
|
|
27
|
+
docker run -it --rm --memory=$(HARD_MEMORY) --cpus="$(CPUS)" --memory-reservation=$(SOFT_MEMORY) --mount type=bind,source=$(PWD)/mount,target=/shared/submission --env-file $(PWD)/.env --entrypoint ash $(IMAGE_NAME):$(TAG_ID) -c 'for partId in $(PARTIDS); do export partId=$$partId; echo "Processing part $$partId"; python entry.py; done'
|
|
28
|
+
|
|
29
|
+
versioning:
|
|
30
|
+
dlai_grader --versioning
|
|
31
|
+
|
|
32
|
+
tag:
|
|
33
|
+
dlai_grader --tag
|
|
34
|
+
|
|
35
|
+
undeletable:
|
|
36
|
+
dlai_grader --undeletable
|
|
37
|
+
|
|
38
|
+
uneditable:
|
|
39
|
+
dlai_grader --uneditable
|
|
40
|
+
|
|
41
|
+
init:
|
|
42
|
+
dlai_grader --versioning
|
|
43
|
+
dlai_grader --tag
|
|
44
|
+
dlai_grader --undeletable
|
|
45
|
+
dlai_grader --uneditable
|
|
46
|
+
|
|
47
|
+
upgrade:
|
|
48
|
+
dlai_grader --upgrade
|
|
49
|
+
|
|
50
|
+
coursera:
|
|
51
|
+
dlai_grader --grade --partids="$(PARTIDS)" --docker=$(IMAGE_NAME):$(TAG_ID) --memory=$(MEMORY_LIMIT) --submission=$(SUB_DIR)
|
|
52
|
+
|
|
53
|
+
zip:
|
|
54
|
+
zip -r $(IMAGE_NAME)$(TAG_ID).zip .
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
#!/bin/bash",
|
|
2
|
+
set -euo pipefail
|
|
3
|
+
|
|
4
|
+
Assignment={{ASSIGNMENT_NAME}}
|
|
5
|
+
|
|
6
|
+
SubmissionFile=submission.ipynb
|
|
7
|
+
SubmissionPath=/shared/submission
|
|
8
|
+
SharedDiskPath=/learner_workplace/$UserId/$CourseId/$LessonId
|
|
9
|
+
|
|
10
|
+
# copy synced files (exam image typically sync all files in lesson folder)
|
|
11
|
+
echo "Copy learner submission from $SharedDiskPath/$Assignment to $SubmissionPath/$SubmissionFile"
|
|
12
|
+
cp $SharedDiskPath/$Assignment $SubmissionPath/$SubmissionFile
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
#!/bin/bash",
|
|
2
|
+
set -euo pipefail
|
|
3
|
+
|
|
4
|
+
Assignment={{ASSIGNMENT_NAME}}
|
|
5
|
+
Extra_file={{EXTRA_FILE_NAME}}
|
|
6
|
+
|
|
7
|
+
SubmissionFile=submission.ipynb
|
|
8
|
+
SubmissionPath=/shared/submission
|
|
9
|
+
SharedDiskPath=/learner_workplace/$UserId/$CourseId/$LessonId
|
|
10
|
+
|
|
11
|
+
# copy synced files (exam image typically sync all files in lesson folder)
|
|
12
|
+
echo "Copy learner submission from $SharedDiskPath/$Assignment to $SubmissionPath/$SubmissionFile"
|
|
13
|
+
cp $SharedDiskPath/$Assignment $SubmissionPath/$SubmissionFile
|
|
14
|
+
|
|
15
|
+
echo "Copy learner submission from $SharedDiskPath/$Extra_file to $SubmissionPath/$Extra_file"
|
|
16
|
+
cp $SharedDiskPath/$Extra_file $SubmissionPath/$Extra_file
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
FROM continuumio/miniconda3@sha256:d601a04ea48fd45e60808c7072243d33703d29434d2067816b7f26b0705d889a
|
|
2
|
+
|
|
3
|
+
RUN apk update && apk add libstdc++
|
|
4
|
+
|
|
5
|
+
COPY requirements.txt .
|
|
6
|
+
|
|
7
|
+
RUN pip install -r requirements.txt && rm requirements.txt
|
|
8
|
+
|
|
9
|
+
RUN mkdir /grader && \
|
|
10
|
+
mkdir /grader/submission
|
|
11
|
+
|
|
12
|
+
COPY .conf /grader/.conf
|
|
13
|
+
COPY entry.py /grader/entry.py
|
|
14
|
+
COPY grader.py /grader/grader.py
|
|
15
|
+
|
|
16
|
+
RUN chmod a+rwx /grader/
|
|
17
|
+
|
|
18
|
+
WORKDIR /grader/
|
|
19
|
+
|
|
20
|
+
ENTRYPOINT ["python", "entry.py"]
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
FROM continuumio/miniconda3@sha256:d601a04ea48fd45e60808c7072243d33703d29434d2067816b7f26b0705d889a
|
|
2
|
+
|
|
3
|
+
RUN apk update && apk add libstdc++
|
|
4
|
+
|
|
5
|
+
COPY requirements.txt .
|
|
6
|
+
|
|
7
|
+
RUN pip install -r requirements.txt && rm requirements.txt
|
|
8
|
+
|
|
9
|
+
RUN mkdir /grader && \
|
|
10
|
+
mkdir /grader/submission
|
|
11
|
+
|
|
12
|
+
COPY .conf /grader/.conf
|
|
13
|
+
COPY solution/ /grader/solution/
|
|
14
|
+
COPY entry.py /grader/entry.py
|
|
15
|
+
COPY grader.py /grader/grader.py
|
|
16
|
+
|
|
17
|
+
RUN chmod a+rwx /grader/
|
|
18
|
+
|
|
19
|
+
WORKDIR /grader/
|
|
20
|
+
|
|
21
|
+
ENTRYPOINT ["python", "entry.py"]
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
FROM continuumio/miniconda3@sha256:d601a04ea48fd45e60808c7072243d33703d29434d2067816b7f26b0705d889a
|
|
2
|
+
|
|
3
|
+
RUN apk update && apk add libstdc++
|
|
4
|
+
|
|
5
|
+
COPY requirements.txt .
|
|
6
|
+
|
|
7
|
+
RUN pip install -r requirements.txt && rm requirements.txt
|
|
8
|
+
|
|
9
|
+
RUN mkdir /grader && \
|
|
10
|
+
mkdir /grader/submission
|
|
11
|
+
|
|
12
|
+
COPY .conf /grader/.conf
|
|
13
|
+
COPY data/ /grader/data/
|
|
14
|
+
COPY entry.py /grader/entry.py
|
|
15
|
+
COPY grader.py /grader/grader.py
|
|
16
|
+
|
|
17
|
+
RUN chmod a+rwx /grader/
|
|
18
|
+
|
|
19
|
+
WORKDIR /grader/
|
|
20
|
+
|
|
21
|
+
ENTRYPOINT ["python", "entry.py"]
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
FROM continuumio/miniconda3@sha256:d601a04ea48fd45e60808c7072243d33703d29434d2067816b7f26b0705d889a
|
|
2
|
+
|
|
3
|
+
RUN apk update && apk add libstdc++
|
|
4
|
+
|
|
5
|
+
COPY requirements.txt .
|
|
6
|
+
|
|
7
|
+
RUN pip install -r requirements.txt && rm requirements.txt
|
|
8
|
+
|
|
9
|
+
RUN mkdir /grader && \
|
|
10
|
+
mkdir /grader/submission
|
|
11
|
+
|
|
12
|
+
COPY .conf /grader/.conf
|
|
13
|
+
COPY data/ /grader/data/
|
|
14
|
+
COPY solution/ /grader/solution/
|
|
15
|
+
COPY entry.py /grader/entry.py
|
|
16
|
+
COPY grader.py /grader/grader.py
|
|
17
|
+
|
|
18
|
+
RUN chmod a+rwx /grader/
|
|
19
|
+
|
|
20
|
+
WORKDIR /grader/
|
|
21
|
+
|
|
22
|
+
ENTRYPOINT ["python", "entry.py"]
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import traceback
|
|
2
|
+
from dlai_grader.config import Config, get_part_id
|
|
3
|
+
from dlai_grader.compiler import compile_partial_module
|
|
4
|
+
from dlai_grader.io import read_notebook, copy_submission_to_workdir, send_feedback
|
|
5
|
+
from dlai_grader.notebook import keep_tagged_cells
|
|
6
|
+
from dlai_grader.grading import compute_grading_score, graded_obj_missing
|
|
7
|
+
from grader import handle_part_id
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def notebook_grading(config, compile_solution=False):
|
|
11
|
+
try:
|
|
12
|
+
nb = read_notebook(config.submission_file_path)
|
|
13
|
+
except Exception as e:
|
|
14
|
+
msg = f"There was a problem reading your notebook. Details:\n{e!s}"
|
|
15
|
+
send_feedback(0.0, msg, err=True)
|
|
16
|
+
|
|
17
|
+
transformations = [keep_tagged_cells()]
|
|
18
|
+
for t in transformations:
|
|
19
|
+
nb = t(nb)
|
|
20
|
+
|
|
21
|
+
try:
|
|
22
|
+
learner_mod = compile_partial_module(nb, "learner_mod", verbose=False)
|
|
23
|
+
except Exception as e:
|
|
24
|
+
msg = f"There was a problem compiling the code from your notebook, please check that you saved before submitting. Details:\n{e!s}"
|
|
25
|
+
send_feedback(0.0, msg, err=True)
|
|
26
|
+
|
|
27
|
+
solution_mod = None
|
|
28
|
+
if compile_solution:
|
|
29
|
+
solution_nb = read_notebook(config.solution_file_path)
|
|
30
|
+
for t in transformations:
|
|
31
|
+
solution_nb = t(solution_nb)
|
|
32
|
+
solution_mod = compile_partial_module(
|
|
33
|
+
solution_nb,
|
|
34
|
+
"solution_mod",
|
|
35
|
+
verbose=False,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
return learner_mod, solution_mod
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def main() -> None:
|
|
42
|
+
copy_submission_to_workdir()
|
|
43
|
+
|
|
44
|
+
part_id = get_part_id()
|
|
45
|
+
|
|
46
|
+
c = Config()
|
|
47
|
+
|
|
48
|
+
learner_mod, _ = notebook_grading(c)
|
|
49
|
+
|
|
50
|
+
g_func = handle_part_id(part_id)(learner_mod)
|
|
51
|
+
|
|
52
|
+
try:
|
|
53
|
+
cases = g_func()
|
|
54
|
+
except Exception as e:
|
|
55
|
+
msg = f"There was an error grading your submission. Details:\n{e!s}"
|
|
56
|
+
send_feedback(0.0, msg, err=True)
|
|
57
|
+
|
|
58
|
+
if graded_obj_missing(cases):
|
|
59
|
+
msg = "Object required for grading not found. If you haven't completed the exercise this might be expected. Otherwise, check your solution as grader omits cells that throw errors."
|
|
60
|
+
send_feedback(0.0, msg, err=True)
|
|
61
|
+
|
|
62
|
+
score, feedback = compute_grading_score(cases)
|
|
63
|
+
send_feedback(score, feedback)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
if __name__ == "__main__":
|
|
67
|
+
try:
|
|
68
|
+
main()
|
|
69
|
+
except Exception as e:
|
|
70
|
+
msg = f"There was an error with the program. Exception:\n{e!s}.\nTraceback:\n{traceback.format_exc()}"
|
|
71
|
+
send_feedback(0.0, msg, err=True)
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import traceback
|
|
2
|
+
from dlai_grader.config import Config, get_part_id
|
|
3
|
+
from dlai_grader.compiler import compile_partial_module
|
|
4
|
+
from dlai_grader.io import read_notebook, copy_submission_to_workdir, send_feedback
|
|
5
|
+
from dlai_grader.notebook import keep_tagged_cells
|
|
6
|
+
from dlai_grader.grading import (
|
|
7
|
+
compute_grading_score,
|
|
8
|
+
graded_obj_missing,
|
|
9
|
+
LearnerSubmission,
|
|
10
|
+
)
|
|
11
|
+
from grader import handle_part_id
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def notebook_grading(config, compile_solution=False):
|
|
15
|
+
try:
|
|
16
|
+
nb = read_notebook(config.submission_file_path)
|
|
17
|
+
except Exception as e:
|
|
18
|
+
msg = f"There was a problem reading your notebook. Details:\n{e!s}"
|
|
19
|
+
send_feedback(0.0, msg, err=True)
|
|
20
|
+
|
|
21
|
+
transformations = [keep_tagged_cells()]
|
|
22
|
+
for t in transformations:
|
|
23
|
+
nb = t(nb)
|
|
24
|
+
|
|
25
|
+
try:
|
|
26
|
+
learner_mod = compile_partial_module(nb, "learner_mod", verbose=False)
|
|
27
|
+
except Exception as e:
|
|
28
|
+
msg = f"There was a problem compiling the code from your notebook, please check that you saved before submitting. Details:\n{e!s}"
|
|
29
|
+
send_feedback(0.0, msg, err=True)
|
|
30
|
+
|
|
31
|
+
solution_mod = None
|
|
32
|
+
if compile_solution:
|
|
33
|
+
solution_nb = read_notebook(config.solution_file_path)
|
|
34
|
+
for t in transformations:
|
|
35
|
+
solution_nb = t(solution_nb)
|
|
36
|
+
solution_mod = compile_partial_module(
|
|
37
|
+
solution_nb,
|
|
38
|
+
"solution_mod",
|
|
39
|
+
verbose=False,
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
return learner_mod, solution_mod
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def non_notebook_grading(config):
|
|
46
|
+
try:
|
|
47
|
+
with open(config.submission_file_path, "r") as file:
|
|
48
|
+
contents = file.read()
|
|
49
|
+
except Exception as e:
|
|
50
|
+
msg = f"There was an error reading your submission. Details:\n{e!s}"
|
|
51
|
+
send_feedback(0.0, msg, err=True)
|
|
52
|
+
|
|
53
|
+
return LearnerSubmission(submission=contents)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def main() -> None:
|
|
57
|
+
part_id = get_part_id()
|
|
58
|
+
|
|
59
|
+
match part_id:
|
|
60
|
+
case "123":
|
|
61
|
+
copy_submission_to_workdir(file_name="{{EXTRA_FILE_NAME}}")
|
|
62
|
+
c = Config(submission_file="{{EXTRA_FILE_NAME}}")
|
|
63
|
+
learner_mod = non_notebook_grading(c)
|
|
64
|
+
case _:
|
|
65
|
+
copy_submission_to_workdir()
|
|
66
|
+
c = Config()
|
|
67
|
+
learner_mod, _ = notebook_grading(c)
|
|
68
|
+
|
|
69
|
+
g_func = handle_part_id(part_id)(learner_mod)
|
|
70
|
+
|
|
71
|
+
try:
|
|
72
|
+
cases = g_func()
|
|
73
|
+
except Exception as e:
|
|
74
|
+
msg = f"There was an error grading your submission. Details:\n{e!s}"
|
|
75
|
+
send_feedback(0.0, msg, err=True)
|
|
76
|
+
|
|
77
|
+
if graded_obj_missing(cases):
|
|
78
|
+
msg = "Object required for grading not found. If you haven't completed the exercise this might be expected. Otherwise, check your solution as grader omits cells that throw errors."
|
|
79
|
+
send_feedback(0.0, msg, err=True)
|
|
80
|
+
|
|
81
|
+
score, feedback = compute_grading_score(cases)
|
|
82
|
+
send_feedback(score, feedback)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
if __name__ == "__main__":
|
|
86
|
+
try:
|
|
87
|
+
main()
|
|
88
|
+
except Exception as e:
|
|
89
|
+
msg = f"There was an error with the program. Exception:\n{e!s}.\nTraceback:\n{traceback.format_exc()}"
|
|
90
|
+
send_feedback(0.0, msg, err=True)
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import traceback
|
|
2
|
+
from dlai_grader.config import Config, get_part_id
|
|
3
|
+
from dlai_grader.compiler import compile_partial_module
|
|
4
|
+
from dlai_grader.io import read_notebook, copy_submission_to_workdir, send_feedback
|
|
5
|
+
from dlai_grader.notebook import keep_tagged_cells
|
|
6
|
+
from dlai_grader.grading import compute_grading_score, graded_obj_missing
|
|
7
|
+
from grader import handle_part_id
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def notebook_grading(config, compile_solution=False):
|
|
11
|
+
try:
|
|
12
|
+
nb = read_notebook(config.submission_file_path)
|
|
13
|
+
except Exception as e:
|
|
14
|
+
msg = f"There was a problem reading your notebook. Details:\n{e!s}"
|
|
15
|
+
send_feedback(0.0, msg, err=True)
|
|
16
|
+
|
|
17
|
+
transformations = [keep_tagged_cells()]
|
|
18
|
+
for t in transformations:
|
|
19
|
+
nb = t(nb)
|
|
20
|
+
|
|
21
|
+
try:
|
|
22
|
+
learner_mod = compile_partial_module(nb, "learner_mod", verbose=False)
|
|
23
|
+
except Exception as e:
|
|
24
|
+
msg = f"There was a problem compiling the code from your notebook, please check that you saved before submitting. Details:\n{e!s}"
|
|
25
|
+
send_feedback(0.0, msg, err=True)
|
|
26
|
+
|
|
27
|
+
solution_mod = None
|
|
28
|
+
if compile_solution:
|
|
29
|
+
solution_nb = read_notebook(config.solution_file_path)
|
|
30
|
+
for t in transformations:
|
|
31
|
+
solution_nb = t(solution_nb)
|
|
32
|
+
solution_mod = compile_partial_module(
|
|
33
|
+
solution_nb,
|
|
34
|
+
"solution_mod",
|
|
35
|
+
verbose=False,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
return learner_mod, solution_mod
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def main() -> None:
|
|
42
|
+
copy_submission_to_workdir()
|
|
43
|
+
|
|
44
|
+
part_id = get_part_id()
|
|
45
|
+
|
|
46
|
+
c = Config()
|
|
47
|
+
|
|
48
|
+
learner_mod, solution_mod = notebook_grading(c, compile_solution=True)
|
|
49
|
+
|
|
50
|
+
g_func = handle_part_id(part_id)(learner_mod, solution_mod)
|
|
51
|
+
|
|
52
|
+
try:
|
|
53
|
+
cases = g_func()
|
|
54
|
+
except Exception as e:
|
|
55
|
+
msg = f"There was an error grading your submission. Details:\n{e!s}"
|
|
56
|
+
send_feedback(0.0, msg, err=True)
|
|
57
|
+
|
|
58
|
+
if graded_obj_missing(cases):
|
|
59
|
+
msg = "Object required for grading not found. If you haven't completed the exercise this might be expected. Otherwise, check your solution as grader omits cells that throw errors."
|
|
60
|
+
send_feedback(0.0, msg, err=True)
|
|
61
|
+
|
|
62
|
+
score, feedback = compute_grading_score(cases)
|
|
63
|
+
send_feedback(score, feedback)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
if __name__ == "__main__":
|
|
67
|
+
try:
|
|
68
|
+
main()
|
|
69
|
+
except Exception as e:
|
|
70
|
+
msg = f"There was an error with the program. Exception:\n{e!s}.\nTraceback:\n{traceback.format_exc()}"
|
|
71
|
+
send_feedback(0.0, msg, err=True)
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import traceback
|
|
2
|
+
from dlai_grader.config import Config, get_part_id
|
|
3
|
+
from dlai_grader.compiler import compile_partial_module
|
|
4
|
+
from dlai_grader.io import read_notebook, copy_submission_to_workdir, send_feedback
|
|
5
|
+
from dlai_grader.notebook import keep_tagged_cells
|
|
6
|
+
from dlai_grader.grading import (
|
|
7
|
+
compute_grading_score,
|
|
8
|
+
graded_obj_missing,
|
|
9
|
+
LearnerSubmission,
|
|
10
|
+
)
|
|
11
|
+
from grader import handle_part_id
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def notebook_grading(config, compile_solution=False):
|
|
15
|
+
try:
|
|
16
|
+
nb = read_notebook(config.submission_file_path)
|
|
17
|
+
except Exception as e:
|
|
18
|
+
msg = f"There was a problem reading your notebook. Details:\n{e!s}"
|
|
19
|
+
send_feedback(0.0, msg, err=True)
|
|
20
|
+
|
|
21
|
+
transformations = [keep_tagged_cells()]
|
|
22
|
+
for t in transformations:
|
|
23
|
+
nb = t(nb)
|
|
24
|
+
|
|
25
|
+
try:
|
|
26
|
+
learner_mod = compile_partial_module(nb, "learner_mod", verbose=False)
|
|
27
|
+
except Exception as e:
|
|
28
|
+
msg = f"There was a problem compiling the code from your notebook, please check that you saved before submitting. Details:\n{e!s}"
|
|
29
|
+
send_feedback(0.0, msg, err=True)
|
|
30
|
+
|
|
31
|
+
solution_mod = None
|
|
32
|
+
if compile_solution:
|
|
33
|
+
solution_nb = read_notebook(config.solution_file_path)
|
|
34
|
+
for t in transformations:
|
|
35
|
+
solution_nb = t(solution_nb)
|
|
36
|
+
solution_mod = compile_partial_module(
|
|
37
|
+
solution_nb,
|
|
38
|
+
"solution_mod",
|
|
39
|
+
verbose=False,
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
return learner_mod, solution_mod
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def non_notebook_grading(config):
|
|
46
|
+
try:
|
|
47
|
+
with open(config.submission_file_path, "r") as file:
|
|
48
|
+
contents = file.read()
|
|
49
|
+
except Exception as e:
|
|
50
|
+
msg = f"There was an error reading your submission. Details:\n{e!s}"
|
|
51
|
+
send_feedback(0.0, msg, err=True)
|
|
52
|
+
|
|
53
|
+
return LearnerSubmission(submission=contents)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def main() -> None:
|
|
57
|
+
part_id = get_part_id()
|
|
58
|
+
|
|
59
|
+
match part_id:
|
|
60
|
+
case "123":
|
|
61
|
+
copy_submission_to_workdir(file_name="{{EXTRA_FILE_NAME}}")
|
|
62
|
+
c = Config(submission_file="{{EXTRA_FILE_NAME}}")
|
|
63
|
+
learner_mod = non_notebook_grading(c)
|
|
64
|
+
case _:
|
|
65
|
+
copy_submission_to_workdir()
|
|
66
|
+
c = Config()
|
|
67
|
+
learner_mod, solution_mod = notebook_grading(c, compile_solution=True)
|
|
68
|
+
|
|
69
|
+
g_func = handle_part_id(part_id)(learner_mod, solution_mod)
|
|
70
|
+
|
|
71
|
+
try:
|
|
72
|
+
cases = g_func()
|
|
73
|
+
except Exception as e:
|
|
74
|
+
msg = f"There was an error grading your submission. Details:\n{e!s}"
|
|
75
|
+
send_feedback(0.0, msg, err=True)
|
|
76
|
+
|
|
77
|
+
if graded_obj_missing(cases):
|
|
78
|
+
msg = "Object required for grading not found. If you haven't completed the exercise this might be expected. Otherwise, check your solution as grader omits cells that throw errors."
|
|
79
|
+
send_feedback(0.0, msg, err=True)
|
|
80
|
+
|
|
81
|
+
score, feedback = compute_grading_score(cases)
|
|
82
|
+
send_feedback(score, feedback)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
if __name__ == "__main__":
|
|
86
|
+
try:
|
|
87
|
+
main()
|
|
88
|
+
except Exception as e:
|
|
89
|
+
msg = f"There was an error with the program. Exception:\n{e!s}.\nTraceback:\n{traceback.format_exc()}"
|
|
90
|
+
send_feedback(0.0, msg, err=True)
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
from types import ModuleType, FunctionType
|
|
2
|
+
from dlai_grader.grading import test_case, object_to_grade
|
|
3
|
+
from dlai_grader.types import grading_function, grading_wrapper, learner_submission
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def part_1(
|
|
7
|
+
learner_mod: learner_submission,
|
|
8
|
+
solution_mod: ModuleType | None = None,
|
|
9
|
+
) -> grading_function:
|
|
10
|
+
@object_to_grade(learner_mod, "learner_func")
|
|
11
|
+
def g(learner_func: FunctionType) -> list[test_case]:
|
|
12
|
+
|
|
13
|
+
cases: list[test_case] = []
|
|
14
|
+
|
|
15
|
+
t = test_case()
|
|
16
|
+
if not isinstance(learner_func, FunctionType):
|
|
17
|
+
t.fail()
|
|
18
|
+
t.msg = "learner_func has incorrect type"
|
|
19
|
+
t.want = FunctionType
|
|
20
|
+
t.got = type(learner_func)
|
|
21
|
+
return [t]
|
|
22
|
+
|
|
23
|
+
return cases
|
|
24
|
+
|
|
25
|
+
return g
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def handle_part_id(part_id: str) -> grading_wrapper:
|
|
29
|
+
grader_dict: dict[str, grading_wrapper] = {
|
|
30
|
+
"": part_1,
|
|
31
|
+
}
|
|
32
|
+
return grader_dict[part_id]
|
dlai_grader/templates.py
CHANGED
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
import sys
|
|
2
|
+
import textwrap
|
|
2
3
|
from textwrap import dedent
|
|
4
|
+
import shutil
|
|
5
|
+
import os
|
|
6
|
+
from pathlib import Path
|
|
3
7
|
|
|
4
8
|
|
|
5
9
|
def generate_copy_assignment_script(
|
|
@@ -8,213 +12,69 @@ def generate_copy_assignment_script(
|
|
|
8
12
|
extra_file_name="foo.txt",
|
|
9
13
|
):
|
|
10
14
|
"""
|
|
11
|
-
|
|
15
|
+
Copy the appropriate copy_assignment_to_submission.sh script from templates depending on whether an extra file is required.
|
|
12
16
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
extra_file_name (str): The name of the extra file to copy (if required)
|
|
17
|
+
Template files should be named:
|
|
18
|
+
extrafile_n (no extra file)
|
|
19
|
+
extrafile_y (with extra file)
|
|
17
20
|
|
|
18
21
|
Returns:
|
|
19
|
-
str: The
|
|
22
|
+
str: The final shell script contents after variable substitution.
|
|
20
23
|
|
|
21
24
|
"""
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
"
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
"cp $SharedDiskPath/$Extra_file $SubmissionPath/$Extra_file",
|
|
53
|
-
]
|
|
54
|
-
|
|
55
|
-
# Combine all sections
|
|
56
|
-
content = header + variables + extra_file_copy
|
|
57
|
-
|
|
58
|
-
return "\n".join(content)
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
def generate_entry_py(
|
|
62
|
-
non_notebook_grading="n",
|
|
25
|
+
if extra_file_required not in ("y", "n"):
|
|
26
|
+
raise ValueError(f"Invalid extra_file_required value: {extra_file_required!r}")
|
|
27
|
+
|
|
28
|
+
# Define template name pattern
|
|
29
|
+
template_name = f"extrafile_{extra_file_required}"
|
|
30
|
+
|
|
31
|
+
# Paths
|
|
32
|
+
base_dir = Path(__file__).parent
|
|
33
|
+
src = base_dir / "templates" / "copy_assignment_sh" / template_name
|
|
34
|
+
dst = Path("copy_assignment_to_submission.sh")
|
|
35
|
+
|
|
36
|
+
# Validate existence
|
|
37
|
+
if not src.exists():
|
|
38
|
+
raise FileNotFoundError(f"Template not found: {src}")
|
|
39
|
+
|
|
40
|
+
# Read and substitute placeholders
|
|
41
|
+
content = src.read_text(encoding="utf-8")
|
|
42
|
+
content = content.replace("{{ASSIGNMENT_NAME}}", assignment_name).replace(
|
|
43
|
+
"{{EXTRA_FILE_NAME}}", extra_file_name
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
# Write output
|
|
47
|
+
dst.write_text(content, encoding="utf-8")
|
|
48
|
+
dst.chmod(0o755) # make executable
|
|
49
|
+
return content
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def copy_entry_script(
|
|
53
|
+
sol_dir_required: str,
|
|
54
|
+
non_notebook_grading: str,
|
|
63
55
|
extra_file_name="foo.txt",
|
|
64
|
-
):
|
|
65
|
-
|
|
66
|
-
|
|
56
|
+
) -> str:
|
|
57
|
+
# Validate inputs
|
|
58
|
+
if sol_dir_required not in ("y", "n"):
|
|
59
|
+
raise ValueError(f"Invalid sol_dir_required value: {sol_dir_required!r}")
|
|
60
|
+
if non_notebook_grading not in ("y", "n"):
|
|
61
|
+
raise ValueError(
|
|
62
|
+
f"Invalid non_notebook_grading value: {non_notebook_grading!r}"
|
|
63
|
+
)
|
|
67
64
|
|
|
68
|
-
|
|
69
|
-
non_notebook_grading (str): Include non-notebook grading if "y"
|
|
70
|
-
extra_file_name (str): Name of extra file to grade
|
|
65
|
+
template_name = f"solution_{sol_dir_required}_file_{non_notebook_grading}.py"
|
|
71
66
|
|
|
72
|
-
|
|
73
|
-
|
|
67
|
+
base_dir = Path(__file__).parent
|
|
68
|
+
src = base_dir / "templates" / "entry_py" / template_name
|
|
69
|
+
content = src.read_text(encoding="utf-8")
|
|
70
|
+
content = content.replace("{{EXTRA_FILE_NAME}}", extra_file_name)
|
|
71
|
+
dst = Path("entry.py")
|
|
74
72
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
"from dlai_grader.io import read_notebook, copy_submission_to_workdir, send_feedback",
|
|
81
|
-
"from dlai_grader.notebook import keep_tagged_cells",
|
|
82
|
-
"from dlai_grader.grading import (",
|
|
83
|
-
" compute_grading_score,",
|
|
84
|
-
" graded_obj_missing,",
|
|
85
|
-
" LearnerSubmission,",
|
|
86
|
-
")",
|
|
87
|
-
"from grader import handle_part_id",
|
|
88
|
-
"",
|
|
89
|
-
]
|
|
90
|
-
|
|
91
|
-
# Notebook grading function (common to both versions)
|
|
92
|
-
notebook_grading_func = [
|
|
93
|
-
"",
|
|
94
|
-
"def notebook_grading(config, compile_solution=False):",
|
|
95
|
-
" try:",
|
|
96
|
-
" nb = read_notebook(config.submission_file_path)",
|
|
97
|
-
" except Exception as e:",
|
|
98
|
-
' msg = f"There was a problem reading your notebook. Details:\\n{e!s}"',
|
|
99
|
-
" send_feedback(0.0, msg, err=True)",
|
|
100
|
-
"",
|
|
101
|
-
" transformations = [keep_tagged_cells()]",
|
|
102
|
-
" for t in transformations:",
|
|
103
|
-
" nb = t(nb)",
|
|
104
|
-
"",
|
|
105
|
-
" try:",
|
|
106
|
-
' learner_mod = compile_partial_module(nb, "learner_mod", verbose=False)',
|
|
107
|
-
" except Exception as e:",
|
|
108
|
-
' msg = f"There was a problem compiling the code from your notebook, please check that you saved before submitting. Details:\\n{e!s}"',
|
|
109
|
-
" send_feedback(0.0, msg, err=True)",
|
|
110
|
-
"",
|
|
111
|
-
" solution_mod = None",
|
|
112
|
-
" if compile_solution:",
|
|
113
|
-
" solution_nb = read_notebook(config.solution_file_path)",
|
|
114
|
-
" for t in transformations:",
|
|
115
|
-
" solution_nb = t(solution_nb)",
|
|
116
|
-
" solution_mod = compile_partial_module(",
|
|
117
|
-
' solution_nb, "solution_mod", verbose=False',
|
|
118
|
-
" )",
|
|
119
|
-
"",
|
|
120
|
-
" return learner_mod, solution_mod",
|
|
121
|
-
"",
|
|
122
|
-
]
|
|
123
|
-
|
|
124
|
-
# Non-notebook grading function (only for version with non_notebook_grading)
|
|
125
|
-
non_notebook_grading_func = [
|
|
126
|
-
"",
|
|
127
|
-
"def non_notebook_grading(config):",
|
|
128
|
-
" try:",
|
|
129
|
-
' with open(config.submission_file_path, "r") as file:',
|
|
130
|
-
" contents = file.read()",
|
|
131
|
-
" except Exception as e:",
|
|
132
|
-
' msg = f"There was an error reading your submission. Details:\\n{e!s}"',
|
|
133
|
-
" send_feedback(0.0, msg, err=True)",
|
|
134
|
-
"",
|
|
135
|
-
" return LearnerSubmission(submission=contents)",
|
|
136
|
-
"",
|
|
137
|
-
]
|
|
138
|
-
|
|
139
|
-
# Main function for version without non-notebook grading
|
|
140
|
-
main_func_simple = [
|
|
141
|
-
"def main() -> None:",
|
|
142
|
-
" copy_submission_to_workdir()",
|
|
143
|
-
"",
|
|
144
|
-
" part_id = get_part_id()",
|
|
145
|
-
"",
|
|
146
|
-
" c = Config()",
|
|
147
|
-
"",
|
|
148
|
-
" learner_mod, _ = notebook_grading(c)",
|
|
149
|
-
"",
|
|
150
|
-
" g_func = handle_part_id(part_id)(learner_mod)",
|
|
151
|
-
"",
|
|
152
|
-
" try:",
|
|
153
|
-
" cases = g_func()",
|
|
154
|
-
" except Exception as e:",
|
|
155
|
-
' msg = f"There was an error grading your submission. Details:\\n{e!s}"',
|
|
156
|
-
" send_feedback(0.0, msg, err=True)",
|
|
157
|
-
"",
|
|
158
|
-
" if graded_obj_missing(cases):",
|
|
159
|
-
' msg = "Object required for grading not found. If you haven\'t completed the exercise this might be expected. Otherwise, check your solution as grader omits cells that throw errors."',
|
|
160
|
-
" send_feedback(0.0, msg, err=True)",
|
|
161
|
-
"",
|
|
162
|
-
" score, feedback = compute_grading_score(cases)",
|
|
163
|
-
" send_feedback(score, feedback)",
|
|
164
|
-
"",
|
|
165
|
-
]
|
|
166
|
-
|
|
167
|
-
# Main function for version with non-notebook grading
|
|
168
|
-
main_func_with_non_notebook = [
|
|
169
|
-
"def main() -> None:",
|
|
170
|
-
" copy_submission_to_workdir()",
|
|
171
|
-
"",
|
|
172
|
-
" part_id = get_part_id()",
|
|
173
|
-
"",
|
|
174
|
-
" match part_id:",
|
|
175
|
-
' case "123":',
|
|
176
|
-
f' c = Config(submission_file="{extra_file_name}")',
|
|
177
|
-
" learner_mod = non_notebook_grading(c)",
|
|
178
|
-
" case _:",
|
|
179
|
-
" c = Config()",
|
|
180
|
-
" learner_mod, _ = notebook_grading(c)",
|
|
181
|
-
"",
|
|
182
|
-
" g_func = handle_part_id(part_id)(learner_mod)",
|
|
183
|
-
"",
|
|
184
|
-
" try:",
|
|
185
|
-
" cases = g_func()",
|
|
186
|
-
" except Exception as e:",
|
|
187
|
-
' msg = f"There was an error grading your submission. Details:\\n{e!s}"',
|
|
188
|
-
" send_feedback(0.0, msg, err=True)",
|
|
189
|
-
"",
|
|
190
|
-
" if graded_obj_missing(cases):",
|
|
191
|
-
' msg = "Object required for grading not found. If you haven\'t completed the exercise this might be expected. Otherwise, check your solution as grader omits cells that throw errors."',
|
|
192
|
-
" send_feedback(0.0, msg, err=True)",
|
|
193
|
-
"",
|
|
194
|
-
" score, feedback = compute_grading_score(cases)",
|
|
195
|
-
" send_feedback(score, feedback)",
|
|
196
|
-
"",
|
|
197
|
-
]
|
|
198
|
-
|
|
199
|
-
# Common script entry point
|
|
200
|
-
entry_point = [
|
|
201
|
-
'if __name__ == "__main__":',
|
|
202
|
-
" main()",
|
|
203
|
-
"",
|
|
204
|
-
]
|
|
205
|
-
|
|
206
|
-
# Combine all sections based on configuration
|
|
207
|
-
content = imports + notebook_grading_func
|
|
208
|
-
|
|
209
|
-
if non_notebook_grading == "y":
|
|
210
|
-
content.extend(non_notebook_grading_func)
|
|
211
|
-
content.extend(main_func_with_non_notebook)
|
|
212
|
-
else:
|
|
213
|
-
content.extend(main_func_simple)
|
|
214
|
-
|
|
215
|
-
content.extend(entry_point)
|
|
216
|
-
|
|
217
|
-
return "\n".join(content)
|
|
73
|
+
if not src.exists():
|
|
74
|
+
raise FileNotFoundError(f"Template not found: {src}")
|
|
75
|
+
|
|
76
|
+
# shutil.copy(src, dst)
|
|
77
|
+
return content
|
|
218
78
|
|
|
219
79
|
|
|
220
80
|
def generate_dockerfile(data_dir_required="n", sol_dir_required="n"):
|
|
@@ -225,46 +85,57 @@ def generate_dockerfile(data_dir_required="n", sol_dir_required="n"):
|
|
|
225
85
|
data_dir_required (str): Include data directory if "y"
|
|
226
86
|
sol_dir_required (str): Include solution directory if "y"
|
|
227
87
|
|
|
228
|
-
Returns:
|
|
229
|
-
str: The complete Dockerfile content
|
|
230
|
-
|
|
231
88
|
"""
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
""
|
|
235
|
-
|
|
236
|
-
""
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
)
|
|
89
|
+
# Validate inputs
|
|
90
|
+
if data_dir_required not in ("y", "n"):
|
|
91
|
+
raise ValueError(f"Invalid data_dir_required value: {data_dir_required!r}")
|
|
92
|
+
if sol_dir_required not in ("y", "n"):
|
|
93
|
+
raise ValueError(f"Invalid sol_dir_required value: {sol_dir_required!r}")
|
|
94
|
+
|
|
95
|
+
template_name = f"data_{data_dir_required}_solution_{sol_dir_required}"
|
|
96
|
+
|
|
97
|
+
# Define paths
|
|
98
|
+
base_dir = Path(__file__).parent
|
|
99
|
+
src = base_dir / "templates" / "dockerfile" / template_name
|
|
100
|
+
dst = Path("Dockerfile")
|
|
101
|
+
|
|
102
|
+
# Ensure the source exists
|
|
103
|
+
if not src.exists():
|
|
104
|
+
raise FileNotFoundError(f"Template not found: {src}")
|
|
105
|
+
|
|
106
|
+
# Copy template to current directory
|
|
107
|
+
# shutil.copy(src, dst)
|
|
108
|
+
|
|
109
|
+
# Return the Dockerfile contents
|
|
110
|
+
return src.read_text(encoding="utf-8")
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def copy_makefile() -> str:
|
|
114
|
+
base_dir = Path(__file__).parent
|
|
115
|
+
src = base_dir / "templates" / "Makefile"
|
|
116
|
+
# content = src.read_text(encoding="utf-8")
|
|
117
|
+
# content = content.replace("{{HARD_MEMORY}}", hard_memory)
|
|
118
|
+
# content = content.replace("{{CPUS}}", cpus)
|
|
119
|
+
# content = content.replace("{{SOFT_MEMORY}}", soft_memory)
|
|
120
|
+
dst = Path("Makefile")
|
|
121
|
+
|
|
122
|
+
if not src.exists():
|
|
123
|
+
raise FileNotFoundError(f"Template not found: {src}")
|
|
266
124
|
|
|
267
|
-
|
|
125
|
+
# shutil.copy(src, dst)
|
|
126
|
+
return src.read_text(encoding="utf-8")
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def copy_grader_py() -> str:
|
|
130
|
+
base_dir = Path(__file__).parent
|
|
131
|
+
src = base_dir / "templates" / "grader.py"
|
|
132
|
+
dst = Path("grader.py")
|
|
133
|
+
|
|
134
|
+
if not src.exists():
|
|
135
|
+
raise FileNotFoundError(f"Template not found: {src}")
|
|
136
|
+
|
|
137
|
+
# shutil.copy(src, dst)
|
|
138
|
+
return src.read_text(encoding="utf-8")
|
|
268
139
|
|
|
269
140
|
|
|
270
141
|
def load_templates() -> dict[str, str]:
|
|
@@ -272,38 +143,106 @@ def load_templates() -> dict[str, str]:
|
|
|
272
143
|
course = input("Number of the course: ")
|
|
273
144
|
module = input("Number of the module: ")
|
|
274
145
|
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
version = version if version else "1"
|
|
279
|
-
data_dir_required = input("Do you require a data dir? y/n (leave empty for n): ")
|
|
280
|
-
data_dir_required = data_dir_required if data_dir_required else "n"
|
|
146
|
+
grader_mvp = input(
|
|
147
|
+
"Use minimum grader (no extra config)? y/n (leave empty for n): ",
|
|
148
|
+
)
|
|
281
149
|
|
|
282
|
-
|
|
150
|
+
grader_mvp = grader_mvp if grader_mvp else "n"
|
|
151
|
+
if grader_mvp not in ["y", "n"]:
|
|
283
152
|
print("invalid option selected")
|
|
284
153
|
sys.exit(1)
|
|
285
154
|
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
155
|
+
if grader_mvp == "n":
|
|
156
|
+
unit_test_filename = input(
|
|
157
|
+
"Filename for unit tests (leave empty for unittests): "
|
|
158
|
+
)
|
|
159
|
+
unit_test_filename = unit_test_filename if unit_test_filename else "unittests"
|
|
160
|
+
# version = input("Version of the grader (leave empty for version 1): ")
|
|
161
|
+
version = "1"
|
|
162
|
+
data_dir_required = input(
|
|
163
|
+
"Do you require a data dir? y/n (leave empty for n): "
|
|
164
|
+
)
|
|
165
|
+
data_dir_required = data_dir_required if data_dir_required else "n"
|
|
293
166
|
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
non_notebook_grading = non_notebook_grading if non_notebook_grading else "n"
|
|
298
|
-
if non_notebook_grading not in ["y", "n"]:
|
|
299
|
-
print("invalid option selected")
|
|
300
|
-
sys.exit(1)
|
|
167
|
+
if data_dir_required not in ["y", "n"]:
|
|
168
|
+
print("invalid option selected")
|
|
169
|
+
sys.exit(1)
|
|
301
170
|
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
extra_file_name = input(
|
|
305
|
-
"Name of the extra file to grade: ",
|
|
171
|
+
sol_dir_required = input(
|
|
172
|
+
"Do you require a solution file? y/n (leave empty for n): "
|
|
306
173
|
)
|
|
174
|
+
sol_dir_required = sol_dir_required if sol_dir_required else "n"
|
|
175
|
+
if sol_dir_required not in ["y", "n"]:
|
|
176
|
+
print("invalid option selected")
|
|
177
|
+
sys.exit(1)
|
|
178
|
+
|
|
179
|
+
non_notebook_grading = input(
|
|
180
|
+
"Will you grade a file different from a notebook? y/n (leave empty for n): ",
|
|
181
|
+
)
|
|
182
|
+
non_notebook_grading = non_notebook_grading if non_notebook_grading else "n"
|
|
183
|
+
if non_notebook_grading not in ["y", "n"]:
|
|
184
|
+
print("invalid option selected")
|
|
185
|
+
sys.exit(1)
|
|
186
|
+
|
|
187
|
+
extra_file_name = ""
|
|
188
|
+
if non_notebook_grading == "y":
|
|
189
|
+
extra_file_name = input(
|
|
190
|
+
"Name of the extra file to grade: ",
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
cpus = input("CPU Units (leave empty for 0.25): ")
|
|
194
|
+
cpus = cpus if cpus else "0.25"
|
|
195
|
+
|
|
196
|
+
if cpus not in ["0.25", "0.5", "0.75", "1"]:
|
|
197
|
+
print("invalid option selected")
|
|
198
|
+
sys.exit(1)
|
|
199
|
+
|
|
200
|
+
soft_memory = input("Memory Reservation (leave empty for 512m): ")
|
|
201
|
+
soft_memory = soft_memory if soft_memory else "512m"
|
|
202
|
+
|
|
203
|
+
if soft_memory not in [
|
|
204
|
+
"512m",
|
|
205
|
+
"768m",
|
|
206
|
+
"1024m",
|
|
207
|
+
"2048m",
|
|
208
|
+
"4096m",
|
|
209
|
+
"8192m",
|
|
210
|
+
"1g",
|
|
211
|
+
"2g",
|
|
212
|
+
"4g",
|
|
213
|
+
"8g",
|
|
214
|
+
]:
|
|
215
|
+
print("invalid option selected")
|
|
216
|
+
sys.exit(1)
|
|
217
|
+
|
|
218
|
+
hard_memory = input("Memory Limit (leave empty for 1g): ")
|
|
219
|
+
hard_memory = hard_memory if hard_memory else "1g"
|
|
220
|
+
|
|
221
|
+
if hard_memory not in [
|
|
222
|
+
"1024m",
|
|
223
|
+
"2048m",
|
|
224
|
+
"4096m",
|
|
225
|
+
"8192m",
|
|
226
|
+
"15000m",
|
|
227
|
+
"1g",
|
|
228
|
+
"2g",
|
|
229
|
+
"4g",
|
|
230
|
+
"8g",
|
|
231
|
+
"15g",
|
|
232
|
+
]:
|
|
233
|
+
print("invalid option selected")
|
|
234
|
+
sys.exit(1)
|
|
235
|
+
|
|
236
|
+
if grader_mvp == "y":
|
|
237
|
+
unit_test_filename = "unittests"
|
|
238
|
+
version = "1"
|
|
239
|
+
data_dir_required = "n"
|
|
240
|
+
sol_dir_required = "n"
|
|
241
|
+
non_notebook_grading = "n"
|
|
242
|
+
extra_file_name = ""
|
|
243
|
+
cpus = "0.25"
|
|
244
|
+
soft_memory = "512m"
|
|
245
|
+
hard_memory = "1g"
|
|
307
246
|
|
|
308
247
|
dockerfile = generate_dockerfile(
|
|
309
248
|
data_dir_required=data_dir_required,
|
|
@@ -318,121 +257,36 @@ def load_templates() -> dict[str, str]:
|
|
|
318
257
|
TAG_ID=V$(GRADER_VERSION)
|
|
319
258
|
SUB_DIR=mount
|
|
320
259
|
MEMORY_LIMIT=4096
|
|
260
|
+
HARD_MEMORY={hard_memory}
|
|
261
|
+
CPUS={cpus}
|
|
262
|
+
SOFT_MEMORY={soft_memory}
|
|
321
263
|
"""
|
|
322
264
|
|
|
323
|
-
assignment_name = f"C{course}M{module}_Assignment.ipynb"
|
|
324
|
-
|
|
325
|
-
copy_assignment_to_submission_sh = generate_copy_assignment_script(
|
|
326
|
-
extra_file_required=non_notebook_grading,
|
|
327
|
-
assignment_name=assignment_name,
|
|
328
|
-
extra_file_name=extra_file_name,
|
|
329
|
-
)
|
|
330
|
-
|
|
331
|
-
makefile = """
|
|
332
|
-
.PHONY: sync learner build debug-unsafe debug-safe grade versioning tag undeletable uneditable init upgrade coursera zip
|
|
333
|
-
|
|
334
|
-
include .conf
|
|
335
|
-
|
|
336
|
-
PARTIDS = 123 456
|
|
337
|
-
OS := $(shell uname)
|
|
338
|
-
|
|
339
|
-
sync:
|
|
340
|
-
cp mount/submission.ipynb ../$(ASSIGNMENT_NAME)_Solution.ipynb
|
|
341
|
-
cp learner/$(ASSIGNMENT_NAME).ipynb ../$(ASSIGNMENT_NAME).ipynb
|
|
342
|
-
cp mount/$(UNIT_TESTS_NAME).py ../$(UNIT_TESTS_NAME).py
|
|
343
|
-
|
|
344
|
-
learner:
|
|
345
|
-
dlai_grader --learner --output_notebook=./learner/$(ASSIGNMENT_NAME).ipynb
|
|
346
|
-
rsync -a --exclude="submission.ipynb" --exclude="__pycache__" --exclude=".mypy_cache" ./mount/ ./learner/
|
|
347
|
-
|
|
348
|
-
build:
|
|
349
|
-
docker build -t $(IMAGE_NAME):$(TAG_ID) .
|
|
350
|
-
|
|
351
|
-
debug-unsafe:
|
|
352
|
-
docker run -it --rm --mount type=bind,source=$(PWD)/mount,target=/shared/submission --mount type=bind,source=$(PWD),target=/grader/ --env-file $(PWD)/.env --entrypoint ash $(IMAGE_NAME):$(TAG_ID)
|
|
353
|
-
|
|
354
|
-
debug-safe:
|
|
355
|
-
docker run -it --rm --mount type=bind,source=$(PWD)/mount,target=/shared/submission --env-file $(PWD)/.env --entrypoint ash $(IMAGE_NAME):$(TAG_ID)
|
|
356
|
-
|
|
357
|
-
grade:
|
|
358
|
-
docker run -it --rm --mount type=bind,source=$(PWD)/mount,target=/shared/submission --env-file $(PWD)/.env --entrypoint ash $(IMAGE_NAME):$(TAG_ID) -c 'for partId in $(PARTIDS); do export partId=$$partId; echo "Processing part $$partId"; python entry.py; done'
|
|
265
|
+
# assignment_name = f"C{course}M{module}_Assignment.ipynb"
|
|
359
266
|
|
|
360
|
-
|
|
361
|
-
|
|
267
|
+
# copy_assignment_to_submission_sh = generate_copy_assignment_script(
|
|
268
|
+
# extra_file_required=non_notebook_grading,
|
|
269
|
+
# assignment_name=assignment_name,
|
|
270
|
+
# extra_file_name=extra_file_name,
|
|
271
|
+
# )
|
|
362
272
|
|
|
363
|
-
|
|
364
|
-
dlai_grader --tag
|
|
273
|
+
makefile = copy_makefile()
|
|
365
274
|
|
|
366
|
-
|
|
367
|
-
dlai_grader --undeletable
|
|
275
|
+
grader_py = copy_grader_py()
|
|
368
276
|
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
init:
|
|
373
|
-
dlai_grader --versioning
|
|
374
|
-
dlai_grader --tag
|
|
375
|
-
dlai_grader --undeletable
|
|
376
|
-
dlai_grader --uneditable
|
|
377
|
-
|
|
378
|
-
upgrade:
|
|
379
|
-
dlai_grader --upgrade
|
|
380
|
-
|
|
381
|
-
coursera:
|
|
382
|
-
dlai_grader --grade --partids="$(PARTIDS)" --docker=$(IMAGE_NAME):$(TAG_ID) --memory=$(MEMORY_LIMIT) --submission=$(SUB_DIR)
|
|
383
|
-
|
|
384
|
-
zip:
|
|
385
|
-
zip -r $(IMAGE_NAME)$(TAG_ID).zip .
|
|
386
|
-
|
|
387
|
-
"""
|
|
388
|
-
|
|
389
|
-
grader_py = """
|
|
390
|
-
from types import ModuleType, FunctionType
|
|
391
|
-
from dlai_grader.grading import test_case, object_to_grade
|
|
392
|
-
from dlai_grader.types import grading_function, grading_wrapper, learner_submission
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
def part_1(
|
|
396
|
-
learner_mod: learner_submission,
|
|
397
|
-
solution_mod: ModuleType | None = None,
|
|
398
|
-
) -> grading_function:
|
|
399
|
-
@object_to_grade(learner_mod, "learner_func")
|
|
400
|
-
def g(learner_func: FunctionType) -> list[test_case]:
|
|
401
|
-
|
|
402
|
-
cases: list[test_case] = []
|
|
403
|
-
|
|
404
|
-
t = test_case()
|
|
405
|
-
if not isinstance(learner_func, FunctionType):
|
|
406
|
-
t.fail()
|
|
407
|
-
t.msg = "learner_func has incorrect type"
|
|
408
|
-
t.want = FunctionType
|
|
409
|
-
t.got = type(learner_func)
|
|
410
|
-
return [t]
|
|
411
|
-
|
|
412
|
-
return cases
|
|
413
|
-
|
|
414
|
-
return g
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
def handle_part_id(part_id: str) -> grading_wrapper:
|
|
418
|
-
grader_dict: dict[str, grading_wrapper] = {
|
|
419
|
-
"": part_1,
|
|
420
|
-
}
|
|
421
|
-
return grader_dict[part_id]
|
|
422
|
-
"""
|
|
423
|
-
|
|
424
|
-
entry_py = generate_entry_py(
|
|
277
|
+
entry_py = copy_entry_script(
|
|
278
|
+
sol_dir_required=sol_dir_required,
|
|
425
279
|
non_notebook_grading=non_notebook_grading,
|
|
426
280
|
extra_file_name=extra_file_name,
|
|
427
281
|
)
|
|
428
282
|
|
|
429
283
|
template_dict = {
|
|
430
284
|
"dockerfile": dedent(dockerfile),
|
|
431
|
-
"makefile": dedent(makefile
|
|
285
|
+
"makefile": dedent(makefile),
|
|
432
286
|
"conf": dedent(conf[1:]),
|
|
433
|
-
"grader_py": dedent(grader_py
|
|
287
|
+
"grader_py": dedent(grader_py),
|
|
434
288
|
"entry_py": dedent(entry_py),
|
|
435
|
-
"copy_assignment_to_submission_sh": dedent(copy_assignment_to_submission_sh),
|
|
289
|
+
# "copy_assignment_to_submission_sh": dedent(copy_assignment_to_submission_sh),
|
|
436
290
|
}
|
|
437
291
|
|
|
438
292
|
return template_dict
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
dlai_grader/__init__.py,sha256=p-V3tVdD4pOdvAnNIMgMndm9u8oM-swvPyX7xrgkb0M,210
|
|
2
|
+
dlai_grader/cli.py,sha256=NIwboE-AFn1LXOFmF4O70Ow0fkRxgclG_eMwmWiua38,4917
|
|
3
|
+
dlai_grader/compiler.py,sha256=elbHNUCqBCoOOoNmMRXbgeNL0nt0RM57eZi0-6AqycA,3036
|
|
4
|
+
dlai_grader/config.py,sha256=DokK1tVF_r7v0p9tWpBN-7lOAlPmHSpFXDZiI8cGw7s,1821
|
|
5
|
+
dlai_grader/grading.py,sha256=BMIoZ_loQDuNCEk1Dj3on4IWz-FGgIbnhbDyk8HmQ7c,5041
|
|
6
|
+
dlai_grader/io.py,sha256=5hs7Bv5zfyHKZUrFP-2gVH7y9dbIFsyGiKgsqFYbrMs,9250
|
|
7
|
+
dlai_grader/notebook.py,sha256=MgxZFuetTXwwZ-HXSB5ItLVD_9LP45E0xHAngS0g4EU,12101
|
|
8
|
+
dlai_grader/templates.py,sha256=vtOic70NtJael8d5l9cX1KsNCwXKZrchhQgjHiSFhow,8898
|
|
9
|
+
dlai_grader/types.py,sha256=5uiFaF3aDn-vjxTp9ec-ND-PRqeeV2_NfPHS2ngGsRo,306
|
|
10
|
+
dlai_grader/templates/Makefile,sha256=PhRJ-87fU3IMqYMt9ChrAAfr2BPlk0uwogygLZItZL8,1795
|
|
11
|
+
dlai_grader/templates/grader.py,sha256=492Dzs3enoCGfDviq_mdnrzeF5e1qNl21i42M5tjv4Y,896
|
|
12
|
+
dlai_grader/templates/copy_assignment_sh/extrafile_n,sha256=qB9ZViBm69r69nT9WXDaJfrW57sVho03aCZXbTS-lTc,430
|
|
13
|
+
dlai_grader/templates/copy_assignment_sh/extrafile_y,sha256=Zh07IAViyN-rELp4_-ZOJJSmTIigh0GxHLexW0ZGnMQ,616
|
|
14
|
+
dlai_grader/templates/dockerfile/data_n_solution_n,sha256=mD_fHhkzrGCCG3EPwXmafYVn1AoIcgBmSZLX8eCeH_g,447
|
|
15
|
+
dlai_grader/templates/dockerfile/data_n_solution_y,sha256=9iSyDj3-Jwm6RyAHNtVUDeD-XrYODB61i-HB3HfA70g,480
|
|
16
|
+
dlai_grader/templates/dockerfile/data_y_solution_n,sha256=i9nWcTAUCQzyhx3kM1aH64j0dkEMlYlEb6FFiE5k5oc,472
|
|
17
|
+
dlai_grader/templates/dockerfile/data_y_solution_y,sha256=xs6p-puJ-j5AeIuHiESL7X9kNSFZVCZ3Wb9SAW9KlUU,505
|
|
18
|
+
dlai_grader/templates/entry_py/solution_n_file_n.py,sha256=Y9cuwGuCrRHWfWmcvNP-Gh8mFcJL_JE0e9Ed09O7x6Q,2349
|
|
19
|
+
dlai_grader/templates/entry_py/solution_n_file_y.py,sha256=yHXK4iRVaxlhAwq3FiOfFEJdSRApfOgs3FMQtIe-A_c,2984
|
|
20
|
+
dlai_grader/templates/entry_py/solution_y_file_n.py,sha256=MKVmgXJVC1UtxtChsah7jUQ6-SL93kafZ02KrSayN0k,2397
|
|
21
|
+
dlai_grader/templates/entry_py/solution_y_file_y.py,sha256=gLauAYo-bieHR4ad20M-Zh1jYzoaYOyXHVKCXZReJ_A,3032
|
|
22
|
+
dlai_grader-2.0b2.dist-info/licenses/LICENSE,sha256=a_kch_UqdJPtyxk35QJr9O84K_koPixqWPYW9On4-io,1072
|
|
23
|
+
dlai_grader-2.0b2.dist-info/METADATA,sha256=V5bDW25vUyH7zf7mhe5hZKu-zTqxSxwIYYEBrYiaXU0,8776
|
|
24
|
+
dlai_grader-2.0b2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
25
|
+
dlai_grader-2.0b2.dist-info/entry_points.txt,sha256=4OcSAUIluONXa3ymViQ7CBQ2Lk52nb6xZnfph1rlMnk,71
|
|
26
|
+
dlai_grader-2.0b2.dist-info/top_level.txt,sha256=4YKtA3ztisFtx_g4hsGivy3J2NHnXxFziIMqawC8HWg,12
|
|
27
|
+
dlai_grader-2.0b2.dist-info/RECORD,,
|
dlai_grader/py.typed
DELETED
|
File without changes
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
dlai_grader/__init__.py,sha256=V0DW6eFVqfyWdxFvJavFxD2QV63rZCpT400IzY1QBkk,210
|
|
2
|
-
dlai_grader/cli.py,sha256=NIwboE-AFn1LXOFmF4O70Ow0fkRxgclG_eMwmWiua38,4917
|
|
3
|
-
dlai_grader/compiler.py,sha256=elbHNUCqBCoOOoNmMRXbgeNL0nt0RM57eZi0-6AqycA,3036
|
|
4
|
-
dlai_grader/config.py,sha256=DokK1tVF_r7v0p9tWpBN-7lOAlPmHSpFXDZiI8cGw7s,1821
|
|
5
|
-
dlai_grader/grading.py,sha256=BMIoZ_loQDuNCEk1Dj3on4IWz-FGgIbnhbDyk8HmQ7c,5041
|
|
6
|
-
dlai_grader/io.py,sha256=TEp5R8014EL9e-_RID5qp9INOujAGi7KLuy-SWWsPkM,8654
|
|
7
|
-
dlai_grader/notebook.py,sha256=MgxZFuetTXwwZ-HXSB5ItLVD_9LP45E0xHAngS0g4EU,12101
|
|
8
|
-
dlai_grader/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
|
-
dlai_grader/templates.py,sha256=SEMv4KFlq_2jy4UORMzHStOOex9P6LXr48Ie-r1JZcc,14627
|
|
10
|
-
dlai_grader/types.py,sha256=5uiFaF3aDn-vjxTp9ec-ND-PRqeeV2_NfPHS2ngGsRo,306
|
|
11
|
-
dlai_grader-2.0b1.dist-info/licenses/LICENSE,sha256=a_kch_UqdJPtyxk35QJr9O84K_koPixqWPYW9On4-io,1072
|
|
12
|
-
dlai_grader-2.0b1.dist-info/METADATA,sha256=0-7wdm6Otf4nHWASbjMHMBt8ruXvl3RrDakgAVPCMlA,8776
|
|
13
|
-
dlai_grader-2.0b1.dist-info/WHEEL,sha256=lTU6B6eIfYoiQJTZNc-fyaR6BpL6ehTzU3xGYxn2n8k,91
|
|
14
|
-
dlai_grader-2.0b1.dist-info/entry_points.txt,sha256=4OcSAUIluONXa3ymViQ7CBQ2Lk52nb6xZnfph1rlMnk,71
|
|
15
|
-
dlai_grader-2.0b1.dist-info/top_level.txt,sha256=4YKtA3ztisFtx_g4hsGivy3J2NHnXxFziIMqawC8HWg,12
|
|
16
|
-
dlai_grader-2.0b1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|