dlai-grader 1.8.0__py3-none-any.whl → 1.19.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dlai-grader might be problematic. Click here for more details.
- dlai_grader/__init__.py +1 -1
- dlai_grader/cli.py +19 -1
- dlai_grader/grading.py +45 -4
- dlai_grader/io.py +29 -3
- dlai_grader/notebook.py +99 -6
- dlai_grader/py.typed +0 -0
- dlai_grader/templates.py +199 -96
- {dlai_grader-1.8.0.dist-info → dlai_grader-1.19.0.dist-info}/METADATA +4 -7
- dlai_grader-1.19.0.dist-info/RECORD +16 -0
- {dlai_grader-1.8.0.dist-info → dlai_grader-1.19.0.dist-info}/WHEEL +1 -1
- {dlai_grader-1.8.0.dist-info → dlai_grader-1.19.0.dist-info}/entry_points.txt +0 -1
- dlai_grader-1.8.0.dist-info/RECORD +0 -15
- {dlai_grader-1.8.0.dist-info → dlai_grader-1.19.0.dist-info}/LICENSE +0 -0
- {dlai_grader-1.8.0.dist-info → dlai_grader-1.19.0.dist-info}/top_level.txt +0 -0
dlai_grader/__init__.py
CHANGED
dlai_grader/cli.py
CHANGED
|
@@ -4,6 +4,8 @@ from .io import (
|
|
|
4
4
|
update_grader_and_notebook_version,
|
|
5
5
|
update_notebook_version,
|
|
6
6
|
tag_notebook,
|
|
7
|
+
undeletable_notebook,
|
|
8
|
+
uneditable_notebook,
|
|
7
9
|
init_grader,
|
|
8
10
|
generate_learner_version,
|
|
9
11
|
grade_parts,
|
|
@@ -40,6 +42,18 @@ def parse_dlai_grader_args() -> None:
|
|
|
40
42
|
action="store_true",
|
|
41
43
|
help="Add graded tag to all code cells of notebook.",
|
|
42
44
|
)
|
|
45
|
+
parser.add_argument(
|
|
46
|
+
"-ud",
|
|
47
|
+
"--undeletable",
|
|
48
|
+
action="store_true",
|
|
49
|
+
help="Make all code cells of notebook not deletable.",
|
|
50
|
+
)
|
|
51
|
+
parser.add_argument(
|
|
52
|
+
"-ue",
|
|
53
|
+
"--uneditable",
|
|
54
|
+
action="store_true",
|
|
55
|
+
help="Make all non-graded code cells of notebook not editable.",
|
|
56
|
+
)
|
|
43
57
|
parser.add_argument(
|
|
44
58
|
"-l",
|
|
45
59
|
"--learner",
|
|
@@ -100,8 +114,12 @@ def parse_dlai_grader_args() -> None:
|
|
|
100
114
|
)
|
|
101
115
|
if args.tag:
|
|
102
116
|
tag_notebook("./mount/submission.ipynb")
|
|
103
|
-
if args.
|
|
117
|
+
if args.undeletable:
|
|
118
|
+
undeletable_notebook("./mount/submission.ipynb")
|
|
119
|
+
if args.uneditable:
|
|
120
|
+
uneditable_notebook("./mount/submission.ipynb")
|
|
104
121
|
|
|
122
|
+
if args.learner:
|
|
105
123
|
filename_source = "./mount/submission.ipynb"
|
|
106
124
|
filename_target = f"./learner/{c.assignment_name}.ipynb"
|
|
107
125
|
|
dlai_grader/grading.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from dataclasses import dataclass
|
|
2
2
|
from functools import wraps
|
|
3
|
+
from typing import Any, Callable, List, Tuple, Union
|
|
3
4
|
from types import ModuleType
|
|
4
|
-
from typing import Any, Callable, List, Tuple
|
|
5
5
|
|
|
6
6
|
|
|
7
7
|
@dataclass
|
|
@@ -11,6 +11,9 @@ class LearnerSubmission:
|
|
|
11
11
|
submission: Any = None
|
|
12
12
|
|
|
13
13
|
|
|
14
|
+
learner_submission = Union[ModuleType, LearnerSubmission]
|
|
15
|
+
|
|
16
|
+
|
|
14
17
|
@dataclass
|
|
15
18
|
class test_case:
|
|
16
19
|
"""Class that represents a test case"""
|
|
@@ -40,7 +43,13 @@ def compute_grading_score(
|
|
|
40
43
|
"""
|
|
41
44
|
|
|
42
45
|
num_cases = len(test_cases)
|
|
43
|
-
|
|
46
|
+
if num_cases == 0:
|
|
47
|
+
return (
|
|
48
|
+
0.0,
|
|
49
|
+
"The grader was unable to generate test cases for your implementation. This suggests a bug with your code, please revise your solution and try again.",
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
failed_cases = [t for t in test_cases if t.failed]
|
|
44
53
|
score = 1.0 - len(failed_cases) / num_cases
|
|
45
54
|
feedback_msg = "All tests passed! Congratulations!"
|
|
46
55
|
|
|
@@ -68,7 +77,7 @@ def compute_aggregated_grading_score(
|
|
|
68
77
|
for test_cases in aggregated_test_cases:
|
|
69
78
|
feedback_msg = f"All tests passed for {test_cases.test_name}!\n"
|
|
70
79
|
num_cases = len(test_cases.tests)
|
|
71
|
-
failed_cases = [t for t in test_cases.tests if t.failed
|
|
80
|
+
failed_cases = [t for t in test_cases.tests if t.failed]
|
|
72
81
|
score = 1.0 - len(failed_cases) / num_cases
|
|
73
82
|
score = round(score, 2)
|
|
74
83
|
scores.append(score)
|
|
@@ -87,7 +96,7 @@ def compute_aggregated_grading_score(
|
|
|
87
96
|
|
|
88
97
|
|
|
89
98
|
def object_to_grade(
|
|
90
|
-
origin_module:
|
|
99
|
+
origin_module: learner_submission,
|
|
91
100
|
*attr_names: str,
|
|
92
101
|
) -> Callable[[Callable[[Any], List[test_case]]], Callable[[Any], List[test_case]]]:
|
|
93
102
|
"""Used as a parameterized decorator to get any number of attributes from a module.
|
|
@@ -109,3 +118,35 @@ def object_to_grade(
|
|
|
109
118
|
return wrapper
|
|
110
119
|
|
|
111
120
|
return middle
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def print_feedback(test_cases: List[test_case]) -> None:
|
|
124
|
+
"""Prints feedback of public unit tests within notebook.
|
|
125
|
+
|
|
126
|
+
Args:
|
|
127
|
+
test_cases (List[test_case]): List of public test cases.
|
|
128
|
+
"""
|
|
129
|
+
failed_cases = [t for t in test_cases if t.failed]
|
|
130
|
+
feedback_msg = "\033[92m All tests passed!"
|
|
131
|
+
|
|
132
|
+
if failed_cases:
|
|
133
|
+
feedback_msg = ""
|
|
134
|
+
for failed_case in failed_cases:
|
|
135
|
+
feedback_msg += f"\033[91mFailed test case: {failed_case.msg}.\nExpected: {failed_case.want}\nGot: {failed_case.got}\n\n"
|
|
136
|
+
|
|
137
|
+
print(feedback_msg)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def graded_obj_missing(test_cases: List[test_case]) -> bool:
|
|
141
|
+
"""Check if the object to grade was found in the learned module.
|
|
142
|
+
|
|
143
|
+
Args:
|
|
144
|
+
test_cases (List[test_case]): List of test cases.
|
|
145
|
+
|
|
146
|
+
Returns:
|
|
147
|
+
bool: True if object is missing. False otherwise.
|
|
148
|
+
"""
|
|
149
|
+
if len(test_cases) == 1 and test_cases[0].got == type(None):
|
|
150
|
+
return True
|
|
151
|
+
|
|
152
|
+
return False
|
dlai_grader/io.py
CHANGED
|
@@ -10,7 +10,12 @@ from textwrap import dedent
|
|
|
10
10
|
from zipfile import ZipFile
|
|
11
11
|
from nbformat.notebooknode import NotebookNode
|
|
12
12
|
from contextlib import contextmanager, redirect_stderr, redirect_stdout
|
|
13
|
-
from .notebook import
|
|
13
|
+
from .notebook import (
|
|
14
|
+
add_metadata_all_code_cells,
|
|
15
|
+
add_metadata_code_cells_without_pattern,
|
|
16
|
+
tag_code_cells,
|
|
17
|
+
solution_to_learner_format,
|
|
18
|
+
)
|
|
14
19
|
from .templates import load_templates
|
|
15
20
|
|
|
16
21
|
|
|
@@ -47,6 +52,28 @@ def tag_notebook(
|
|
|
47
52
|
jupytext.write(nb, path)
|
|
48
53
|
|
|
49
54
|
|
|
55
|
+
def undeletable_notebook(path: str) -> None:
|
|
56
|
+
"""Makes all code cells of a notebook non-deletable.
|
|
57
|
+
|
|
58
|
+
Args:
|
|
59
|
+
path (str): Path to the notebook.
|
|
60
|
+
"""
|
|
61
|
+
nb = read_notebook(path)
|
|
62
|
+
nb = add_metadata_all_code_cells(nb, {"deletable": False})
|
|
63
|
+
jupytext.write(nb, path)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def uneditable_notebook(path: str) -> None:
|
|
67
|
+
"""Makes all non-graded code cells of a notebook non-editable.
|
|
68
|
+
|
|
69
|
+
Args:
|
|
70
|
+
path (str): Path to the notebook.
|
|
71
|
+
"""
|
|
72
|
+
nb = read_notebook(path)
|
|
73
|
+
nb = add_metadata_code_cells_without_pattern(nb, {"editable": False})
|
|
74
|
+
jupytext.write(nb, path)
|
|
75
|
+
|
|
76
|
+
|
|
50
77
|
def extract_tar(
|
|
51
78
|
file_path: str,
|
|
52
79
|
destination: str,
|
|
@@ -199,7 +226,7 @@ def init_grader() -> None:
|
|
|
199
226
|
write_file_from_template("./grader.py", template_dict["grader_py"])
|
|
200
227
|
write_file_from_template("./Makefile", template_dict["makefile"])
|
|
201
228
|
write_file_from_template("./.conf", template_dict["conf"])
|
|
202
|
-
write_file_from_template("./entry.py", "")
|
|
229
|
+
write_file_from_template("./entry.py", template_dict["entry_py"])
|
|
203
230
|
write_file_from_template("./requirements.txt", "dlai-grader")
|
|
204
231
|
os.makedirs("data")
|
|
205
232
|
os.makedirs("learner")
|
|
@@ -245,7 +272,6 @@ def grade_parts(
|
|
|
245
272
|
"""
|
|
246
273
|
|
|
247
274
|
for p in partids.split(" "):
|
|
248
|
-
|
|
249
275
|
print(f"\nGrading part_id: {p}\n")
|
|
250
276
|
cmd = """coursera_autograder grade local $2 $3 '{"partId": "$1", "fileName": "submission.ipynb"}' --mem-limit $4"""
|
|
251
277
|
|
dlai_grader/notebook.py
CHANGED
|
@@ -19,7 +19,7 @@ def notebook_to_script(
|
|
|
19
19
|
|
|
20
20
|
def cut_notebook(
|
|
21
21
|
regex_pattern: str = "(grade)(.|[ \t]*)(up)(.|[ \t]*)(to)(.|[ \t]*)(here)",
|
|
22
|
-
) -> NotebookNode:
|
|
22
|
+
) -> Callable[[NotebookNode], NotebookNode]:
|
|
23
23
|
"""Cuts a notebook, this allows for partial grading. Written as a closure so it can be consumed as a functional option for notebooks.
|
|
24
24
|
Args:
|
|
25
25
|
regex_pattern (str): Regexp pattern to look for. Cells after match will be omitted.
|
|
@@ -29,7 +29,7 @@ def cut_notebook(
|
|
|
29
29
|
|
|
30
30
|
def inner(
|
|
31
31
|
notebook: NotebookNode,
|
|
32
|
-
):
|
|
32
|
+
) -> NotebookNode:
|
|
33
33
|
"""Cuts a notebook by excluding all cells after a pattern is matched.
|
|
34
34
|
Args:
|
|
35
35
|
notebook (NotebookNode): Notebook to filter.
|
|
@@ -116,6 +116,25 @@ def omit_tagged_cells(
|
|
|
116
116
|
return inner
|
|
117
117
|
|
|
118
118
|
|
|
119
|
+
def partial_grading_enabled(
|
|
120
|
+
notebook: NotebookNode,
|
|
121
|
+
regex_pattern: str = "(grade)(.|[ \t]*)(up)(.|[ \t]*)(to)(.|[ \t]*)(here)",
|
|
122
|
+
) -> bool:
|
|
123
|
+
"""Checks for the existence of the partial grading tag.
|
|
124
|
+
|
|
125
|
+
Args:
|
|
126
|
+
notebook (NotebookNode): The notebook from the learner.
|
|
127
|
+
regex_pattern (str, optional): Pattern to check. Defaults to "(grade)(.|[ \t]*)(up)(.|[ \t]*)(to)(.|[ \t]*)(here)".
|
|
128
|
+
|
|
129
|
+
Returns:
|
|
130
|
+
bool: True if the pattern if found, False otherwise.
|
|
131
|
+
"""
|
|
132
|
+
for cell in notebook["cells"]:
|
|
133
|
+
if cell["cell_type"] == "code" and re.search(regex_pattern, cell["source"]):
|
|
134
|
+
return True
|
|
135
|
+
return False
|
|
136
|
+
|
|
137
|
+
|
|
119
138
|
def get_named_cells(
|
|
120
139
|
notebook: NotebookNode,
|
|
121
140
|
) -> dict:
|
|
@@ -149,9 +168,7 @@ def tag_code_cells(
|
|
|
149
168
|
filtered_cells = []
|
|
150
169
|
|
|
151
170
|
for cell in notebook["cells"]:
|
|
152
|
-
|
|
153
171
|
if cell["cell_type"] == "code":
|
|
154
|
-
|
|
155
172
|
if not "tags" in cell["metadata"]:
|
|
156
173
|
cell["metadata"]["tags"] = []
|
|
157
174
|
|
|
@@ -168,6 +185,84 @@ def tag_code_cells(
|
|
|
168
185
|
return notebook
|
|
169
186
|
|
|
170
187
|
|
|
188
|
+
def add_metadata_all_code_cells(
|
|
189
|
+
notebook: NotebookNode,
|
|
190
|
+
metadata: dict,
|
|
191
|
+
) -> NotebookNode:
|
|
192
|
+
"""Adds metadata to all code cells of a notebook.
|
|
193
|
+
Args:
|
|
194
|
+
notebook (NotebookNode): Notebook to filter.
|
|
195
|
+
metadata (dict): The metadata which should be a key-value pair.
|
|
196
|
+
Returns:
|
|
197
|
+
NotebookNode: The notebook with the new metadata.
|
|
198
|
+
"""
|
|
199
|
+
filtered_cells = []
|
|
200
|
+
|
|
201
|
+
for cell in notebook["cells"]:
|
|
202
|
+
if cell["cell_type"] == "code":
|
|
203
|
+
current_metadata = cell["metadata"]
|
|
204
|
+
current_metadata.update(metadata)
|
|
205
|
+
cell["metadata"] = current_metadata
|
|
206
|
+
|
|
207
|
+
filtered_cells.append(cell)
|
|
208
|
+
|
|
209
|
+
notebook["cells"] = filtered_cells
|
|
210
|
+
|
|
211
|
+
return notebook
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def add_metadata_code_cells_with_pattern(
|
|
215
|
+
notebook: NotebookNode, metadata: dict, regex_pattern: str = "w[1-9]_unittest"
|
|
216
|
+
) -> NotebookNode:
|
|
217
|
+
"""Adds metadata to code cells of a notebook that match a regexp pattern.
|
|
218
|
+
Args:
|
|
219
|
+
notebook (NotebookNode): Notebook to filter.
|
|
220
|
+
metadata (dict): The metadata which should be a key-value pair.
|
|
221
|
+
regex_pattern (str, optional): Pattern to check. Defaults to "w[1-9]_unittest".
|
|
222
|
+
Returns:
|
|
223
|
+
NotebookNode: The notebook with the new metadata.
|
|
224
|
+
"""
|
|
225
|
+
filtered_cells = []
|
|
226
|
+
|
|
227
|
+
for cell in notebook["cells"]:
|
|
228
|
+
if cell["cell_type"] == "code" and re.search(regex_pattern, cell["source"]):
|
|
229
|
+
current_metadata = cell["metadata"]
|
|
230
|
+
current_metadata.update(metadata)
|
|
231
|
+
cell["metadata"] = current_metadata
|
|
232
|
+
|
|
233
|
+
filtered_cells.append(cell)
|
|
234
|
+
|
|
235
|
+
notebook["cells"] = filtered_cells
|
|
236
|
+
|
|
237
|
+
return notebook
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def add_metadata_code_cells_without_pattern(
|
|
241
|
+
notebook: NotebookNode, metadata: dict, regex_pattern: str = "^# GRADED "
|
|
242
|
+
) -> NotebookNode:
|
|
243
|
+
"""Adds metadata to code cells of a notebook that don't match a regexp pattern.
|
|
244
|
+
Args:
|
|
245
|
+
notebook (NotebookNode): Notebook to filter.
|
|
246
|
+
metadata (dict): The metadata which should be a key-value pair.
|
|
247
|
+
regex_pattern (str, optional): Pattern to check. Defaults to "w[1-9]_unittest".
|
|
248
|
+
Returns:
|
|
249
|
+
NotebookNode: The notebook with the new metadata.
|
|
250
|
+
"""
|
|
251
|
+
filtered_cells = []
|
|
252
|
+
|
|
253
|
+
for cell in notebook["cells"]:
|
|
254
|
+
if cell["cell_type"] == "code" and not re.search(regex_pattern, cell["source"]):
|
|
255
|
+
current_metadata = cell["metadata"]
|
|
256
|
+
current_metadata.update(metadata)
|
|
257
|
+
cell["metadata"] = current_metadata
|
|
258
|
+
|
|
259
|
+
filtered_cells.append(cell)
|
|
260
|
+
|
|
261
|
+
notebook["cells"] = filtered_cells
|
|
262
|
+
|
|
263
|
+
return notebook
|
|
264
|
+
|
|
265
|
+
|
|
171
266
|
def notebook_version(
|
|
172
267
|
notebook: NotebookNode,
|
|
173
268
|
) -> str:
|
|
@@ -250,7 +345,6 @@ def solution_to_learner_format(
|
|
|
250
345
|
formatted_lines = []
|
|
251
346
|
|
|
252
347
|
for ln in cell_code.splitlines():
|
|
253
|
-
|
|
254
348
|
if tag_start in ln:
|
|
255
349
|
code_block = True
|
|
256
350
|
|
|
@@ -284,7 +378,6 @@ def solution_to_learner_format(
|
|
|
284
378
|
continue
|
|
285
379
|
|
|
286
380
|
if (tag_start not in ln) and code_block:
|
|
287
|
-
|
|
288
381
|
if tag_replace_equals in ln:
|
|
289
382
|
splitted = ln.split(tag_replace_equals)
|
|
290
383
|
new_val = splitted[1]
|
dlai_grader/py.typed
ADDED
|
File without changes
|
dlai_grader/templates.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
from re import template
|
|
2
1
|
from textwrap import dedent
|
|
3
2
|
from typing import Dict
|
|
4
3
|
|
|
@@ -6,153 +5,257 @@ from typing import Dict
|
|
|
6
5
|
def load_templates() -> Dict[str, str]:
|
|
7
6
|
specialization = input("Name of the specialization: ")
|
|
8
7
|
course = input("Number of the course: ")
|
|
9
|
-
|
|
8
|
+
week_or_module = input("Weeks or Modules?\n1 for weeks\n2 for modules: ")
|
|
9
|
+
|
|
10
|
+
if week_or_module == "1":
|
|
11
|
+
week = input("Number of the week: ")
|
|
12
|
+
module = None
|
|
13
|
+
elif week_or_module == "2":
|
|
14
|
+
module = input("Number of the module: ")
|
|
15
|
+
week = None
|
|
16
|
+
else:
|
|
17
|
+
print("invalid option selected")
|
|
18
|
+
exit(1)
|
|
19
|
+
|
|
10
20
|
version = input("Version of the grader (leave empty for version 1): ")
|
|
11
21
|
version = "1" if not version else version
|
|
12
22
|
|
|
13
23
|
dockerfile = """
|
|
14
|
-
|
|
24
|
+
FROM continuumio/miniconda3@sha256:d601a04ea48fd45e60808c7072243d33703d29434d2067816b7f26b0705d889a
|
|
15
25
|
|
|
16
|
-
|
|
26
|
+
RUN apk update && apk add libstdc++
|
|
17
27
|
|
|
18
|
-
|
|
28
|
+
COPY requirements.txt .
|
|
19
29
|
|
|
20
|
-
|
|
21
|
-
|
|
30
|
+
RUN pip install -r requirements.txt && \
|
|
31
|
+
rm requirements.txt
|
|
22
32
|
|
|
23
|
-
|
|
24
|
-
|
|
33
|
+
RUN mkdir /grader && \
|
|
34
|
+
mkdir /grader/submission
|
|
25
35
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
36
|
+
COPY .conf /grader/.conf
|
|
37
|
+
COPY data/ /grader/data/
|
|
38
|
+
COPY solution/ /grader/solution/
|
|
39
|
+
COPY entry.py /grader/entry.py
|
|
40
|
+
COPY grader.py /grader/grader.py
|
|
31
41
|
|
|
32
|
-
|
|
42
|
+
RUN chmod a+rwx /grader/
|
|
33
43
|
|
|
34
|
-
|
|
44
|
+
WORKDIR /grader/
|
|
35
45
|
|
|
36
|
-
|
|
46
|
+
ENTRYPOINT ["python", "entry.py"]
|
|
37
47
|
"""
|
|
38
48
|
|
|
49
|
+
if week:
|
|
50
|
+
W_OR_M = "W"
|
|
51
|
+
W_OR_M_num = week
|
|
52
|
+
|
|
53
|
+
if module:
|
|
54
|
+
W_OR_M = "M"
|
|
55
|
+
W_OR_M_num = module
|
|
56
|
+
|
|
39
57
|
conf = f"""
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
LINUX_UPLOAD_DIR='/mnt/c/Users/Andres/dlai/upload'
|
|
47
|
-
MAC_UPLOAD_DIR='/Users/andreszarta/Desktop/upload-temp'
|
|
58
|
+
ASSIGNMENT_NAME=C{course}{W_OR_M}{W_OR_M_num}_Assignment
|
|
59
|
+
IMAGE_NAME={specialization}c{course}{W_OR_M.lower()}{W_OR_M_num}-grader
|
|
60
|
+
GRADER_VERSION={version}
|
|
61
|
+
TAG_ID=V$(GRADER_VERSION)
|
|
62
|
+
SUB_DIR=mount
|
|
63
|
+
MEMORY_LIMIT=4096
|
|
48
64
|
"""
|
|
49
65
|
|
|
50
66
|
makefile = """
|
|
51
|
-
|
|
67
|
+
.PHONY: learner build entry submit-solution upgrade test grade mem zip clean upload move-zip move-learner tag undeletable uneditable versioning upgrade sync
|
|
68
|
+
|
|
69
|
+
include .conf
|
|
52
70
|
|
|
53
|
-
|
|
71
|
+
PARTIDS = ""
|
|
72
|
+
OS := $(shell uname)
|
|
54
73
|
|
|
55
|
-
|
|
56
|
-
|
|
74
|
+
sync:
|
|
75
|
+
cp mount/submission.ipynb ../$(ASSIGNMENT_NAME)_Solution.ipynb
|
|
76
|
+
cp learner/$(ASSIGNMENT_NAME).ipynb ../$(ASSIGNMENT_NAME).ipynb
|
|
77
|
+
cp mount/unittests.py ../unittests.py
|
|
57
78
|
|
|
58
|
-
|
|
59
|
-
|
|
79
|
+
learner:
|
|
80
|
+
dlai_grader --learner --output_notebook=./learner/$(ASSIGNMENT_NAME).ipynb
|
|
60
81
|
|
|
61
|
-
|
|
62
|
-
|
|
82
|
+
build:
|
|
83
|
+
docker build -t $(IMAGE_NAME):$(TAG_ID) .
|
|
63
84
|
|
|
64
|
-
|
|
65
|
-
|
|
85
|
+
debug:
|
|
86
|
+
docker run -it --rm --mount type=bind,source=$(PWD)/mount,target=/shared/submission --mount type=bind,source=$(PWD),target=/grader/ --entrypoint ash $(IMAGE_NAME):$(TAG_ID)
|
|
66
87
|
|
|
67
|
-
|
|
68
|
-
|
|
88
|
+
submit-solution:
|
|
89
|
+
cp solution/solution.ipynb mount/submission.ipynb
|
|
69
90
|
|
|
70
|
-
|
|
71
|
-
|
|
91
|
+
versioning:
|
|
92
|
+
dlai_grader --versioning
|
|
72
93
|
|
|
73
|
-
|
|
74
|
-
|
|
94
|
+
tag:
|
|
95
|
+
dlai_grader --tag
|
|
75
96
|
|
|
76
|
-
|
|
77
|
-
|
|
97
|
+
undeletable:
|
|
98
|
+
dlai_grader --undeletable
|
|
78
99
|
|
|
79
|
-
|
|
80
|
-
|
|
100
|
+
uneditable:
|
|
101
|
+
dlai_grader --uneditable
|
|
81
102
|
|
|
82
|
-
|
|
83
|
-
|
|
103
|
+
upgrade:
|
|
104
|
+
dlai_grader --upgrade
|
|
84
105
|
|
|
85
|
-
|
|
86
|
-
|
|
106
|
+
test:
|
|
107
|
+
docker run -it --rm --mount type=bind,source=$(PWD)/mount,target=/shared/submission --mount type=bind,source=$(PWD),target=/grader/ --entrypoint pytest $(IMAGE_NAME):$(TAG_ID)
|
|
87
108
|
|
|
88
|
-
|
|
89
|
-
|
|
109
|
+
grade:
|
|
110
|
+
dlai_grader --grade --partids=$(PARTIDS) --docker=$(IMAGE_NAME):$(TAG_ID) --memory=$(MEMORY_LIMIT) --submission=$(SUB_DIR)
|
|
90
111
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
docker rm $$(docker ps -qa --no-trunc --filter "status=exited")
|
|
94
|
-
docker rmi $$(docker images --filter "dangling=true" -q --no-trunc)
|
|
112
|
+
mem:
|
|
113
|
+
memthis $(PARTIDS)
|
|
95
114
|
|
|
96
|
-
|
|
97
|
-
|
|
115
|
+
zip:
|
|
116
|
+
zip -r $(IMAGE_NAME)$(TAG_ID).zip .
|
|
98
117
|
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
118
|
+
clean:
|
|
119
|
+
find . -maxdepth 1 -type f -name "*.zip" -exec rm {} +
|
|
120
|
+
docker rm $$(docker ps -qa --no-trunc --filter "status=exited")
|
|
121
|
+
docker rmi $$(docker images --filter "dangling=true" -q --no-trunc)
|
|
122
|
+
|
|
123
|
+
upload:
|
|
124
|
+
coursera_autograder --timeout 1800 upload --grader-memory-limit $(MEMORY_LIMIT) --grading-timeout 1800 $(IMAGE_NAME)$(TAG_ID).zip $(COURSE_ID) $(ITEM_ID) $(PART_ID)
|
|
106
125
|
|
|
107
|
-
move-learner:
|
|
108
|
-
if [[ "$(OS)" == "Darwin" ]]; \
|
|
109
|
-
then \
|
|
110
|
-
cp learner/$(ASSIGNMENT_NAME).ipynb $(MAC_UPLOAD_DIR); \
|
|
111
|
-
else \
|
|
112
|
-
cp learner/$(ASSIGNMENT_NAME).ipynb $(LINUX_UPLOAD_DIR); \
|
|
113
|
-
fi
|
|
114
126
|
"""
|
|
115
127
|
|
|
116
128
|
grader_py = """
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
129
|
+
from types import ModuleType, FunctionType
|
|
130
|
+
from typing import Dict, List, Optional
|
|
131
|
+
from dlai_grader.grading import test_case, object_to_grade
|
|
132
|
+
from dlai_grader.types import grading_function, grading_wrapper, learner_submission
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def part_1(
|
|
136
|
+
learner_mod: learner_submission, solution_mod: Optional[ModuleType]
|
|
137
|
+
) -> grading_function:
|
|
138
|
+
@object_to_grade(learner_mod, "learner_func")
|
|
139
|
+
def g(learner_func: FunctionType) -> List[test_case]:
|
|
140
|
+
|
|
141
|
+
t = test_case()
|
|
142
|
+
if not isinstance(learner_func, FunctionType):
|
|
143
|
+
t.failed = True
|
|
144
|
+
t.msg = "learner_func has incorrect type"
|
|
145
|
+
t.want = FunctionType
|
|
146
|
+
t.got = type(learner_func)
|
|
147
|
+
return [t]
|
|
148
|
+
|
|
149
|
+
cases: List[test_case] = []
|
|
150
|
+
|
|
151
|
+
return cases
|
|
152
|
+
|
|
153
|
+
return g
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def handle_part_id(part_id: str) -> grading_wrapper:
|
|
157
|
+
grader_dict: Dict[str, grading_wrapper] = {
|
|
158
|
+
"": part_1,
|
|
159
|
+
}
|
|
160
|
+
return grader_dict[part_id]
|
|
161
|
+
"""
|
|
162
|
+
|
|
163
|
+
entry_py = """
|
|
164
|
+
from dlai_grader.config import Config
|
|
165
|
+
from dlai_grader.compiler import compile_module
|
|
166
|
+
from dlai_grader.io import read_notebook, copy_submission_to_workdir, send_feedback
|
|
121
167
|
|
|
168
|
+
from dlai_grader.notebook import (
|
|
169
|
+
notebook_to_script,
|
|
170
|
+
keep_tagged_cells,
|
|
171
|
+
notebook_is_up_to_date,
|
|
172
|
+
notebook_version,
|
|
173
|
+
cut_notebook,
|
|
174
|
+
partial_grading_enabled,
|
|
175
|
+
)
|
|
176
|
+
from dlai_grader.grading import compute_grading_score, graded_obj_missing
|
|
177
|
+
from grader import handle_part_id
|
|
122
178
|
|
|
123
|
-
def part_1(
|
|
124
|
-
learner_mod: ModuleType, solution_mod: Optional[ModuleType]
|
|
125
|
-
) -> grading_function:
|
|
126
|
-
@object_to_grade(learner_mod, "learner_func")
|
|
127
|
-
def g(learner_func: FunctionType) -> List[test_case]:
|
|
128
179
|
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
t.failed = True
|
|
132
|
-
t.msg = "learner_func has incorrect type"
|
|
133
|
-
t.want = FunctionType
|
|
134
|
-
t.got = type(learner_func)
|
|
135
|
-
return [t]
|
|
180
|
+
def main() -> None:
|
|
181
|
+
c = Config()
|
|
136
182
|
|
|
137
|
-
|
|
183
|
+
copy_submission_to_workdir()
|
|
138
184
|
|
|
139
|
-
|
|
185
|
+
try:
|
|
186
|
+
nb = read_notebook(c.submission_file_path)
|
|
187
|
+
except Exception as e:
|
|
188
|
+
send_feedback(
|
|
189
|
+
0.0,
|
|
190
|
+
f"There was a problem reading your notebook. Details:\\n{str(e)}",
|
|
191
|
+
err=True,
|
|
192
|
+
)
|
|
140
193
|
|
|
141
|
-
|
|
194
|
+
if not notebook_is_up_to_date(nb):
|
|
195
|
+
msg = f"You are submitting a version of the assignment that is behind the latest version.\\nThe latest version is {c.latest_version} and you are on version {notebook_version(nb)}."
|
|
142
196
|
|
|
197
|
+
send_feedback(0.0, msg)
|
|
143
198
|
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
199
|
+
transformations = [cut_notebook(), keep_tagged_cells()]
|
|
200
|
+
|
|
201
|
+
for t in transformations:
|
|
202
|
+
nb = t(nb)
|
|
203
|
+
|
|
204
|
+
script = notebook_to_script(nb)
|
|
205
|
+
|
|
206
|
+
try:
|
|
207
|
+
learner_mod = compile_module(script, "learner_mod", verbose=False)
|
|
208
|
+
except Exception as e:
|
|
209
|
+
send_feedback(
|
|
210
|
+
0.0,
|
|
211
|
+
f"There was a problem compiling the code from your notebook, please check that you saved before submitting. Details:\\n{str(e)}",
|
|
212
|
+
)
|
|
213
|
+
|
|
214
|
+
solution_nb = read_notebook(c.solution_file_path)
|
|
215
|
+
|
|
216
|
+
for t in transformations:
|
|
217
|
+
solution_nb = t(solution_nb)
|
|
218
|
+
|
|
219
|
+
solution_script = notebook_to_script(solution_nb)
|
|
220
|
+
solution_mod = compile_module(solution_script, "solution_mod", verbose=False)
|
|
221
|
+
|
|
222
|
+
g_func = handle_part_id(c.part_id)(learner_mod, solution_mod)
|
|
223
|
+
|
|
224
|
+
try:
|
|
225
|
+
cases = g_func()
|
|
226
|
+
except Exception as e:
|
|
227
|
+
send_feedback(
|
|
228
|
+
0.0,
|
|
229
|
+
f"There was an error grading your submission. Details:\\n{str(e)}",
|
|
230
|
+
err=True,
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
if graded_obj_missing(cases):
|
|
234
|
+
additional_msg = ""
|
|
235
|
+
if partial_grading_enabled(nb):
|
|
236
|
+
additional_msg = "The # grade-up-to-here comment in the notebook might be causing the problem."
|
|
237
|
+
|
|
238
|
+
send_feedback(
|
|
239
|
+
0.0,
|
|
240
|
+
f"Unable to find object required for grading in your code.\\n{additional_msg}",
|
|
241
|
+
err=True,
|
|
242
|
+
)
|
|
243
|
+
|
|
244
|
+
score, feedback = compute_grading_score(cases)
|
|
245
|
+
|
|
246
|
+
send_feedback(score, feedback)
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
if __name__ == "__main__":
|
|
250
|
+
main()
|
|
251
|
+
"""
|
|
150
252
|
|
|
151
253
|
template_dict = {
|
|
152
254
|
"dockerfile": dedent(dockerfile[1:]),
|
|
153
255
|
"makefile": dedent(makefile[1:]),
|
|
154
256
|
"conf": dedent(conf[1:]),
|
|
155
257
|
"grader_py": dedent(grader_py[1:]),
|
|
258
|
+
"entry_py": dedent(entry_py[1:]),
|
|
156
259
|
}
|
|
157
260
|
|
|
158
261
|
return template_dict
|
|
@@ -1,19 +1,18 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: dlai-grader
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.19.0
|
|
4
4
|
Summary: Grading utilities for DLAI courses
|
|
5
5
|
Home-page: https://github.com/https-deeplearning-ai/grader
|
|
6
6
|
Author: Andres Zarta
|
|
7
7
|
Author-email: andrezb5@gmail.com
|
|
8
8
|
License: MIT License
|
|
9
|
-
Platform: UNKNOWN
|
|
10
9
|
Classifier: Programming Language :: Python :: 3
|
|
11
10
|
Classifier: License :: OSI Approved :: MIT License
|
|
12
11
|
Classifier: Operating System :: OS Independent
|
|
13
12
|
Description-Content-Type: text/markdown
|
|
14
13
|
License-File: LICENSE
|
|
15
|
-
Requires-Dist: nbformat
|
|
16
|
-
Requires-Dist: jupytext
|
|
14
|
+
Requires-Dist: nbformat >=5.1.3
|
|
15
|
+
Requires-Dist: jupytext >=1.13.0
|
|
17
16
|
|
|
18
17
|
# grader
|
|
19
18
|
Automatic grading for DLAI courses. Designed to be compatible with Coursera's grading requirements.
|
|
@@ -43,7 +42,7 @@ dlai_grader --init
|
|
|
43
42
|
This will ask you for:
|
|
44
43
|
- Name of the course (abbreviation is recommended)
|
|
45
44
|
- Number of the course
|
|
46
|
-
- Number of the week
|
|
45
|
+
- Number of the week or module
|
|
47
46
|
- Version of the grader (defaults to 1 but can be 2 or any other number)
|
|
48
47
|
|
|
49
48
|
This will generate a file system tree like this:
|
|
@@ -185,5 +184,3 @@ print(prediction)
|
|
|
185
184
|
|
|
186
185
|
## Download the assignment
|
|
187
186
|
|
|
188
|
-
|
|
189
|
-
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
dlai_grader/__init__.py,sha256=edIpwgi8qwJ1gBBlMmOfhnWsSLg7PdAjCUUpY0aPxHY,211
|
|
2
|
+
dlai_grader/cli.py,sha256=NIwboE-AFn1LXOFmF4O70Ow0fkRxgclG_eMwmWiua38,4917
|
|
3
|
+
dlai_grader/compiler.py,sha256=FiORpcrAbxWKRuOnyADrknTOwsIXb_2PFO26mVaLTt4,1315
|
|
4
|
+
dlai_grader/config.py,sha256=HQ3dzaFpRswIA_7EC8XdP8DdJH-XePsbMQMHG8Esblc,1638
|
|
5
|
+
dlai_grader/grading.py,sha256=Gmft9b7M8At_y_WZDatYdW6tinZMfqQoT7bDXp6uz2I,4606
|
|
6
|
+
dlai_grader/io.py,sha256=TB9d01AK5FIbFUQwM8AqOOfuMWzjzrit98i3MhK5AqU,8234
|
|
7
|
+
dlai_grader/notebook.py,sha256=noMU6DzPVylSjkHmSBUcmquVvAz4JigbRtbQrVYJdic,11830
|
|
8
|
+
dlai_grader/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
|
+
dlai_grader/templates.py,sha256=0S6smsno2jjfzD4nGR96v33crES7F0KMZFHusmEOCgU,7746
|
|
10
|
+
dlai_grader/types.py,sha256=_IIVbYL9cMmwA6in0aI5fEWCIaAMNcQbxG64X1P1CkE,335
|
|
11
|
+
dlai_grader-1.19.0.dist-info/LICENSE,sha256=a_kch_UqdJPtyxk35QJr9O84K_koPixqWPYW9On4-io,1072
|
|
12
|
+
dlai_grader-1.19.0.dist-info/METADATA,sha256=Ag6GmW4T1gtHmnObCvLuyP6BQnn0GQTWMEZFhMAZRAc,8620
|
|
13
|
+
dlai_grader-1.19.0.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
|
|
14
|
+
dlai_grader-1.19.0.dist-info/entry_points.txt,sha256=4OcSAUIluONXa3ymViQ7CBQ2Lk52nb6xZnfph1rlMnk,71
|
|
15
|
+
dlai_grader-1.19.0.dist-info/top_level.txt,sha256=4YKtA3ztisFtx_g4hsGivy3J2NHnXxFziIMqawC8HWg,12
|
|
16
|
+
dlai_grader-1.19.0.dist-info/RECORD,,
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
dlai_grader/__init__.py,sha256=I1kCwRVAXbhf5eiEI3wSGJnlDYNrmcoHSQtH2enTjus,210
|
|
2
|
-
dlai_grader/cli.py,sha256=kJfpLsHwAV9QevP9cd_MqF7x1yxaO0l09ZT-L_9IXBc,4369
|
|
3
|
-
dlai_grader/compiler.py,sha256=FiORpcrAbxWKRuOnyADrknTOwsIXb_2PFO26mVaLTt4,1315
|
|
4
|
-
dlai_grader/config.py,sha256=HQ3dzaFpRswIA_7EC8XdP8DdJH-XePsbMQMHG8Esblc,1638
|
|
5
|
-
dlai_grader/grading.py,sha256=9NPCtFwXEj4M-t5QPzvJH6Rr-BE9QHRZp_HoksniKUI,3376
|
|
6
|
-
dlai_grader/io.py,sha256=yk84G2CjltZ34w9bLSG3CZ1uGZ5DsHdF6X-p3Yk4Yhk,7529
|
|
7
|
-
dlai_grader/notebook.py,sha256=VShLiSuNswHS_WaCuDV_4OrXDpk74mFr1kLGa4Vf3AM,8599
|
|
8
|
-
dlai_grader/templates.py,sha256=OiMfG1gR_QaZjyAfZS7Mfxkzk1D9KrBpdqyriGONWhs,4119
|
|
9
|
-
dlai_grader/types.py,sha256=_IIVbYL9cMmwA6in0aI5fEWCIaAMNcQbxG64X1P1CkE,335
|
|
10
|
-
dlai_grader-1.8.0.dist-info/LICENSE,sha256=a_kch_UqdJPtyxk35QJr9O84K_koPixqWPYW9On4-io,1072
|
|
11
|
-
dlai_grader-1.8.0.dist-info/METADATA,sha256=JFAm26MQuvTO7aBrorX8sesDSPeIJaU9ge7M31amHi8,8633
|
|
12
|
-
dlai_grader-1.8.0.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
|
|
13
|
-
dlai_grader-1.8.0.dist-info/entry_points.txt,sha256=kbbh-PzqkWSypl2fOk0n8tbfcMCXUd7yiF97kC-E47I,72
|
|
14
|
-
dlai_grader-1.8.0.dist-info/top_level.txt,sha256=4YKtA3ztisFtx_g4hsGivy3J2NHnXxFziIMqawC8HWg,12
|
|
15
|
-
dlai_grader-1.8.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|