dlai-grader 2.0b1__tar.gz → 2.0b2__tar.gz

dlai_grader-2.0b2/MANIFEST.in

@@ -0,0 +1,3 @@
+recursive-include dlai_grader/templates *
+
+include dlai_grader/py.typed

{dlai_grader-2.0b1 → dlai_grader-2.0b2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dlai-grader
-Version: 2.0b1
+Version: 2.0b2
 Summary: Grading utilities for DLAI courses
 Home-page: https://github.com/https-deeplearning-ai/grader
 Author: Andres Zarta

{dlai_grader-2.0b1 → dlai_grader-2.0b2}/dlai_grader/__init__.py

@@ -6,6 +6,6 @@ from . import grading
 from . import types
 
 
-__version__ = "2.0b1"
+__version__ = "2.0b2"
 __author__ = "Andres Zarta"
 __credits__ = "DeepLearning.AI"

{dlai_grader-2.0b1 → dlai_grader-2.0b2}/dlai_grader/io.py

@@ -5,6 +5,7 @@ import subprocess
 import tarfile
 from contextlib import contextmanager, redirect_stderr, redirect_stdout
 from os import devnull
+from pathlib import Path
 from textwrap import dedent
 from zipfile import ZipFile
 
@@ -245,10 +246,10 @@ def init_grader() -> None:
     write_file_from_template("./Makefile", template_dict["makefile"])
     write_file_from_template("./.conf", template_dict["conf"])
     write_file_from_template("./entry.py", template_dict["entry_py"])
-    write_file_from_template(
-        "./copy_assignment_to_submission.sh",
-        template_dict["copy_assignment_to_submission_sh"],
-    )
+    # write_file_from_template(
+    #     "./copy_assignment_to_submission.sh",
+    #     template_dict["copy_assignment_to_submission_sh"],
+    # )
     write_file_from_template("./requirements.txt", "dlai-grader")
     write_file_from_template("./.env", "")
 
@@ -313,3 +314,24 @@ def grade_parts(
             subprocess.run(cmd, shell=True, executable="/bin/bash")
         except Exception as e:
             print(f"There was an error grading with coursera_autograder. Details: {e}")
+
+
+def generate_learner_file(
+    filename_source: str,
+    filename_target: str,
+) -> None:
+    """
+    Generates the learning facing version of any file.
+
+    Args:
+        filename_source (str): Path to original notebook.
+        filename_target (str): Path where to save reformatted notebook.
+
+    """
+    solution_code = Path(filename_source).read_text()
+
+    # format the code to replace with placeholders
+    fmt_code = solution_to_learner_format(solution_code)
+
+    # save the learner files
+    Path(filename_target).write_text(fmt_code, encoding="utf-8")

dlai_grader-2.0b2/dlai_grader/templates/Makefile

@@ -0,0 +1,54 @@
+.PHONY: sync learner build debug-unsafe debug-safe grade versioning tag undeletable uneditable init upgrade coursera zip
+
+include .conf
+
+PARTIDS = 123 456
+OS := $(shell uname)
+
+sync:
+	cp mount/submission.ipynb ../$(ASSIGNMENT_NAME)_Solution.ipynb
+	cp learner/$(ASSIGNMENT_NAME).ipynb ../$(ASSIGNMENT_NAME).ipynb
+	cp mount/$(UNIT_TESTS_NAME).py ../$(UNIT_TESTS_NAME).py
+
+learner:
+	dlai_grader --learner --output_notebook=./learner/$(ASSIGNMENT_NAME).ipynb
+	rsync -a --exclude="submission.ipynb" --exclude="__pycache__" --exclude=".mypy_cache" ./mount/ ./learner/
+
+build:
+	docker build -t $(IMAGE_NAME):$(TAG_ID) .
+
+debug-unsafe:
+	docker run -it --rm --mount type=bind,source=$(PWD)/mount,target=/shared/submission --mount type=bind,source=$(PWD),target=/grader/ --env-file $(PWD)/.env --entrypoint ash $(IMAGE_NAME):$(TAG_ID)
+
+debug-safe:
+	docker run -it --rm --mount type=bind,source=$(PWD)/mount,target=/shared/submission --env-file $(PWD)/.env --entrypoint ash $(IMAGE_NAME):$(TAG_ID)
+
+grade:
+	docker run -it --rm --memory=$(HARD_MEMORY) --cpus="$(CPUS)" --memory-reservation=$(SOFT_MEMORY) --mount type=bind,source=$(PWD)/mount,target=/shared/submission --env-file $(PWD)/.env --entrypoint ash $(IMAGE_NAME):$(TAG_ID) -c 'for partId in $(PARTIDS); do export partId=$$partId; echo "Processing part $$partId"; python entry.py; done'
+
+versioning:
+	dlai_grader --versioning
+
+tag:
+	dlai_grader --tag
+
+undeletable:
+	dlai_grader --undeletable
+
+uneditable:
+	dlai_grader --uneditable
+
+init:
+	dlai_grader --versioning
+	dlai_grader --tag
+	dlai_grader --undeletable
+	dlai_grader --uneditable
+
+upgrade:
+	dlai_grader --upgrade
+
+coursera:
+	dlai_grader --grade --partids="$(PARTIDS)" --docker=$(IMAGE_NAME):$(TAG_ID) --memory=$(MEMORY_LIMIT) --submission=$(SUB_DIR)
+
+zip:
+	zip -r $(IMAGE_NAME)$(TAG_ID).zip .

dlai_grader-2.0b2/dlai_grader/templates/copy_assignment_sh/extrafile_n

@@ -0,0 +1,12 @@
+#!/bin/bash",
+set -euo pipefail
+
+Assignment={{ASSIGNMENT_NAME}}
+
+SubmissionFile=submission.ipynb
+SubmissionPath=/shared/submission
+SharedDiskPath=/learner_workplace/$UserId/$CourseId/$LessonId
+
+# copy synced files (exam image typically sync all files in lesson folder)
+echo "Copy learner submission from $SharedDiskPath/$Assignment to $SubmissionPath/$SubmissionFile"
+cp $SharedDiskPath/$Assignment $SubmissionPath/$SubmissionFile

dlai_grader-2.0b2/dlai_grader/templates/copy_assignment_sh/extrafile_y

@@ -0,0 +1,16 @@
+#!/bin/bash",
+set -euo pipefail
+
+Assignment={{ASSIGNMENT_NAME}}
+Extra_file={{EXTRA_FILE_NAME}}
+
+SubmissionFile=submission.ipynb
+SubmissionPath=/shared/submission
+SharedDiskPath=/learner_workplace/$UserId/$CourseId/$LessonId
+
+# copy synced files (exam image typically sync all files in lesson folder)
+echo "Copy learner submission from $SharedDiskPath/$Assignment to $SubmissionPath/$SubmissionFile"
+cp $SharedDiskPath/$Assignment $SubmissionPath/$SubmissionFile
+
+echo "Copy learner submission from $SharedDiskPath/$Extra_file to $SubmissionPath/$Extra_file"
+cp $SharedDiskPath/$Extra_file $SubmissionPath/$Extra_file

dlai_grader-2.0b2/dlai_grader/templates/dockerfile/data_n_solution_n

@@ -0,0 +1,20 @@
+FROM continuumio/miniconda3@sha256:d601a04ea48fd45e60808c7072243d33703d29434d2067816b7f26b0705d889a
+
+RUN apk update && apk add libstdc++
+
+COPY requirements.txt .
+
+RUN pip install -r requirements.txt && rm requirements.txt
+
+RUN mkdir /grader && \
+mkdir /grader/submission
+
+COPY .conf /grader/.conf
+COPY entry.py /grader/entry.py
+COPY grader.py /grader/grader.py
+
+RUN chmod a+rwx /grader/
+
+WORKDIR /grader/
+        
+ENTRYPOINT ["python", "entry.py"]

dlai_grader-2.0b2/dlai_grader/templates/dockerfile/data_n_solution_y

@@ -0,0 +1,21 @@
+FROM continuumio/miniconda3@sha256:d601a04ea48fd45e60808c7072243d33703d29434d2067816b7f26b0705d889a
+
+RUN apk update && apk add libstdc++
+
+COPY requirements.txt .
+
+RUN pip install -r requirements.txt && rm requirements.txt
+
+RUN mkdir /grader && \
+mkdir /grader/submission
+
+COPY .conf /grader/.conf
+COPY solution/ /grader/solution/
+COPY entry.py /grader/entry.py
+COPY grader.py /grader/grader.py
+
+RUN chmod a+rwx /grader/
+
+WORKDIR /grader/
+        
+ENTRYPOINT ["python", "entry.py"]

dlai_grader-2.0b2/dlai_grader/templates/dockerfile/data_y_solution_n

@@ -0,0 +1,21 @@
+FROM continuumio/miniconda3@sha256:d601a04ea48fd45e60808c7072243d33703d29434d2067816b7f26b0705d889a
+
+RUN apk update && apk add libstdc++
+
+COPY requirements.txt .
+
+RUN pip install -r requirements.txt && rm requirements.txt
+
+RUN mkdir /grader && \
+mkdir /grader/submission
+
+COPY .conf /grader/.conf
+COPY data/ /grader/data/
+COPY entry.py /grader/entry.py
+COPY grader.py /grader/grader.py
+
+RUN chmod a+rwx /grader/
+
+WORKDIR /grader/
+        
+ENTRYPOINT ["python", "entry.py"]

dlai_grader-2.0b2/dlai_grader/templates/dockerfile/data_y_solution_y

@@ -0,0 +1,22 @@
+FROM continuumio/miniconda3@sha256:d601a04ea48fd45e60808c7072243d33703d29434d2067816b7f26b0705d889a
+
+RUN apk update && apk add libstdc++
+
+COPY requirements.txt .
+
+RUN pip install -r requirements.txt && rm requirements.txt
+
+RUN mkdir /grader && \
+mkdir /grader/submission
+
+COPY .conf /grader/.conf
+COPY data/ /grader/data/
+COPY solution/ /grader/solution/
+COPY entry.py /grader/entry.py
+COPY grader.py /grader/grader.py
+
+RUN chmod a+rwx /grader/
+
+WORKDIR /grader/
+        
+ENTRYPOINT ["python", "entry.py"]

dlai_grader-2.0b2/dlai_grader/templates/entry_py/solution_n_file_n.py

@@ -0,0 +1,71 @@
+import traceback
+from dlai_grader.config import Config, get_part_id
+from dlai_grader.compiler import compile_partial_module
+from dlai_grader.io import read_notebook, copy_submission_to_workdir, send_feedback
+from dlai_grader.notebook import keep_tagged_cells
+from dlai_grader.grading import compute_grading_score, graded_obj_missing
+from grader import handle_part_id
+
+
+def notebook_grading(config, compile_solution=False):
+    try:
+        nb = read_notebook(config.submission_file_path)
+    except Exception as e:
+        msg = f"There was a problem reading your notebook. Details:\n{e!s}"
+        send_feedback(0.0, msg, err=True)
+
+    transformations = [keep_tagged_cells()]
+    for t in transformations:
+        nb = t(nb)
+
+    try:
+        learner_mod = compile_partial_module(nb, "learner_mod", verbose=False)
+    except Exception as e:
+        msg = f"There was a problem compiling the code from your notebook, please check that you saved before submitting. Details:\n{e!s}"
+        send_feedback(0.0, msg, err=True)
+
+    solution_mod = None
+    if compile_solution:
+        solution_nb = read_notebook(config.solution_file_path)
+        for t in transformations:
+            solution_nb = t(solution_nb)
+        solution_mod = compile_partial_module(
+            solution_nb,
+            "solution_mod",
+            verbose=False,
+        )
+
+    return learner_mod, solution_mod
+
+
+def main() -> None:
+    copy_submission_to_workdir()
+
+    part_id = get_part_id()
+
+    c = Config()
+
+    learner_mod, _ = notebook_grading(c)
+
+    g_func = handle_part_id(part_id)(learner_mod)
+
+    try:
+        cases = g_func()
+    except Exception as e:
+        msg = f"There was an error grading your submission. Details:\n{e!s}"
+        send_feedback(0.0, msg, err=True)
+
+    if graded_obj_missing(cases):
+        msg = "Object required for grading not found. If you haven't completed the exercise this might be expected. Otherwise, check your solution as grader omits cells that throw errors."
+        send_feedback(0.0, msg, err=True)
+
+    score, feedback = compute_grading_score(cases)
+    send_feedback(score, feedback)
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as e:
+        msg = f"There was an error with the program. Exception:\n{e!s}.\nTraceback:\n{traceback.format_exc()}"
+        send_feedback(0.0, msg, err=True)

dlai_grader-2.0b2/dlai_grader/templates/entry_py/solution_n_file_y.py

@@ -0,0 +1,90 @@
+import traceback
+from dlai_grader.config import Config, get_part_id
+from dlai_grader.compiler import compile_partial_module
+from dlai_grader.io import read_notebook, copy_submission_to_workdir, send_feedback
+from dlai_grader.notebook import keep_tagged_cells
+from dlai_grader.grading import (
+    compute_grading_score,
+    graded_obj_missing,
+    LearnerSubmission,
+)
+from grader import handle_part_id
+
+
+def notebook_grading(config, compile_solution=False):
+    try:
+        nb = read_notebook(config.submission_file_path)
+    except Exception as e:
+        msg = f"There was a problem reading your notebook. Details:\n{e!s}"
+        send_feedback(0.0, msg, err=True)
+
+    transformations = [keep_tagged_cells()]
+    for t in transformations:
+        nb = t(nb)
+
+    try:
+        learner_mod = compile_partial_module(nb, "learner_mod", verbose=False)
+    except Exception as e:
+        msg = f"There was a problem compiling the code from your notebook, please check that you saved before submitting. Details:\n{e!s}"
+        send_feedback(0.0, msg, err=True)
+
+    solution_mod = None
+    if compile_solution:
+        solution_nb = read_notebook(config.solution_file_path)
+        for t in transformations:
+            solution_nb = t(solution_nb)
+        solution_mod = compile_partial_module(
+            solution_nb,
+            "solution_mod",
+            verbose=False,
+        )
+
+    return learner_mod, solution_mod
+
+
+def non_notebook_grading(config):
+    try:
+        with open(config.submission_file_path, "r") as file:
+            contents = file.read()
+    except Exception as e:
+        msg = f"There was an error reading your submission. Details:\n{e!s}"
+        send_feedback(0.0, msg, err=True)
+
+    return LearnerSubmission(submission=contents)
+
+
+def main() -> None:
+    part_id = get_part_id()
+
+    match part_id:
+        case "123":
+            copy_submission_to_workdir(file_name="{{EXTRA_FILE_NAME}}")
+            c = Config(submission_file="{{EXTRA_FILE_NAME}}")
+            learner_mod = non_notebook_grading(c)
+        case _:
+            copy_submission_to_workdir()
+            c = Config()
+            learner_mod, _ = notebook_grading(c)
+
+    g_func = handle_part_id(part_id)(learner_mod)
+
+    try:
+        cases = g_func()
+    except Exception as e:
+        msg = f"There was an error grading your submission. Details:\n{e!s}"
+        send_feedback(0.0, msg, err=True)
+
+    if graded_obj_missing(cases):
+        msg = "Object required for grading not found. If you haven't completed the exercise this might be expected. Otherwise, check your solution as grader omits cells that throw errors."
+        send_feedback(0.0, msg, err=True)
+
+    score, feedback = compute_grading_score(cases)
+    send_feedback(score, feedback)
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as e:
+        msg = f"There was an error with the program. Exception:\n{e!s}.\nTraceback:\n{traceback.format_exc()}"
+        send_feedback(0.0, msg, err=True)

dlai_grader-2.0b2/dlai_grader/templates/entry_py/solution_y_file_n.py

@@ -0,0 +1,71 @@
+import traceback
+from dlai_grader.config import Config, get_part_id
+from dlai_grader.compiler import compile_partial_module
+from dlai_grader.io import read_notebook, copy_submission_to_workdir, send_feedback
+from dlai_grader.notebook import keep_tagged_cells
+from dlai_grader.grading import compute_grading_score, graded_obj_missing
+from grader import handle_part_id
+
+
+def notebook_grading(config, compile_solution=False):
+    try:
+        nb = read_notebook(config.submission_file_path)
+    except Exception as e:
+        msg = f"There was a problem reading your notebook. Details:\n{e!s}"
+        send_feedback(0.0, msg, err=True)
+
+    transformations = [keep_tagged_cells()]
+    for t in transformations:
+        nb = t(nb)
+
+    try:
+        learner_mod = compile_partial_module(nb, "learner_mod", verbose=False)
+    except Exception as e:
+        msg = f"There was a problem compiling the code from your notebook, please check that you saved before submitting. Details:\n{e!s}"
+        send_feedback(0.0, msg, err=True)
+
+    solution_mod = None
+    if compile_solution:
+        solution_nb = read_notebook(config.solution_file_path)
+        for t in transformations:
+            solution_nb = t(solution_nb)
+        solution_mod = compile_partial_module(
+            solution_nb,
+            "solution_mod",
+            verbose=False,
+        )
+
+    return learner_mod, solution_mod
+
+
+def main() -> None:
+    copy_submission_to_workdir()
+
+    part_id = get_part_id()
+
+    c = Config()
+
+    learner_mod, solution_mod = notebook_grading(c, compile_solution=True)
+
+    g_func = handle_part_id(part_id)(learner_mod, solution_mod)
+
+    try:
+        cases = g_func()
+    except Exception as e:
+        msg = f"There was an error grading your submission. Details:\n{e!s}"
+        send_feedback(0.0, msg, err=True)
+
+    if graded_obj_missing(cases):
+        msg = "Object required for grading not found. If you haven't completed the exercise this might be expected. Otherwise, check your solution as grader omits cells that throw errors."
+        send_feedback(0.0, msg, err=True)
+
+    score, feedback = compute_grading_score(cases)
+    send_feedback(score, feedback)
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as e:
+        msg = f"There was an error with the program. Exception:\n{e!s}.\nTraceback:\n{traceback.format_exc()}"
+        send_feedback(0.0, msg, err=True)

dlai_grader-2.0b2/dlai_grader/templates/entry_py/solution_y_file_y.py

@@ -0,0 +1,90 @@
+import traceback
+from dlai_grader.config import Config, get_part_id
+from dlai_grader.compiler import compile_partial_module
+from dlai_grader.io import read_notebook, copy_submission_to_workdir, send_feedback
+from dlai_grader.notebook import keep_tagged_cells
+from dlai_grader.grading import (
+    compute_grading_score,
+    graded_obj_missing,
+    LearnerSubmission,
+)
+from grader import handle_part_id
+
+
+def notebook_grading(config, compile_solution=False):
+    try:
+        nb = read_notebook(config.submission_file_path)
+    except Exception as e:
+        msg = f"There was a problem reading your notebook. Details:\n{e!s}"
+        send_feedback(0.0, msg, err=True)
+
+    transformations = [keep_tagged_cells()]
+    for t in transformations:
+        nb = t(nb)
+
+    try:
+        learner_mod = compile_partial_module(nb, "learner_mod", verbose=False)
+    except Exception as e:
+        msg = f"There was a problem compiling the code from your notebook, please check that you saved before submitting. Details:\n{e!s}"
+        send_feedback(0.0, msg, err=True)
+
+    solution_mod = None
+    if compile_solution:
+        solution_nb = read_notebook(config.solution_file_path)
+        for t in transformations:
+            solution_nb = t(solution_nb)
+        solution_mod = compile_partial_module(
+            solution_nb,
+            "solution_mod",
+            verbose=False,
+        )
+
+    return learner_mod, solution_mod
+
+
+def non_notebook_grading(config):
+    try:
+        with open(config.submission_file_path, "r") as file:
+            contents = file.read()
+    except Exception as e:
+        msg = f"There was an error reading your submission. Details:\n{e!s}"
+        send_feedback(0.0, msg, err=True)
+
+    return LearnerSubmission(submission=contents)
+
+
+def main() -> None:
+    part_id = get_part_id()
+
+    match part_id:
+        case "123":
+            copy_submission_to_workdir(file_name="{{EXTRA_FILE_NAME}}")
+            c = Config(submission_file="{{EXTRA_FILE_NAME}}")
+            learner_mod = non_notebook_grading(c)
+        case _:
+            copy_submission_to_workdir()
+            c = Config()
+            learner_mod, solution_mod = notebook_grading(c, compile_solution=True)
+
+    g_func = handle_part_id(part_id)(learner_mod, solution_mod)
+
+    try:
+        cases = g_func()
+    except Exception as e:
+        msg = f"There was an error grading your submission. Details:\n{e!s}"
+        send_feedback(0.0, msg, err=True)
+
+    if graded_obj_missing(cases):
+        msg = "Object required for grading not found. If you haven't completed the exercise this might be expected. Otherwise, check your solution as grader omits cells that throw errors."
+        send_feedback(0.0, msg, err=True)
+
+    score, feedback = compute_grading_score(cases)
+    send_feedback(score, feedback)
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as e:
+        msg = f"There was an error with the program. Exception:\n{e!s}.\nTraceback:\n{traceback.format_exc()}"
+        send_feedback(0.0, msg, err=True)

dlai_grader-2.0b2/dlai_grader/templates/grader.py

@@ -0,0 +1,32 @@
+from types import ModuleType, FunctionType
+from dlai_grader.grading import test_case, object_to_grade
+from dlai_grader.types import grading_function, grading_wrapper, learner_submission
+
+
+def part_1(
+    learner_mod: learner_submission,
+    solution_mod: ModuleType | None = None,
+) -> grading_function:
+    @object_to_grade(learner_mod, "learner_func")
+    def g(learner_func: FunctionType) -> list[test_case]:
+
+        cases: list[test_case] = []
+
+        t = test_case()
+        if not isinstance(learner_func, FunctionType):
+            t.fail()
+            t.msg = "learner_func has incorrect type"
+            t.want = FunctionType
+            t.got = type(learner_func)
+            return [t]
+
+        return cases
+
+    return g
+
+
+def handle_part_id(part_id: str) -> grading_wrapper:
+    grader_dict: dict[str, grading_wrapper] = {
+        "": part_1,
+    }
+    return grader_dict[part_id]