dlai-grader 1.22.1__tar.gz → 2.0.0__tar.gz

dlai_grader-2.0.0/MANIFEST.in

@@ -0,0 +1,3 @@
+recursive-include dlai_grader/templates *
+
+include dlai_grader/py.typed

{dlai_grader-1.22.1 → dlai_grader-2.0.0}/PKG-INFO

@@ -1,27 +1,23 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: dlai-grader
-Version: 1.22.1
+Version: 2.0.0
 Summary: Grading utilities for DLAI courses
 Home-page: https://github.com/https-deeplearning-ai/grader
 Author: Andres Zarta
-Author-email: andrezb5@gmail.com
-License: MIT License
+Author-email: Andres Zarta <andrezb5@gmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/https-deeplearning-ai/grader
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: nbformat>=5.1.3
 Requires-Dist: jupytext>=1.13.0
 Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
 Dynamic: home-page
-Dynamic: license
-Dynamic: requires-dist
-Dynamic: summary
+Dynamic: license-file
 
 # grader
 Automatic grading for DLAI courses. Designed to be compatible with Coursera's grading requirements.

{dlai_grader-1.22.1 → dlai_grader-2.0.0}/dlai_grader/__init__.py

@@ -6,6 +6,6 @@ from . import grading
 from . import types
 
 
-__version__ = "1.22.1"
+__version__ = "2.0.0"
 __author__ = "Andres Zarta"
 __credits__ = "DeepLearning.AI"

{dlai_grader-1.22.1 → dlai_grader-2.0.0}/dlai_grader/config.py

@@ -2,12 +2,11 @@ import os
 import re
 import csv
 from dataclasses import dataclass
-from typing import Dict
 
 
 def parse_conf(
     path: str = "./.conf",
-) -> Dict[str, str]:
+) -> dict[str, str]:
     """Parses variables from .conf file
 
     Args:
@@ -45,10 +44,9 @@ class Config:
     submission_workdir: str = "./submission/"
     solution_workdir: str = "./solution/"
     solution_file: str = "solution.ipynb"
-    solution_file_path: str = os.path.join(solution_workdir, solution_file)
+    solution_file_path: str = ""
     submission_file: str = "submission.ipynb"
-    submission_file_path: str = os.path.join(submission_workdir, submission_file)
-    feedback_file_path: str = "/shared/feedback.json"
+    submission_file_path: str = ""
     part_id: str = get_part_id()
     latest_version: str = (
         parse_conf()["GRADER_VERSION"] if os.path.exists("./.conf") else ""
@@ -56,3 +54,12 @@ class Config:
     assignment_name: str = (
         parse_conf()["ASSIGNMENT_NAME"] if os.path.exists("./.conf") else ""
     )
+
+    def __post_init__(self):
+        # This is where we set the file paths after the instance variables are initialized
+        self.solution_file_path = os.path.join(
+            self.solution_workdir, self.solution_file
+        )
+        self.submission_file_path = os.path.join(
+            self.submission_workdir, self.submission_file
+        )

{dlai_grader-1.22.1 → dlai_grader-2.0.0}/dlai_grader/grading.py

@@ -1,75 +1,96 @@
 from dataclasses import dataclass
 from functools import wraps
-from typing import Any, Callable, List, Tuple, Union
+from typing import Any, Callable
 from types import ModuleType
 
 
 @dataclass
 class LearnerSubmission:
-    """Class that represents a file from the learner. Useful when grading does not depend on the notebook"""
+    """Class that represents a file from the learner. Useful when grading does not depend on the notebook."""
 
     submission: Any = None
 
 
-learner_submission = Union[ModuleType, LearnerSubmission]
+learner_submission = ModuleType | LearnerSubmission
 
 
 @dataclass
 class test_case:
-    """Class that represents a test case"""
+    """Class that represents a test case."""
 
     msg: str = ""
     want: Any = None
     got: Any = None
     failed: bool = False
 
+    def fail(self):
+        """Sets the failed attribute to True."""
+        self.failed = True
+
 
 @dataclass
 class aggregated_test_case:
-    """Class that represents an aggregated collection of test cases"""
+    """Class that represents an aggregated collection of test cases."""
 
     test_name: str
-    tests: List[test_case]
+    tests: list[test_case]
+
 
+def compute_grading_score(test_cases: list[test_case]) -> tuple[float, str]:
+    """
+    Computes the score based on the number of failed and total cases.
 
-def compute_grading_score(
-    test_cases: List[test_case],
-) -> Tuple[float, str]:
-    """Computes the score based on the number of failed and total cases.
     Args:
         test_cases (List): Test cases.
+
     Returns:
         Tuple[float, str]: The grade and feedback message.
-    """
 
+    """
     num_cases = len(test_cases)
     if num_cases == 0:
-        return (
-            0.0,
-            "The grader was unable to generate test cases for your implementation. This suggests a bug with your code, please revise your solution and try again.",
-        )
+        msg = "The grader was unable to generate test cases for your implementation. This suggests a bug with your code, please revise your solution and try again."
+        return 0.0, msg
 
     failed_cases = [t for t in test_cases if t.failed]
     score = 1.0 - len(failed_cases) / num_cases
-    feedback_msg = "All tests passed! Congratulations!"
+    score = round(score, 2)
+
+    if not failed_cases:
+        msg = "All tests passed! ✅ Congratulations! 🎉"
+        return 1.0, msg
+
+    feedback_msg = ""
+    # if print_passed_tests:
+    #     passed_cases = [t for t in test_cases if not t.failed]
+    #     for passed_case in passed_cases:
+    #         feedback_msg += f"✅ {passed_case.msg}.\n"
 
     if failed_cases:
-        feedback_msg = ""
         for failed_case in failed_cases:
-            feedback_msg += f"Failed test case: {failed_case.msg}.\nExpected:\n{failed_case.want},\nbut got:\n{failed_case.got}.\n\n"
-        return round(score, 2), feedback_msg
+            feedback_msg += f"❌ {failed_case.msg}.\n"
+
+            if failed_case.want:
+                feedback_msg += f"Expected:\n{failed_case.want}\n"
+
+            if failed_case.got:
+                feedback_msg += f"but got:\n{failed_case.got}\n\n"
 
     return score, feedback_msg
 
 
 def compute_aggregated_grading_score(
-    aggregated_test_cases: List[aggregated_test_case],
-) -> Tuple[float, str]:
-    """Computes the score based on the number of failed and total cases.
+    aggregated_test_cases: list[aggregated_test_case],
+) -> tuple[float, str]:
+    """
+    Computes the score based on the number of failed and total cases.
+
     Args:
         aggregated_test_cases (List): Aggregated test cases for every part.
+
     Returns:
         Tuple[float, str]: The grade and feedback message.
+
     """
     scores = []
     msgs = []
@@ -98,11 +119,14 @@ def compute_aggregated_grading_score(
 def object_to_grade(
     origin_module: learner_submission,
     *attr_names: str,
-) -> Callable[[Callable[[Any], List[test_case]]], Callable[[Any], List[test_case]]]:
-    """Used as a parameterized decorator to get any number of attributes from a module.
+) -> Callable[[Callable[[Any], list[test_case]]], Callable[[Any], list[test_case]]]:
+    """
+    Used as a parameterized decorator to get any number of attributes from a module.
+
     Args:
         origin_module (ModuleType): A module.
-        attrs_name (Tuple[str, ...]): Names of the attributes to extract from the module.
+        *attrs_name (Tuple[str, ...]): Names of the attributes to extract from the module.
+
     """
 
     def middle(func):
@@ -120,11 +144,13 @@ def object_to_grade(
     return middle
 
 
-def print_feedback(test_cases: List[test_case]) -> None:
-    """Prints feedback of public unit tests within notebook.
+def print_feedback(test_cases: list[test_case]) -> None:
+    """
+    Prints feedback of public unit tests within notebook.
 
     Args:
         test_cases (List[test_case]): List of public test cases.
+
     """
     failed_cases = [t for t in test_cases if t.failed]
     feedback_msg = "\033[92m All tests passed!"
@@ -137,16 +163,18 @@ def print_feedback(test_cases: List[test_case]) -> None:
     print(feedback_msg)
 
 
-def graded_obj_missing(test_cases: List[test_case]) -> bool:
-    """Check if the object to grade was found in the learned module.
+def graded_obj_missing(test_cases: list[test_case]) -> bool:
+    """
+    Check if the object to grade was found in the learned module.
 
     Args:
         test_cases (List[test_case]): List of test cases.
 
     Returns:
         bool: True if object is missing. False otherwise.
+
     """
-    if len(test_cases) == 1 and test_cases[0].got == type(None):
+    if len(test_cases) == 1 and test_cases[0].got is type(None):
         return True
 
     return False

{dlai_grader-1.22.1 → dlai_grader-2.0.0}/dlai_grader/io.py

@@ -1,27 +1,30 @@
-import os
 import json
+import os
 import shutil
-import tarfile
-import nbformat
-import jupytext
 import subprocess
+import tarfile
+from contextlib import contextmanager, redirect_stderr, redirect_stdout
 from os import devnull
+from pathlib import Path
 from textwrap import dedent
 from zipfile import ZipFile
+
+import jupytext
+import nbformat
 from nbformat.notebooknode import NotebookNode
-from contextlib import contextmanager, redirect_stderr, redirect_stdout
+
 from .notebook import (
     add_metadata_all_code_cells,
     add_metadata_code_cells_without_pattern,
-    tag_code_cells,
     solution_to_learner_format,
+    tag_code_cells,
 )
 from .templates import load_templates
 
 
 @contextmanager
 def suppress_stdout_stderr():
-    """A context manager that redirects stdout and stderr to devnull"""
+    """A context manager that redirects stdout and stderr to devnull."""
     with open(devnull, "w") as fnull:
         with redirect_stderr(fnull) as err, redirect_stdout(fnull) as out:
             yield (err, out)
@@ -30,11 +33,15 @@ def suppress_stdout_stderr():
 def read_notebook(
     path: str,
 ) -> NotebookNode:
-    """Reads a notebook found in the given path and returns a serialized version.
+    """
+    Reads a notebook found in the given path and returns a serialized version.
+
     Args:
         path (str): Path of the notebook file to read.
+
     Returns:
         NotebookNode: Representation of the notebook following nbformat convention.
+
     """
     return nbformat.read(path, as_version=nbformat.NO_CONVERT)
 
@@ -42,10 +49,12 @@ def read_notebook(
 def tag_notebook(
     path: str,
 ) -> None:
-    """Adds 'graded' tag to all code cells of a notebook.
+    """
+    Adds 'graded' tag to all code cells of a notebook.
 
     Args:
         path (str): Path to the notebook.
+
     """
     nb = read_notebook(path)
     nb = tag_code_cells(nb)
@@ -53,10 +62,12 @@ def tag_notebook(
 
 
 def undeletable_notebook(path: str) -> None:
-    """Makes all code cells of a notebook non-deletable.
+    """
+    Makes all code cells of a notebook non-deletable.
 
     Args:
         path (str): Path to the notebook.
+
     """
     nb = read_notebook(path)
     nb = add_metadata_all_code_cells(nb, {"deletable": False})
@@ -64,13 +75,17 @@ def undeletable_notebook(path: str) -> None:
 
 
 def uneditable_notebook(path: str) -> None:
-    """Makes all non-graded code cells of a notebook non-editable.
+    """
+    Makes all non-graded code cells of a notebook non-editable.
 
     Args:
         path (str): Path to the notebook.
+
     """
     nb = read_notebook(path)
-    nb = add_metadata_code_cells_without_pattern(nb, {"editable": False})
+    nb = add_metadata_code_cells_without_pattern(
+        nb, {"editable": False}, ignore_pattern="^# EDITABLE"
+    )
     jupytext.write(nb, path)
 
 
@@ -79,12 +94,14 @@ def extract_tar(
     destination: str,
     post_cleanup: bool = True,
 ) -> None:
-    """Extracts a tar file unto the desired destination.
+    """
+    Extracts a tar file unto the desired destination.
 
     Args:
         file_path (str): Path to tar file.
         destination (str): Path where to save uncompressed files.
         post_cleanup (bool, optional): If true, deletes the compressed tar file. Defaults to True.
+
     """
     with tarfile.open(file_path, "r") as my_tar:
         my_tar.extractall(destination)
@@ -118,14 +135,16 @@ def send_feedback(
     feedback_path: str = "/shared/feedback.json",
     err: bool = False,
 ) -> None:
-    """Sends feedback to the learner.
+    """
+    Sends feedback to the learner.
+
     Args:
         score (float): Grading score to show on Coursera for the assignment.
         msg (str): Message providing additional feedback.
         feedback_path (str): Path where the json feedback will be saved. Defaults to /shared/feedback.json
         err (bool, optional): True if there was an error while grading. Defaults to False.
-    """
 
+    """
     post = {"fractionalScore": score, "feedback": msg}
     print(json.dumps(post))
 
@@ -166,7 +185,7 @@ def update_grader_version() -> str:
 
     new_lines = []
     for l in lines:
-        if ("GRADER_VERSION" in l) and (not "TAG_ID" in l):
+        if ("GRADER_VERSION" in l) and ("TAG_ID" not in l):
             _, v = l.split("=")
             num_v = int(v)
             new_v = num_v + 1
@@ -227,13 +246,23 @@ def init_grader() -> None:
     write_file_from_template("./Makefile", template_dict["makefile"])
     write_file_from_template("./.conf", template_dict["conf"])
     write_file_from_template("./entry.py", template_dict["entry_py"])
+    # write_file_from_template(
+    #     "./copy_assignment_to_submission.sh",
+    #     template_dict["copy_assignment_to_submission_sh"],
+    # )
     write_file_from_template("./requirements.txt", "dlai-grader")
-    os.makedirs("data")
+    write_file_from_template("./.env", "")
+
     os.makedirs("learner")
     os.makedirs("mount")
-    os.makedirs("solution")
     os.makedirs("submission")
 
+    if "COPY data/ /grader/data/" in template_dict["dockerfile"]:
+        os.makedirs("data")
+
+    if "COPY solution/ /grader/solution/" in template_dict["dockerfile"]:
+        os.makedirs("solution")
+
 
 def generate_learner_version(
     filename_source: str,
@@ -285,3 +314,24 @@ def grade_parts(
             subprocess.run(cmd, shell=True, executable="/bin/bash")
         except Exception as e:
             print(f"There was an error grading with coursera_autograder. Details: {e}")
+
+
+def generate_learner_file(
+    filename_source: str,
+    filename_target: str,
+) -> None:
+    """
+    Generates the learning facing version of any file.
+
+    Args:
+        filename_source (str): Path to original notebook.
+        filename_target (str): Path where to save reformatted notebook.
+
+    """
+    solution_code = Path(filename_source).read_text()
+
+    # format the code to replace with placeholders
+    fmt_code = solution_to_learner_format(solution_code)
+
+    # save the learner files
+    Path(filename_target).write_text(fmt_code, encoding="utf-8")

{dlai_grader-1.22.1 → dlai_grader-2.0.0}/dlai_grader/notebook.py

@@ -72,8 +72,8 @@ def keep_tagged_cells(
         filtered_cells = []
 
         for cell in notebook["cells"]:
-            if (not "tags" in cell["metadata"]) or (
-                not tag in cell["metadata"].get("tags")
+            if ("tags" not in cell["metadata"]) or (
+                tag not in cell["metadata"].get("tags")
             ):
                 continue
             filtered_cells.append(cell)
@@ -147,7 +147,7 @@ def get_named_cells(
     named_cells = {}
     for cell in notebook["cells"]:
         metadata = cell["metadata"]
-        if not "name" in metadata:
+        if "name" not in metadata:
             continue
         named_cells.update({metadata.get("name"): cell})
     return named_cells
@@ -169,12 +169,12 @@ def tag_code_cells(
 
     for cell in notebook["cells"]:
         if cell["cell_type"] == "code":
-            if not "tags" in cell["metadata"]:
+            if "tags" not in cell["metadata"]:
                 cell["metadata"]["tags"] = []
 
             tags = cell["metadata"]["tags"]
 
-            if not tag in tags:
+            if tag not in tags:
                 tags.append(tag)
                 cell["metadata"]["tags"] = tags
 
@@ -238,27 +238,31 @@ def add_metadata_code_cells_with_pattern(
 
 
 def add_metadata_code_cells_without_pattern(
-    notebook: NotebookNode, metadata: dict, regex_pattern: str = "^# GRADED "
+    notebook: NotebookNode,
+    metadata: dict,
+    match_pattern: str = "^# GRADED",
+    ignore_pattern: str = None,
 ) -> NotebookNode:
-    """Adds metadata to code cells of a notebook that don't match a regexp pattern.
+    """Adds metadata to code cells of a notebook that don't match a regexp pattern and aren't ignored by another pattern.
+
     Args:
         notebook (NotebookNode): Notebook to filter.
         metadata (dict): The metadata which should be a key-value pair.
-        regex_pattern (str, optional): Pattern to check. Defaults to "w[1-9]_unittest".
+        match_pattern (str, optional): Pattern to check which cells to add metadata to. Defaults to "^# GRADED ".
+        ignore_pattern (str, optional): Pattern to check which cells to ignore. If a cell matches this pattern, it will be skipped.
+
     Returns:
         NotebookNode: The notebook with the new metadata.
     """
-    filtered_cells = []
-
     for cell in notebook["cells"]:
-        if cell["cell_type"] == "code" and not re.search(regex_pattern, cell["source"]):
-            current_metadata = cell["metadata"]
-            current_metadata.update(metadata)
-            cell["metadata"] = current_metadata
-
-        filtered_cells.append(cell)
+        if cell["cell_type"] == "code":
+            if ignore_pattern and re.search(ignore_pattern, cell["source"]):
+                continue
 
-    notebook["cells"] = filtered_cells
+            if not re.search(match_pattern, cell["source"]):
+                current_metadata = cell.get("metadata", {})
+                current_metadata.update(metadata)
+                cell["metadata"] = current_metadata
 
     return notebook
 

dlai_grader-2.0.0/dlai_grader/templates/Makefile

@@ -0,0 +1,57 @@
+.PHONY: sync learner build debug-unsafe debug-safe grade versioning tag undeletable uneditable init upgrade coursera zip
+
+include .conf
+
+PARTIDS = 123 456
+COURSERA_PARTIDS = "123 456"
+
+OS := $(shell uname)
+
+sync:
+	cp mount/submission.ipynb ../$(ASSIGNMENT_NAME)_Solution.ipynb
+	cp learner/$(ASSIGNMENT_NAME).ipynb ../$(ASSIGNMENT_NAME).ipynb
+	cp mount/$(UNIT_TESTS_NAME).py ../$(UNIT_TESTS_NAME).py
+
+learner:
+	dlai_grader --learner --output_notebook=./learner/$(ASSIGNMENT_NAME).ipynb
+	jupyter nbconvert --clear-output --inplace ./learner/$(ASSIGNMENT_NAME).ipynb
+# 	rsync -a --exclude="submission.ipynb" --exclude="__pycache__" --exclude=".*" ./mount/ ./learner/
+
+build:
+	docker build -t $(IMAGE_NAME):$(TAG_ID) .
+
+debug-unsafe:
+	docker run -it --rm --mount type=bind,source=$(PWD)/mount,target=/shared/submission --mount type=bind,source=$(PWD),target=/grader/ --env-file $(PWD)/.env --entrypoint ash $(IMAGE_NAME):$(TAG_ID)
+
+debug-safe:
+	docker run -it --rm --mount type=bind,source=$(PWD)/mount,target=/shared/submission --env-file $(PWD)/.env --entrypoint ash $(IMAGE_NAME):$(TAG_ID)
+
+grade:
+	docker run -it --rm --memory=$(HARD_MEMORY) --cpus="$(CPUS)" --memory-reservation=$(SOFT_MEMORY) --mount type=bind,source=$(PWD)/mount,target=/shared/submission --env-file $(PWD)/.env --entrypoint ash $(IMAGE_NAME):$(TAG_ID) -c 'for partId in $(PARTIDS); do export partId=$$partId; echo "Processing part $$partId"; python entry.py; done'
+
+versioning:
+	dlai_grader --versioning
+
+tag:
+	dlai_grader --tag
+
+undeletable:
+	dlai_grader --undeletable
+
+uneditable:
+	dlai_grader --uneditable
+
+init:
+	dlai_grader --versioning
+	dlai_grader --tag
+	dlai_grader --undeletable
+	dlai_grader --uneditable
+
+upgrade:
+	dlai_grader --upgrade
+
+coursera:
+	dlai_grader --grade --partids=$(COURSERA_PARTIDS) --docker=$(IMAGE_NAME):$(TAG_ID) --memory=$(MEMORY_LIMIT) --submission=$(SUB_DIR)
+
+zip:
+	zip -r $(IMAGE_NAME)$(TAG_ID).zip .

dlai_grader-2.0.0/dlai_grader/templates/copy_assignment_sh/extrafile_n

@@ -0,0 +1,12 @@
+#!/bin/bash",
+set -euo pipefail
+
+Assignment={{ASSIGNMENT_NAME}}
+
+SubmissionFile=submission.ipynb
+SubmissionPath=/shared/submission
+SharedDiskPath=/learner_workplace/$UserId/$CourseId/$LessonId
+
+# copy synced files (exam image typically sync all files in lesson folder)
+echo "Copy learner submission from $SharedDiskPath/$Assignment to $SubmissionPath/$SubmissionFile"
+cp $SharedDiskPath/$Assignment $SubmissionPath/$SubmissionFile

dlai_grader-2.0.0/dlai_grader/templates/copy_assignment_sh/extrafile_y

@@ -0,0 +1,16 @@
+#!/bin/bash",
+set -euo pipefail
+
+Assignment={{ASSIGNMENT_NAME}}
+Extra_file={{EXTRA_FILE_NAME}}
+
+SubmissionFile=submission.ipynb
+SubmissionPath=/shared/submission
+SharedDiskPath=/learner_workplace/$UserId/$CourseId/$LessonId
+
+# copy synced files (exam image typically sync all files in lesson folder)
+echo "Copy learner submission from $SharedDiskPath/$Assignment to $SubmissionPath/$SubmissionFile"
+cp $SharedDiskPath/$Assignment $SubmissionPath/$SubmissionFile
+
+echo "Copy learner submission from $SharedDiskPath/$Extra_file to $SubmissionPath/$Extra_file"
+cp $SharedDiskPath/$Extra_file $SubmissionPath/$Extra_file

dlai_grader-2.0.0/dlai_grader/templates/dockerfile/data_n_solution_n

@@ -0,0 +1,20 @@
+FROM continuumio/miniconda3@sha256:d601a04ea48fd45e60808c7072243d33703d29434d2067816b7f26b0705d889a
+
+RUN apk update && apk add libstdc++
+
+COPY requirements.txt .
+
+RUN pip install -r requirements.txt && rm requirements.txt
+
+RUN mkdir /grader && \
+mkdir /grader/submission
+
+COPY .conf /grader/.conf
+COPY entry.py /grader/entry.py
+COPY grader.py /grader/grader.py
+
+RUN chmod a+rwx /grader/
+
+WORKDIR /grader/
+        
+ENTRYPOINT ["python", "entry.py"]

dlai_grader-2.0.0/dlai_grader/templates/dockerfile/data_n_solution_y

@@ -0,0 +1,21 @@
+FROM continuumio/miniconda3@sha256:d601a04ea48fd45e60808c7072243d33703d29434d2067816b7f26b0705d889a
+
+RUN apk update && apk add libstdc++
+
+COPY requirements.txt .
+
+RUN pip install -r requirements.txt && rm requirements.txt
+
+RUN mkdir /grader && \
+mkdir /grader/submission
+
+COPY .conf /grader/.conf
+COPY solution/ /grader/solution/
+COPY entry.py /grader/entry.py
+COPY grader.py /grader/grader.py
+
+RUN chmod a+rwx /grader/
+
+WORKDIR /grader/
+        
+ENTRYPOINT ["python", "entry.py"]