assignment-codeval 0.0.25__tar.gz → 0.0.26__tar.gz

{assignment_codeval-0.0.25 → assignment_codeval-0.0.26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assignment-codeval
-Version: 0.0.25
+Version: 0.0.26
 Summary: CodEval for evaluating programming assignments
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown

{assignment_codeval-0.0.25 → assignment_codeval-0.0.26}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "assignment-codeval"
-version = "0.0.25"
+version = "0.0.26"
 description = "CodEval for evaluating programming assignments"
 readme = "README.md"
 requires-python = ">=3.12"

{assignment_codeval-0.0.25 → assignment_codeval-0.0.26}/src/assignment_codeval/evaluate.py

@@ -28,6 +28,7 @@ def get_testing_path(filename: str) -> str:
 test_args = ""
 cmps = []
 timeout_val = 10
+output_length_limit = 4096
 expected_exit_code = -1
 test_case_count = 0
 test_case_hint = ""
@@ -643,6 +644,19 @@ def timeout(timeout_sec):
     timeout_val = float(timeout_sec)
 
 
+def output_length(length):
+    """Specifies the maximum number of bytes of diff output to render. Defaults to 4096.
+
+    Arguments:
+        length: maximum output length in bytes
+
+    Returns:
+        None
+    """
+    global output_length_limit
+    output_length_limit = int(length)
+
+
 def exit_code(test_case_exit_code):
     """Specifies the expected exit code for a test case. Defaults to zero.
 
@@ -726,6 +740,7 @@ tag_func_map = {
     "EB": check_error_bare,
     "HINT": hint,
     "TO": timeout,
+    "OLEN": output_length,
     "X": exit_code,
     "SS": start_server,
 }
@@ -907,6 +922,72 @@ def parse_diff(diff_lines: list[str], testing_dir: str):
                     outfile.write(line)
 
 
+def _render_diff_output(raw_bytes: bytes) -> str:
+    """Render raw bytes for diff output, making unprintable characters visible.
+
+    Replaces the shell pipeline `cat -te | head -22` with pure Python.
+    """
+    result = []
+    i = 0
+    n = len(raw_bytes)
+
+    while i < n:
+        b = raw_bytes[i]
+
+        if b == 0x0A:  # newline
+            result.append("$\n")
+            i += 1
+        elif b == 0x20 or (0x21 <= b <= 0x7E):  # space or printable ASCII
+            result.append(chr(b))
+            i += 1
+        elif b <= 0x1F:  # control chars (except newline handled above)
+            result.append("^" + chr(b + 0x40))
+            i += 1
+        elif b == 0x7F:  # DEL
+            result.append("^?")
+            i += 1
+        elif b >= 0x80:
+            # Try to decode as UTF-8 multi-byte sequence
+            seq_len = 0
+            if (b & 0xE0) == 0xC0:
+                seq_len = 2
+            elif (b & 0xF0) == 0xE0:
+                seq_len = 3
+            elif (b & 0xF8) == 0xF0:
+                seq_len = 4
+
+            decoded_char = None
+            if seq_len >= 2 and i + seq_len <= n:
+                try:
+                    decoded_char = raw_bytes[i:i + seq_len].decode("utf-8")
+                except UnicodeDecodeError:
+                    pass
+
+            if decoded_char and len(decoded_char) == 1 and decoded_char.isprintable():
+                result.append(decoded_char)
+                i += seq_len
+            elif decoded_char and len(decoded_char) == 1:
+                # Valid UTF-8 but non-printable: render each byte as \xCC
+                for j in range(seq_len):
+                    result.append(f"\\x{raw_bytes[i + j]:02X}")
+                i += seq_len
+            else:
+                # Invalid UTF-8 byte
+                result.append(f"\\x{b:02X}")
+                i += 1
+        else:
+            result.append(chr(b))
+            i += 1
+
+    output = "".join(result)
+
+    # If output doesn't end with \n, append $ at the very end
+    if output and not output.endswith("\n"):
+        output += "$"
+
+    return output
+
+
 def _find_touched_files(pre_run_timestamp):
     """Find files touched (modified or accessed) since pre_run_timestamp.
 
@@ -960,24 +1041,22 @@ def check_test():
     # Difflog handling
     with open(get_testing_path("difflog"), "w") as outfile:
         diff_popen = subprocess.Popen(
-            f"diff -U1 -a ./{TESTING_DIR}/youroutput ./{TESTING_DIR}/expectedoutput | cat -te | head -22",
-            shell=True,
-            stdout=outfile,
-            stderr=outfile,
-            text=True,
+            ["diff", "-U1", "-a",
+             f"./{TESTING_DIR}/youroutput", f"./{TESTING_DIR}/expectedoutput"],
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
         )
-        diff_popen.communicate()
+        raw_output, _ = diff_popen.communicate()
+        outfile.write(_render_diff_output(raw_output[:output_length_limit]))
 
     # Append to difflog second time around
     with open(get_testing_path("difflog"), "a") as outfile:
         diff_popen = subprocess.Popen(
-            f"diff -U1 -a ./{TESTING_DIR}/yourerror ./{TESTING_DIR}/expectederror | cat -te | head -22",
-            shell=True,
-            stdout=outfile,
-            stderr=outfile,
-            text=True,
+            ["diff", "-U1", "-a",
+             f"./{TESTING_DIR}/yourerror", f"./{TESTING_DIR}/expectederror"],
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
         )
-        diff_popen.communicate()
+        raw_output, _ = diff_popen.communicate()
+        outfile.write(_render_diff_output(raw_output[:output_length_limit]))
 
     # Now read all the lines to accumulate both diffs
     with open(get_testing_path("difflog"), "r") as infile:

{assignment_codeval-0.0.25 → assignment_codeval-0.0.26}/src/assignment_codeval/submissions.py

@@ -180,6 +180,37 @@ def delete_submission_comment(canvas, course_id, assignment_id, user_id, comment
     resp.raise_for_status()
 
 
+def _parse_substitutions_file(filepath):
+    """Parse a substitutions file where each line has the form /pattern/replacement/.
+
+    The first character of each line defines the delimiter. The delimiter must appear
+    exactly once in-between (separating pattern from replacement) and once at the end.
+    Returns a list of (pattern, replacement) tuples.
+    Raises click.ClickException on parse errors.
+    """
+    substitutions = []
+    with open(filepath, "r") as fd:
+        for lineno, line in enumerate(fd, start=1):
+            line = line.rstrip("\n")
+            if not line:
+                continue
+            delim = line[0]
+            parts = line[1:].split(delim)
+            if len(parts) != 3 or parts[2] != "":
+                raise click.ClickException(
+                    f"{filepath}:{lineno}: invalid substitution line: {line}"
+                )
+            substitutions.append((parts[0], parts[1]))
+    return substitutions
+
+
+def _apply_substitutions(comment, substitutions):
+    """Apply a list of (pattern, replacement) substitutions to comment text."""
+    for pattern, replacement in substitutions:
+        comment = comment.replace(pattern, replacement)
+    return comment
+
+
 @click.command()
 @click.argument("submissions_dir", metavar="SUBMISSIONS_DIR")
 @click.option("--codeval-prefix", help="prefix for codeval comments", default="codeval: ", show_default=True)
@@ -214,8 +245,12 @@ def upload_submission_comments(submissions_dir, codeval_prefix, delete):
                     write_html_file(dirpath)
                     file_id = upload_file_for_comment(canvas, course.id, assignment.id, student_id, f"{dirpath}/results.html")
                     with open(f"{dirpath}/comments.txt", "r") as fd:
-                        comment = fd.read()
-                        comment = comment.replace("\0", "\\0").strip()
+                        comment = fd.read(4096)
+                        comment = comment.replace("\0", "\\0").strip().replace("<", "&lt;")
+                    subs_file = f"{dirpath}/SUBSTITUTIONS.txt"
+                    if os.path.isfile(subs_file):
+                        substitutions = _parse_substitutions_file(subs_file)
+                        comment = _apply_substitutions(comment, substitutions)
                     if delete:
                         all_comments = sorted(submission.submission_comments, key=lambda c: c['created_at'])
                         if all_comments:
@@ -226,7 +261,7 @@ def upload_submission_comments(submissions_dir, codeval_prefix, delete):
                         f'{canvas_url}/api/v1/courses/{course.id}/assignments/{assignment.id}/submissions/{student_id}',
                         headers={'Authorization': f'Bearer {canvas_token}'},
                         data={
-                            'comment[text_comment]': f'{codeval_prefix.rstrip()}\n{comment}',
+                            'comment[text_comment]': f'{codeval_prefix.rstrip()}\n<pre>{comment}</pre>',
                             'comment[file_ids][]': file_id,
                         }
                     ).raise_for_status()
@@ -312,6 +347,9 @@ def evaluate_submissions(codeval_dir, submissions_dir):
         if not move_to_next_submission:
             for zf_name in zip_files:
                 with ZipFile(os.path.join(codeval_dir, zf_name)) as zf:
+                    if "SUBSTITUTIONS.txt" in zf.namelist():
+                        with open(os.path.join(dirpath, "SUBSTITUTIONS.txt"), "wb") as out_f:
+                            out_f.write(zf.read("SUBSTITUTIONS.txt"))
                     for f in zf.infolist():
                         dest_dir = os.path.join(submission_dir, assignment_working_dir)
                         zf.extract(f, dest_dir)

{assignment_codeval-0.0.25 → assignment_codeval-0.0.26}/src/assignment_codeval.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assignment-codeval
-Version: 0.0.25
+Version: 0.0.26
 Summary: CodEval for evaluating programming assignments
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown

assignment_codeval-0.0.26/tests/test_codeval.py

@@ -0,0 +1,77 @@
+"""Pytest tests for CodEval test files."""
+import glob
+import os
+import subprocess
+import pytest
+
+from assignment_codeval.evaluate import _render_diff_output
+
+# Get all .codeval test files
+TEST_DIR = os.path.dirname(os.path.abspath(__file__))
+CODEVAL_FILES = sorted(glob.glob(os.path.join(TEST_DIR, "*.codeval")))
+
+
+def get_test_ids():
+    """Generate test IDs from filenames."""
+    return [os.path.basename(f).replace(".codeval", "") for f in CODEVAL_FILES]
+
+
+@pytest.mark.parametrize("codeval_file", CODEVAL_FILES, ids=get_test_ids())
+def test_codeval(codeval_file):
+    """Run a .codeval test file and verify it passes."""
+    test_name = os.path.basename(codeval_file)
+
+    result = subprocess.run(
+        ["assignment-codeval", "run-evaluation", test_name],
+        cwd=TEST_DIR,
+        capture_output=True,
+        text=True,
+    )
+
+    # Print output for debugging
+    if result.stdout:
+        print(result.stdout)
+    if result.stderr:
+        print(result.stderr)
+
+    assert result.returncode == 0, f"Test {test_name} failed with exit code {result.returncode}"
+
+
+class TestRenderDiffOutput:
+    def test_plain_ascii(self):
+        assert _render_diff_output(b"hello\n") == "hello$\n"
+
+    def test_multiple_lines(self):
+        assert _render_diff_output(b"a\nb\n") == "a$\nb$\n"
+
+    def test_no_trailing_newline(self):
+        assert _render_diff_output(b"hello") == "hello$"
+
+    def test_tab_rendered_as_caret_i(self):
+        assert _render_diff_output(b"a\tb\n") == "a^Ib$\n"
+
+    def test_control_chars(self):
+        assert _render_diff_output(b"\x01\x02\x03\n") == "^A^B^C$\n"
+
+    def test_null_byte(self):
+        assert _render_diff_output(b"\x00\n") == "^@$\n"
+
+    def test_del_byte(self):
+        assert _render_diff_output(b"\x7f\n") == "^?$\n"
+
+    def test_invalid_utf8_bytes(self):
+        assert _render_diff_output(b"\x80\xff\n") == "\\x80\\xFF$\n"
+
+    def test_valid_utf8_printable(self):
+        # é is U+00E9, encoded as 0xC3 0xA9
+        assert _render_diff_output("é\n".encode("utf-8")) == "é$\n"
+
+    def test_mixed_content(self):
+        raw = b"ok\t\x01\x80end\n"
+        assert _render_diff_output(raw) == "ok^I^A\\x80end$\n"
+
+    def test_empty_input(self):
+        assert _render_diff_output(b"") == ""
+
+    def test_spaces_preserved(self):
+        assert _render_diff_output(b"a b\n") == "a b$\n"

{assignment_codeval-0.0.25 → assignment_codeval-0.0.26}/tests/test_evaluate_submissions.py

@@ -4,8 +4,11 @@ import os
 import tempfile
 import zipfile
 
+import click
 import pytest
 
+from assignment_codeval.submissions import _parse_substitutions_file, _apply_substitutions
+
 
 class TestGitHubSubmissionWorkingDir:
     """Tests for GitHub submission working directory detection."""
@@ -156,3 +159,154 @@ class TestGitHubSubmissionWorkingDir:
             correct_path = os.path.join(submission_dir, "custom_dir", "helper.txt")
             assert os.path.exists(correct_path), \
                 f"helper.txt should be at {correct_path}"
+
+
+class TestSubstitutionsFromZip:
+    """Tests for copying SUBSTITUTIONS.txt from zip to comments.txt directory."""
+
+    def _setup_submission(self, tmpdir, assignment_name, zip_contents):
+        """Helper to create a submission directory with a zip file."""
+        codeval_dir = os.path.join(tmpdir, "codeval")
+        submissions_dir = os.path.join(tmpdir, "submissions")
+        os.makedirs(codeval_dir)
+
+        student_dir = os.path.join(submissions_dir, "Course", assignment_name, "12345")
+        submission_dir = os.path.join(student_dir, "submission")
+        os.makedirs(submission_dir)
+
+        # Create zip
+        zip_path = os.path.join(codeval_dir, "support.zip")
+        with zipfile.ZipFile(zip_path, 'w') as zf:
+            for name, content in zip_contents.items():
+                zf.writestr(name, content)
+
+        # Create codeval file
+        codeval_path = os.path.join(codeval_dir, f"{assignment_name}.codeval")
+        with open(codeval_path, "w") as f:
+            f.write("Z support.zip\n")
+            f.write("T echo test\n")
+            f.write("O test\n")
+
+        return codeval_dir, student_dir, submission_dir
+
+    def test_substitutions_copied_to_dirpath(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            codeval_dir, student_dir, submission_dir = self._setup_submission(
+                tmpdir, "TestAssignment",
+                {"SUBSTITUTIONS.txt": "/foo/bar/\n", "helper.txt": "data"}
+            )
+
+            # Simulate the zip extraction logic from evaluate_submissions
+            from zipfile import ZipFile
+            codeval_file = os.path.join(codeval_dir, "TestAssignment.codeval")
+            zip_files = ["support.zip"]
+
+            for zf_name in zip_files:
+                with ZipFile(os.path.join(codeval_dir, zf_name)) as zf:
+                    if "SUBSTITUTIONS.txt" in zf.namelist():
+                        with open(os.path.join(student_dir, "SUBSTITUTIONS.txt"), "wb") as out_f:
+                            out_f.write(zf.read("SUBSTITUTIONS.txt"))
+
+            subs_path = os.path.join(student_dir, "SUBSTITUTIONS.txt")
+            assert os.path.exists(subs_path)
+            with open(subs_path) as f:
+                assert f.read() == "/foo/bar/\n"
+
+    def test_no_substitutions_in_zip(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            codeval_dir, student_dir, submission_dir = self._setup_submission(
+                tmpdir, "TestAssignment",
+                {"helper.txt": "data"}
+            )
+
+            from zipfile import ZipFile
+            zip_files = ["support.zip"]
+
+            for zf_name in zip_files:
+                with ZipFile(os.path.join(codeval_dir, zf_name)) as zf:
+                    if "SUBSTITUTIONS.txt" in zf.namelist():
+                        with open(os.path.join(student_dir, "SUBSTITUTIONS.txt"), "wb") as out_f:
+                            out_f.write(zf.read("SUBSTITUTIONS.txt"))
+
+            assert not os.path.exists(os.path.join(student_dir, "SUBSTITUTIONS.txt"))
+
+
+class TestParseSubstitutionsFile:
+    def test_basic_substitution(self, tmp_path):
+        f = tmp_path / "SUBSTITUTIONS.txt"
+        f.write_text("/foo/bar/\n")
+        result = _parse_substitutions_file(str(f))
+        assert result == [("foo", "bar")]
+
+    def test_custom_delimiter(self, tmp_path):
+        f = tmp_path / "SUBSTITUTIONS.txt"
+        f.write_text("|foo|bar|\n")
+        result = _parse_substitutions_file(str(f))
+        assert result == [("foo", "bar")]
+
+    def test_multiple_lines(self, tmp_path):
+        f = tmp_path / "SUBSTITUTIONS.txt"
+        f.write_text("/foo/bar/\n#baz#qux#\n")
+        result = _parse_substitutions_file(str(f))
+        assert result == [("foo", "bar"), ("baz", "qux")]
+
+    def test_empty_replacement(self, tmp_path):
+        f = tmp_path / "SUBSTITUTIONS.txt"
+        f.write_text("/foo//\n")
+        result = _parse_substitutions_file(str(f))
+        assert result == [("foo", "")]
+
+    def test_blank_lines_skipped(self, tmp_path):
+        f = tmp_path / "SUBSTITUTIONS.txt"
+        f.write_text("\n/foo/bar/\n\n")
+        result = _parse_substitutions_file(str(f))
+        assert result == [("foo", "bar")]
+
+    def test_invalid_missing_trailing_delimiter(self, tmp_path):
+        f = tmp_path / "SUBSTITUTIONS.txt"
+        f.write_text("/foo/bar\n")
+        with pytest.raises(click.ClickException, match=":1:"):
+            _parse_substitutions_file(str(f))
+
+    def test_invalid_too_many_delimiters(self, tmp_path):
+        f = tmp_path / "SUBSTITUTIONS.txt"
+        f.write_text("/foo/bar/baz/\n")
+        with pytest.raises(click.ClickException, match=":1:"):
+            _parse_substitutions_file(str(f))
+
+    def test_invalid_no_delimiter_in_between(self, tmp_path):
+        f = tmp_path / "SUBSTITUTIONS.txt"
+        f.write_text("/foobar/\n")
+        with pytest.raises(click.ClickException, match=":1:"):
+            _parse_substitutions_file(str(f))
+
+    def test_error_reports_correct_line_number(self, tmp_path):
+        f = tmp_path / "SUBSTITUTIONS.txt"
+        f.write_text("/foo/bar/\nbad line\n")
+        with pytest.raises(click.ClickException, match=":2:"):
+            _parse_substitutions_file(str(f))
+
+    def test_delimiter_in_pattern_via_different_delimiter(self, tmp_path):
+        f = tmp_path / "SUBSTITUTIONS.txt"
+        f.write_text("|a/b|c/d|\n")
+        result = _parse_substitutions_file(str(f))
+        assert result == [("a/b", "c/d")]
+
+
+class TestApplySubstitutions:
+    def test_single_substitution(self):
+        assert _apply_substitutions("hello foo world", [("foo", "bar")]) == "hello bar world"
+
+    def test_multiple_substitutions(self):
+        result = _apply_substitutions("aaa bbb", [("aaa", "xxx"), ("bbb", "yyy")])
+        assert result == "xxx yyy"
+
+    def test_literal_pattern(self):
+        result = _apply_substitutions("a^[[1mb", [("^[[1m", "<b>B</b>")])
+        assert result == "a<b>B</b>b"
+
+    def test_empty_substitutions(self):
+        assert _apply_substitutions("hello", []) == "hello"
+
+    def test_deletion(self):
+        assert _apply_substitutions("hello world", [("world", "")]) == "hello "

assignment_codeval-0.0.25/tests/test_codeval.py

@@ -1,35 +0,0 @@
-"""Pytest tests for CodEval test files."""
-import glob
-import os
-import subprocess
-import pytest
-
-# Get all .codeval test files
-TEST_DIR = os.path.dirname(os.path.abspath(__file__))
-CODEVAL_FILES = sorted(glob.glob(os.path.join(TEST_DIR, "*.codeval")))
-
-
-def get_test_ids():
-    """Generate test IDs from filenames."""
-    return [os.path.basename(f).replace(".codeval", "") for f in CODEVAL_FILES]
-
-
-@pytest.mark.parametrize("codeval_file", CODEVAL_FILES, ids=get_test_ids())
-def test_codeval(codeval_file):
-    """Run a .codeval test file and verify it passes."""
-    test_name = os.path.basename(codeval_file)
-
-    result = subprocess.run(
-        ["assignment-codeval", "run-evaluation", test_name],
-        cwd=TEST_DIR,
-        capture_output=True,
-        text=True,
-    )
-
-    # Print output for debugging
-    if result.stdout:
-        print(result.stdout)
-    if result.stderr:
-        print(result.stderr)
-
-    assert result.returncode == 0, f"Test {test_name} failed with exit code {result.returncode}"