assignment-codeval 0.0.24__py3-none-any.whl → 0.0.26__py3-none-any.whl

assignment_codeval/cli.py

@@ -5,6 +5,7 @@ from assignment_codeval.check_grading import check_grading
 from assignment_codeval.commons import set_config
 from assignment_codeval.create_assignment import create_assignment
 from assignment_codeval.evaluate import run_evaluation
+from assignment_codeval.export_tests import export_tests
 from assignment_codeval.github_connect import github_setup_repo
 from assignment_codeval.install_assignment import install_assignment
 from assignment_codeval.recent_comments import recent_comments
@@ -26,6 +27,7 @@ cli.add_command(list_codeval_assignments)
 cli.add_command(install_assignment)
 cli.add_command(recent_comments)
 cli.add_command(check_grading)
+cli.add_command(export_tests)
 cli.add_command(get_benchmark_command())
 
 if __name__ == "__main__":

assignment_codeval/evaluate.py

@@ -1,5 +1,6 @@
 #! /usr/bin/python3
 
+import ast
 import os
 import re
 import subprocess
@@ -27,6 +28,7 @@ def get_testing_path(filename: str) -> str:
 test_args = ""
 cmps = []
 timeout_val = 10
+output_length_limit = 4096
 expected_exit_code = -1
 test_case_count = 0
 test_case_hint = ""
@@ -80,52 +82,178 @@ def compile_code(compile_command):
         sys.exit(1)
 
 
-def check_function(args):
-    """Will be followed by a function name and a list of files to check to ensure that the function
-    is used by one of those files.
+###########################################################
+# Function detection helpers
+###########################################################
 
-    Arguments:
-        function_name: the function name to check files for usage of
-        *files: the files to check for the function name
 
-    Returns:
-        None
+def _detect_language(files):
+    """Return 'c_cpp', 'java', 'python', or 'unknown' based on file extensions."""
+    for f in files:
+        ext = os.path.splitext(f)[1].lower()
+        if ext in ('.c', '.cpp', '.cc', '.cxx', '.h', '.hpp'):
+            return 'c_cpp'
+        if ext == '.java':
+            return 'java'
+        if ext == '.py':
+            return 'python'
+    return 'unknown'
+
+
+def _find_compiled_artifact(source_file):
+    """Return the compiled artifact for a source file, or None if not found.
+
+    Checks (in order): same base name with .o, .exe, and no extension.
     """
-    check_test()
-    args = args.split()
-    function_name = args[0]
-    files = args[1:]
+    base = os.path.splitext(source_file)[0]
+    for candidate in (base + '.o', base + '.exe', base):
+        if os.path.exists(candidate):
+            return candidate
+    return None
 
-    # Match:
-    #  [1] start of line OR non-identifier character
-    #  [2] function name
-    #  [3] optional whitespace
-    #  [4] opening parenthesis (function call)
-    regex = rf'(^|[^[:alnum:]_]){function_name}[[:space:]]*\('
 
-    # Strip comments, then search
+def _function_used_in_c_cpp(function_name, files):
+    """Use objdump on compiled artifacts to detect C/C++ function usage.
+
+    Returns True if found, False if artifact exists but function absent,
+    None if no compiled artifact was found (caller should fall back).
+    """
+    found_artifact = False
+    for source_file in files:
+        artifact = _find_compiled_artifact(source_file)
+        if artifact is None:
+            continue
+        found_artifact = True
+        # objdump -t dumps the symbol table; pipe through c++filt to demangle C++ names.
+        # Mangled C++ names embed the original identifier, so grep -w finds exact matches.
+        cmd = f"objdump -t '{artifact}' | c++filt | grep -w '{function_name}'"
+        result = subprocess.run(
+            ["bash", "-c", cmd],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        if result.returncode == 0:
+            return True
+    if not found_artifact:
+        return None
+    return False
+
+
+def _function_used_in_java(function_name, files):
+    """Use javap on compiled .class files to detect Java method usage.
+
+    Returns True if found, False if class file exists but method absent,
+    None if no .class file was found (caller should fall back).
+    """
+    found_class = False
+    for source_file in files:
+        base = os.path.splitext(source_file)[0]
+        class_file = base + '.class'
+        if not os.path.exists(class_file):
+            continue
+        found_class = True
+        # javap -c disassembles bytecode; method invocations reference the method name.
+        result = subprocess.run(
+            ["javap", "-c", class_file],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        if function_name.encode() in result.stdout:
+            return True
+    if not found_class:
+        return None
+    return False
+
+
+def _function_used_in_python(function_name, files):
+    """Use the ast module to detect function calls in Python source files.
+
+    Returns True if a matching call is found, False otherwise.
+    Handles both direct calls (func()) and attribute calls (obj.func()).
+    """
+    for source_file in files:
+        try:
+            with open(source_file, 'r') as f:
+                source = f.read()
+            tree = ast.parse(source)
+            for node in ast.walk(tree):
+                if isinstance(node, ast.Call):
+                    func = node.func
+                    if isinstance(func, ast.Name) and func.id == function_name:
+                        return True
+                    if isinstance(func, ast.Attribute) and func.attr == function_name:
+                        return True
+        except (SyntaxError, OSError):
+            continue
+    return False
+
+
+def _function_used_regex(function_name, files):
+    """Legacy regex-based fallback used when no compiled artifact is available."""
+    regex = rf'(^|[^[:alnum:]_]){function_name}[[:space:]]*\('
     cmd = (
         "sed -E "
         "'"
-        "s://.*$::g; " # C/C++ single-line comments
-        "s:#.*$::g; " # Python single-line comments
+        "s://.*$::g; "
+        "s:#.*$::g; "
         ":a; /\\/\\*/{N; s:/\\*.*?\\*/::g; ba}"
         "' "
         + " ".join(files)
         + f" | grep -E '{regex}'"
     )
-
-    function_popen = subprocess.Popen(
+    result = subprocess.run(
         ["bash", "-c", cmd],
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
     )
+    return result.returncode == 0
 
-    function_popen.communicate()
-    if function_popen.returncode:
-        print(f"Not using {function_name} FAILED")
-    else:
+
+def _is_function_used(function_name, files):
+    """Detect whether function_name is used in the given files.
+
+    Dispatches to the appropriate tool based on file language:
+      - C/C++:  objdump on compiled artifact (falls back to regex if none found)
+      - Java:   javap on .class file (falls back to regex if none found)
+      - Python: ast module on source
+      - Other:  regex on source
+    """
+    lang = _detect_language(files)
+    if lang == 'python':
+        return _function_used_in_python(function_name, files)
+    if lang == 'java':
+        result = _function_used_in_java(function_name, files)
+        if result is None:
+            return _function_used_regex(function_name, files)
+        return result
+    if lang == 'c_cpp':
+        result = _function_used_in_c_cpp(function_name, files)
+        if result is None:
+            return _function_used_regex(function_name, files)
+        return result
+    return _function_used_regex(function_name, files)
+
+
+def check_function(args):
+    """Will be followed by a function name and a list of files to check to ensure that the function
+    is used by one of those files.
+
+    Arguments:
+        function_name: the function name to check files for usage of
+        *files: the files to check for the function name
+
+    Returns:
+        None
+    """
+    check_test()
+    args = args.split()
+    function_name = args[0]
+    files = args[1:]
+
+    if _is_function_used(function_name, files):
         print(f"Used {function_name} PASSED")
+    else:
+        print(f"Not using {function_name} FAILED")
 
 def check_object(args):
     """Will be followed by an object and a list of files to ensure that the function is
@@ -228,7 +356,7 @@ def check_not_function(args):
     is not used by any of those files.
 
     Arguments:
-        function_name: the funcion name to check files for usage of
+        function_name: the function name to check files for usage of
         *files: the files to check for the function name
 
     Returns:
@@ -239,18 +367,23 @@ def check_not_function(args):
     function_name = args[0]
     files = args[1:]
 
-    # Surpress output
-    function_popen = subprocess.Popen(
-        ["grep", f"[^[:alpha:]]{function_name}[[:space:]]*("] + files,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-    )
-
-    function_popen.communicate()
-    if function_popen.returncode:
-        print(f"used{function_name} PASSED")
+    if _is_function_used(function_name, files):
+        print(f"Used {function_name} FAILED")
     else:
-        print(f"not using {function_name} FAILED")
+        print(f"Not using {function_name} PASSED")
+
+
+def print_label(text):
+    """Print a label/message to stdout.
+
+    Arguments:
+        text: the text to print
+
+    Returns:
+        None
+    """
+    check_test()
+    print(text)
 
 
 def run_command(command):
@@ -511,6 +644,19 @@ def timeout(timeout_sec):
     timeout_val = float(timeout_sec)
 
 
+def output_length(length):
+    """Specifies the maximum number of bytes of diff output to render. Defaults to 4096.
+
+    Arguments:
+        length: maximum output length in bytes
+
+    Returns:
+        None
+    """
+    global output_length_limit
+    output_length_limit = int(length)
+
+
 def exit_code(test_case_exit_code):
     """Specifies the expected exit code for a test case. Defaults to zero.
 
@@ -578,6 +724,7 @@ tag_func_map = {
     "CO" : check_object,
     "CC": check_container,
     "NCF": check_not_function,
+    "PRINT": print_label,
     "CMD": run_command,
     "TCMD": run_command_noerror,
     "CMP": compare,
@@ -593,6 +740,7 @@ tag_func_map = {
     "EB": check_error_bare,
     "HINT": hint,
     "TO": timeout,
+    "OLEN": output_length,
     "X": exit_code,
     "SS": start_server,
 }
@@ -774,6 +922,93 @@ def parse_diff(diff_lines: list[str], testing_dir: str):
                     outfile.write(line)
 
 
+def _render_diff_output(raw_bytes: bytes) -> str:
+    """Render raw bytes for diff output, making unprintable characters visible.
+
+    Replaces the shell pipeline `cat -te | head -22` with pure Python.
+    """
+    result = []
+    i = 0
+    n = len(raw_bytes)
+
+    while i < n:
+        b = raw_bytes[i]
+
+        if b == 0x0A:  # newline
+            result.append("$\n")
+            i += 1
+        elif b == 0x20 or (0x21 <= b <= 0x7E):  # space or printable ASCII
+            result.append(chr(b))
+            i += 1
+        elif b <= 0x1F:  # control chars (except newline handled above)
+            result.append("^" + chr(b + 0x40))
+            i += 1
+        elif b == 0x7F:  # DEL
+            result.append("^?")
+            i += 1
+        elif b >= 0x80:
+            # Try to decode as UTF-8 multi-byte sequence
+            seq_len = 0
+            if (b & 0xE0) == 0xC0:
+                seq_len = 2
+            elif (b & 0xF0) == 0xE0:
+                seq_len = 3
+            elif (b & 0xF8) == 0xF0:
+                seq_len = 4
+
+            decoded_char = None
+            if seq_len >= 2 and i + seq_len <= n:
+                try:
+                    decoded_char = raw_bytes[i:i + seq_len].decode("utf-8")
+                except UnicodeDecodeError:
+                    pass
+
+            if decoded_char and len(decoded_char) == 1 and decoded_char.isprintable():
+                result.append(decoded_char)
+                i += seq_len
+            elif decoded_char and len(decoded_char) == 1:
+                # Valid UTF-8 but non-printable: render each byte as \xCC
+                for j in range(seq_len):
+                    result.append(f"\\x{raw_bytes[i + j]:02X}")
+                i += seq_len
+            else:
+                # Invalid UTF-8 byte
+                result.append(f"\\x{b:02X}")
+                i += 1
+        else:
+            result.append(chr(b))
+            i += 1
+
+    output = "".join(result)
+
+    # If output doesn't end with \n, append $ at the very end
+    if output and not output.endswith("\n"):
+        output += "$"
+
+    return output
+
+
+def _find_touched_files(pre_run_timestamp):
+    """Find files touched (modified or accessed) since pre_run_timestamp.
+
+    Recursively scans the current working directory, ignoring TESTING_DIR.
+    """
+    touched = []
+    for dirpath, dirnames, filenames in os.walk("."):
+        # Prune TESTING_DIR from traversal
+        if TESTING_DIR in dirnames:
+            dirnames.remove(TESTING_DIR)
+        for filename in filenames:
+            filepath = os.path.join(dirpath, filename)
+            try:
+                st = os.stat(filepath)
+                if st.st_mtime > pre_run_timestamp or st.st_atime > pre_run_timestamp:
+                    touched.append(filepath)
+            except OSError:
+                continue
+    return touched
+
+
 def check_test():
     global test_args
     if test_args == "":
@@ -782,6 +1017,9 @@ def check_test():
     print(f"Test case {test_case_count} of {test_case_total}")
     passed = True
 
+    # Record timestamp before running student code for file tracking
+    pre_run_timestamp = time.time()
+
     with open(get_testing_path("fileinput"), "rb") as fileinput, open(
         get_testing_path("youroutput"), "w"
     ) as youroutput, open(get_testing_path("yourerror"), "w") as yourerror:
@@ -797,27 +1035,28 @@ def check_test():
         print(f"Took more than {timeout_val} seconds to run. FAIL")
         passed = False
 
+    # Find files touched by student's program
+    touched_files = _find_touched_files(pre_run_timestamp)
+
     # Difflog handling
     with open(get_testing_path("difflog"), "w") as outfile:
         diff_popen = subprocess.Popen(
-            f"diff -U1 -a ./{TESTING_DIR}/youroutput ./{TESTING_DIR}/expectedoutput | cat -te | head -22",
-            shell=True,
-            stdout=outfile,
-            stderr=outfile,
-            text=True,
+            ["diff", "-U1", "-a",
+             f"./{TESTING_DIR}/youroutput", f"./{TESTING_DIR}/expectedoutput"],
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
         )
-        diff_popen.communicate()
+        raw_output, _ = diff_popen.communicate()
+        outfile.write(_render_diff_output(raw_output[:output_length_limit]))
 
     # Append to difflog second time around
     with open(get_testing_path("difflog"), "a") as outfile:
         diff_popen = subprocess.Popen(
-            f"diff -U1 -a ./{TESTING_DIR}/yourerror ./{TESTING_DIR}/expectederror | cat -te | head -22",
-            shell=True,
-            stdout=outfile,
-            stderr=outfile,
-            text=True,
+            ["diff", "-U1", "-a",
+             f"./{TESTING_DIR}/yourerror", f"./{TESTING_DIR}/expectederror"],
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
         )
-        diff_popen.communicate()
+        raw_output, _ = diff_popen.communicate()
+        outfile.write(_render_diff_output(raw_output[:output_length_limit]))
 
     # Now read all the lines to accumulate both diffs
     with open(get_testing_path("difflog"), "r") as infile:
@@ -872,6 +1111,33 @@ def check_test():
                     # Print entire file
                     print("".join(file_lines))
 
+            # Show diffs for files touched by student's program
+            if touched_files and cmps:
+                expected_map = {}
+                for file_pair in cmps:
+                    if len(file_pair) >= 2:
+                        expected_map[os.path.normpath(file_pair[0])] = file_pair[1]
+
+                for filepath in touched_files:
+                    normalized = os.path.normpath(filepath)
+                    if normalized in expected_map:
+                        expected_file = expected_map[normalized]
+                        if os.path.exists(expected_file):
+                            diff_proc = subprocess.Popen(
+                                ["diff", "-U1", "-a", filepath, expected_file],
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE,
+                                text=True,
+                            )
+                            diff_output, _ = diff_proc.communicate()
+                            if diff_output:
+                                print(f"    File output diff ({os.path.basename(filepath)}):")
+                                lines = diff_output.splitlines()
+                                for line in lines[:22]:
+                                    print(f"    {line}")
+                                if len(lines) > 22:
+                                    print(f"    ... ({len(lines) - 22} more lines)")
+
         cleanup()
 
         # Exit program after failed test case

assignment_codeval/export_tests.py

@@ -0,0 +1,187 @@
+import os
+import re
+import zipfile
+
+import click
+
+
+def _read_referenced_file(filename, zip_archives):
+    """Read a file referenced by IF/OF/EF from zip archives.
+
+    Args:
+        filename: the filename from the tag
+        zip_archives: list of opened ZipFile objects
+
+    Returns:
+        file contents as string
+    """
+    for zf in zip_archives:
+        try:
+            return zf.read(filename).decode("utf-8")
+        except KeyError:
+            continue
+
+    raise FileNotFoundError(
+        f"Referenced file '{filename}' not found in zip archives"
+    )
+
+
+def parse_codeval_tests(codeval_file):
+    """Parse a codeval file and extract test cases.
+
+    Returns a list of dicts with keys:
+        num, command, input, output, error, exit_code, hidden
+    """
+    codeval_dir = os.path.dirname(os.path.abspath(codeval_file))
+
+    with open(codeval_file, "r") as f:
+        lines = f.readlines()
+
+    # First pass: collect Z tags to open zip archives
+    tag_pattern = r"([A-Z_]+) (.*)"
+    zip_paths = []
+    in_crt_hw = False
+    for line in lines:
+        stripped = line.strip()
+        if stripped.startswith("CRT_HW"):
+            in_crt_hw = not in_crt_hw
+            continue
+        if in_crt_hw:
+            continue
+        m = re.match(tag_pattern, line)
+        if m and m.group(1) == "Z":
+            zip_paths.append(os.path.join(codeval_dir, m.group(2).strip()))
+
+    # Open zip archives for the duration of parsing
+    zip_archives = []
+    try:
+        for zp in zip_paths:
+            if os.path.exists(zp):
+                zip_archives.append(zipfile.ZipFile(zp, "r"))
+
+        tests = _parse_codeval_lines(lines, zip_archives)
+    finally:
+        for zf in zip_archives:
+            zf.close()
+
+    return tests
+
+
+def _parse_codeval_lines(lines, zip_archives):
+    """Parse codeval lines into test case dicts."""
+    tag_pattern = r"([A-Z_]+) (.*)"
+    in_crt_hw_block = False
+    tests = []
+    current = None
+    test_num = 0
+
+    def finalize():
+        if current is not None:
+            tests.append(current)
+
+    for line in lines:
+        stripped = line.strip()
+        if not stripped or stripped.startswith("#"):
+            continue
+
+        if stripped.startswith("CRT_HW"):
+            in_crt_hw_block = not in_crt_hw_block
+            continue
+
+        if in_crt_hw_block:
+            continue
+
+        tag_match = re.match(tag_pattern, line)
+        if not tag_match:
+            continue
+
+        tag = tag_match.group(1)
+        args = tag_match.group(2)
+
+        if tag in ("T", "HT"):
+            finalize()
+            test_num += 1
+            current = {
+                "num": test_num,
+                "command": args,
+                "input": "",
+                "output": "",
+                "error": "",
+                "exit_code": -1,
+                "hidden": tag == "HT",
+            }
+        elif tag == "TCMD":
+            finalize()
+            test_num += 1
+            current = {
+                "num": test_num,
+                "command": args,
+                "input": "",
+                "output": "",
+                "error": "",
+                "exit_code": -1,
+                "hidden": False,
+            }
+        elif current is not None:
+            if tag == "I":
+                current["input"] += args + "\n"
+            elif tag == "IB":
+                current["input"] += args
+            elif tag == "IF":
+                current["input"] += _read_referenced_file(
+                    args.strip(), zip_archives
+                )
+            elif tag == "O":
+                current["output"] += args + "\n"
+            elif tag == "OB":
+                current["output"] += args
+            elif tag == "OF":
+                current["output"] += _read_referenced_file(
+                    args.strip(), zip_archives
+                )
+            elif tag == "E":
+                current["error"] += args + "\n"
+            elif tag == "EB":
+                current["error"] += args
+            elif tag == "X":
+                current["exit_code"] = int(args.strip())
+
+    finalize()
+    return tests
+
+
+@click.command()
+@click.argument("codeval_file", type=click.Path(exists=True))
+@click.option("--include-hidden", is_flag=True, help="Include hidden test cases")
+@click.option("--output", "-o", default=None, help="Output zip file path")
+def export_tests(codeval_file, include_hidden, output):
+    """Export test cases from a codeval file into a zip archive.
+
+    Creates a zip with in.X, out.X, err.X files for each test case
+    and a TESTS.md summary.
+    """
+    tests = parse_codeval_tests(codeval_file)
+
+    if output is None:
+        basename = os.path.splitext(os.path.basename(codeval_file))[0]
+        output = f"{basename}_tests.zip"
+
+    filtered = [t for t in tests if include_hidden or not t["hidden"]]
+
+    with zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as zf:
+        tests_md = ""
+        for test in filtered:
+            num = test["num"]
+            zf.writestr(f"in.{num}", test["input"])
+            zf.writestr(f"out.{num}", test["output"])
+            zf.writestr(f"err.{num}", test["error"])
+
+            tests_md += f"# Test {num}\n"
+            tests_md += f"command: `{test['command']}`\n"
+            if test["exit_code"] != -1:
+                tests_md += f"expected exit code: {test['exit_code']}\n"
+            tests_md += "\n"
+
+        zf.writestr("TESTS.md", tests_md)
+
+    click.echo(f"Exported {len(filtered)} test(s) to {output}")

assignment_codeval/submissions.py

@@ -5,6 +5,7 @@ import subprocess
 import time
 from configparser import ConfigParser
 from datetime import datetime, timedelta, timezone
+from zoneinfo import ZoneInfo
 from functools import cache
 from tempfile import TemporaryDirectory
 from zipfile import ZipFile
@@ -61,10 +62,160 @@ def get_github_repo_url(course, user_id, config_parser, github_field="github"):
         return None
 
 
+def _get_canvas_config():
+    """Get Canvas URL and token from config."""
+    parser = ConfigParser()
+    config_file = click.get_app_dir("codeval.ini")
+    parser.read(config_file)
+    return parser['SERVER']['url'].rstrip('/'), parser['SERVER']['token']
+
+
+def write_html_file(dirpath):
+    import html as _html
+
+    assignment_name = student_name = attempt_no = last_submitted = ""
+    with open(f"{dirpath}/metadata.txt", "r") as f:
+        for line in f:
+            if line.startswith("assignment="):
+                assignment_name = line.strip().split("=", 1)[1]
+            elif line.startswith("name="):
+                student_name = line.strip().split("=", 1)[1]
+            elif line.startswith("attempt="):
+                attempt_no = line.strip().split("=", 1)[1]
+            elif line.startswith("date="):
+                last_submitted = line.strip().split("=", 1)[1]
+
+    with open(f"{dirpath}/comments.txt", "r") as f:
+        comments_content = f.read()
+
+    # Color each line based on PASS/FAIL/error keywords
+    def colorize_lines(text):
+        lines_html = []
+        for line in text.split('\n'):
+            escaped = _html.escape(line)
+            if line.startswith('PASS'):
+                lines_html.append(f'<span class="pass">{escaped}</span>')
+            elif line.startswith('FAIL'):
+                lines_html.append(f'<span class="fail">{escaped}</span>')
+            elif any(line.startswith(k) for k in ('ERROR', 'TOOK LONGER', 'FAILED')):
+                lines_html.append(f'<span class="error">{escaped}</span>')
+            else:
+                lines_html.append(escaped)
+        return '\n'.join(lines_html)
+
+    try:
+        dt = datetime.strptime(last_submitted, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone.utc)
+        dt = dt.astimezone(ZoneInfo('America/Los_Angeles'))
+        last_submitted = dt.strftime('%B %d, %Y at %I:%M %p %Z')
+    except (ValueError, TypeError):
+        pass
+
+    pass_count = sum(1 for l in comments_content.split('\n') if l.startswith('PASS'))
+    fail_count = sum(1 for l in comments_content.split('\n') if l.startswith('FAIL'))
+
+    template_path = os.path.join(os.path.dirname(__file__), 'test_template.html')
+    with open(template_path, "r", encoding="utf-8") as f:
+        html_content = f.read()
+
+    html_content = (html_content
+        .replace('{{ASSIGNMENT_NAME}}', _html.escape(assignment_name))
+        .replace('{{STUDENT_NAME}}', _html.escape(student_name))
+        .replace('{{SUBMITTED}}', _html.escape(last_submitted))
+        .replace('{{ATTEMPT}}', _html.escape(attempt_no))
+        .replace('{{PASS_COUNT}}', str(pass_count))
+        .replace('{{FAIL_COUNT}}', str(fail_count))
+        .replace('{{OUTPUT}}', colorize_lines(comments_content))
+    )
+
+    with open(f"{dirpath}/results.html", "w", encoding="utf-8") as dst:
+        dst.write(html_content)
+
+
+def upload_file_for_comment(canvas, course_id, assignment_id, user_id, file_path):
+    """
+    Upload a file to attach to a submission comment.
+    Returns the file_id to use when creating/editing the comment.
+    """
+    canvas_url, canvas_token = _get_canvas_config()
+    auth = {'Authorization': f'Bearer {canvas_token}'}
+
+    file_name = os.path.basename(file_path)
+    file_size = os.path.getsize(file_path)
+
+    # Step 1: Request upload URL
+    r1 = requests.post(
+        f'{canvas_url}/api/v1/courses/{course_id}/assignments/{assignment_id}/submissions/{user_id}/comments/files',
+        headers=auth,
+        data={'name': file_name, 'size': file_size, 'content_type': 'text/html'}
+    )
+    r1.raise_for_status()
+    upload_info = r1.json()
+
+    # Step 2: Upload file to storage
+    with open(file_path, 'rb') as f:
+        r2 = requests.post(
+            upload_info['upload_url'],
+            data=upload_info['upload_params'],
+            files={'file': (file_name, f, 'text/html')},
+            allow_redirects=False
+        )
+
+    # Step 3: Confirm upload
+    if r2.status_code in (301, 302, 303):
+        r3 = requests.get(r2.headers['Location'], headers=auth)
+        r3.raise_for_status()
+        return r3.json()['id']
+    else:
+        r2.raise_for_status()
+        return r2.json()['id']
+
+
+def delete_submission_comment(canvas, course_id, assignment_id, user_id, comment_id):
+    """Delete a submission comment."""
+    canvas_url, canvas_token = _get_canvas_config()
+    resp = requests.delete(
+        f'{canvas_url}/api/v1/courses/{course_id}/assignments/{assignment_id}/submissions/{user_id}/comments/{comment_id}',
+        headers={'Authorization': f'Bearer {canvas_token}'}
+    )
+    resp.raise_for_status()
+
+
+def _parse_substitutions_file(filepath):
+    """Parse a substitutions file where each line has the form /pattern/replacement/.
+
+    The first character of each line defines the delimiter. The delimiter must appear
+    exactly once in-between (separating pattern from replacement) and once at the end.
+    Returns a list of (pattern, replacement) tuples.
+    Raises click.ClickException on parse errors.
+    """
+    substitutions = []
+    with open(filepath, "r") as fd:
+        for lineno, line in enumerate(fd, start=1):
+            line = line.rstrip("\n")
+            if not line:
+                continue
+            delim = line[0]
+            parts = line[1:].split(delim)
+            if len(parts) != 3 or parts[2] != "":
+                raise click.ClickException(
+                    f"{filepath}:{lineno}: invalid substitution line: {line}"
+                )
+            substitutions.append((parts[0], parts[1]))
+    return substitutions
+
+
+def _apply_substitutions(comment, substitutions):
+    """Apply a list of (pattern, replacement) substitutions to comment text."""
+    for pattern, replacement in substitutions:
+        comment = comment.replace(pattern, replacement)
+    return comment
+
+
 @click.command()
 @click.argument("submissions_dir", metavar="SUBMISSIONS_DIR")
 @click.option("--codeval-prefix", help="prefix for codeval comments", default="codeval: ", show_default=True)
-def upload_submission_comments(submissions_dir, codeval_prefix):
+@click.option("--delete", is_flag=True, help="deletes the previous comment before uploading")
+def upload_submission_comments(submissions_dir, codeval_prefix, delete):
     """
     Upload comments for submissions from a directory.
 
@@ -75,31 +226,49 @@ def upload_submission_comments(submissions_dir, codeval_prefix):
     if the file comments.txt.sent exists, the comment has already been uploaded and will be skipped.
     """
     (canvas, user) = connect_to_canvas()
-    clean_submissions_dir = submissions_dir.rstrip('/')
+    clean_submissions_dir = submissions_dir.rstrip('/').replace('\\', '/')
     for dirpath, dirnames, filenames in os.walk(clean_submissions_dir):
-        match = re.match(fr'^{clean_submissions_dir}/([^/]+)/([^/]+)/([^/]+)$', dirpath)
+        dirpath = dirpath.replace('\\', '/')
+        match = re.match(fr'^{re.escape(clean_submissions_dir)}/([^/]+)/([^/]+)/([^/]+)$', dirpath)
         if match:
             course_name = match.group(1)
             assignment_name = match.group(2)
             student_id = match.group(3)
-            if "comments.txt" in filenames:
-                if "comments.txt.sent" in filenames:
-                    info(f"skipping already uploaded comments for {student_id} in {course_name}: {assignment_name}")
-                else:
-                    info(f"uploading comments for {student_id} in {course_name}: {assignment_name}")
-                    course = get_course(canvas, course_name)
-                    assignment = get_assignment(course, assignment_name)
+            if "comments.txt.sent" in filenames:
+                info(f"skipping already uploaded comments for {student_id} in {course_name}: {assignment_name}")
+            elif "comments.txt" in filenames:
+                info(f"uploading comments for {student_id} in {course_name}: {assignment_name}")
+                course = get_course(canvas, course_name)
+                assignment = get_assignment(course, assignment_name)
+                submission = get_submissions_by_id(assignment).get(student_id)
+                if submission:
+                    write_html_file(dirpath)
+                    file_id = upload_file_for_comment(canvas, course.id, assignment.id, student_id, f"{dirpath}/results.html")
                     with open(f"{dirpath}/comments.txt", "r") as fd:
                         comment = fd.read(4096)
-                        # nulls seem to be very problematic for canvas
                         comment = comment.replace("\0", "\\0").strip().replace("<", "&lt;")
-                        submission = get_submissions_by_id(assignment).get(student_id)
-                        if submission:
-                            submission.edit(comment={'text_comment': f'{codeval_prefix}<pre>\n{comment}</pre>'})
-                        else:
-                            warn(f"no submission found for {student_id} in {course_name}: {assignment_name}")
-                    with open(f"{dirpath}/comments.txt.sent", "w") as fd:
-                        fd.write(datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'))
+                    subs_file = f"{dirpath}/SUBSTITUTIONS.txt"
+                    if os.path.isfile(subs_file):
+                        substitutions = _parse_substitutions_file(subs_file)
+                        comment = _apply_substitutions(comment, substitutions)
+                    if delete:
+                        all_comments = sorted(submission.submission_comments, key=lambda c: c['created_at'])
+                        if all_comments:
+                            last_comment_id = all_comments[-1]["id"]
+                            delete_submission_comment(canvas, course.id, assignment.id, student_id, last_comment_id)
+                    canvas_url, canvas_token = _get_canvas_config()
+                    requests.put(
+                        f'{canvas_url}/api/v1/courses/{course.id}/assignments/{assignment.id}/submissions/{student_id}',
+                        headers={'Authorization': f'Bearer {canvas_token}'},
+                        data={
+                            'comment[text_comment]': f'{codeval_prefix.rstrip()}\n<pre>{comment}</pre>',
+                            'comment[file_ids][]': file_id,
+                        }
+                    ).raise_for_status()
+                else:
+                    warn(f"no submission found for {student_id} in {course_name}: {assignment_name}")
+                with open(f"{dirpath}/comments.txt.sent", "w") as fd:
+                    fd.write(datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'))
 
 
 @click.command()
@@ -178,6 +347,9 @@ def evaluate_submissions(codeval_dir, submissions_dir):
         if not move_to_next_submission:
             for zf_name in zip_files:
                 with ZipFile(os.path.join(codeval_dir, zf_name)) as zf:
+                    if "SUBSTITUTIONS.txt" in zf.namelist():
+                        with open(os.path.join(dirpath, "SUBSTITUTIONS.txt"), "wb") as out_f:
+                            out_f.write(zf.read("SUBSTITUTIONS.txt"))
                     for f in zf.infolist():
                         dest_dir = os.path.join(submission_dir, assignment_working_dir)
                         zf.extract(f, dest_dir)
@@ -334,7 +506,7 @@ def _download_assignment_submissions(canvas, course, assignment, target_dir, inc
             continue
 
         submission_comments = [c['created_at'] for c in submission.submission_comments if
-                               'comment' in c and c['comment'].startswith(codeval_prefix)]
+                               'comment' in c and c['comment'].startswith(codeval_prefix.rstrip())]
         submission_comments.sort()
         if submission_comments:
             last_comment_date = submission_comments[-1]
@@ -397,7 +569,7 @@ github_repo={github_repo or ''}""", file=fd)
 @cache
 def get_submissions_by_id(assignment):
     submissions_by_id = {}
-    for submission in assignment.get_submissions(include=["user"]):
+    for submission in assignment.get_submissions(include=["submission_comments", "user"]):
         student_id = str(submission.user_id)
         submissions_by_id[student_id] = submission
     return submissions_by_id

assignment_codeval/test_template.html

@@ -0,0 +1,111 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <title>CodEval Results – {{ASSIGNMENT_NAME}}</title>
+  <style>
+    body {
+      font-family: system-ui, -apple-system, "Segoe UI", sans-serif;
+      background: #f0f2f5;
+      margin: 0;
+      padding: 24px;
+    }
+    .card {
+      max-width: 860px;
+      margin: 0 auto;
+      background: #fff;
+      border-radius: 8px;
+      box-shadow: 0 2px 8px rgba(0,0,0,0.08);
+      overflow: hidden;
+    }
+    .header {
+      background: #1a73e8;
+      color: #fff;
+      padding: 16px 24px;
+    }
+    .header h1 {
+      margin: 0;
+      font-size: 1.25rem;
+      font-weight: 600;
+    }
+    .header p {
+      margin: 4px 0 0;
+      font-size: 0.9rem;
+      opacity: 0.85;
+    }
+    .meta {
+      display: flex;
+      flex-wrap: wrap;
+      gap: 12px;
+      padding: 14px 24px;
+      background: #f8f9fa;
+      border-bottom: 1px solid #e0e0e0;
+      font-size: 0.88rem;
+      color: #444;
+    }
+    .meta span strong {
+      color: #222;
+    }
+    .score {
+      display: flex;
+      gap: 12px;
+      padding: 12px 24px;
+      border-bottom: 1px solid #e0e0e0;
+      font-size: 0.9rem;
+    }
+    .score .pass-count {
+      color: #2e7d32;
+      font-weight: 600;
+    }
+    .score .fail-count {
+      color: #c62828;
+      font-weight: 600;
+    }
+    .output {
+      padding: 16px 24px;
+    }
+    .output h2 {
+      font-size: 0.95rem;
+      font-weight: 600;
+      margin: 0 0 8px;
+      color: #333;
+    }
+    pre {
+      margin: 0;
+      font-family: "SFMono-Regular", Menlo, Monaco, Consolas, monospace;
+      font-size: 0.82rem;
+      line-height: 1.6;
+      background: #f8f9fa;
+      border: 1px solid #e0e0e0;
+      border-radius: 6px;
+      padding: 12px 14px;
+      white-space: pre-wrap;
+      word-break: break-word;
+    }
+    .pass { color: #2e7d32; font-weight: 600; }
+    .fail { color: #c62828; font-weight: 600; }
+    .error { color: #e65100; font-weight: 600; }
+</style>
+</head>
+<body>
+  <div class="card">
+    <div class="header">
+      <h1>CodEval Results</h1>
+      <p>{{ASSIGNMENT_NAME}}</p>
+    </div>
+    <div class="meta">
+      <span><strong>Student:</strong> {{STUDENT_NAME}}</span>
+      <span><strong>Submitted:</strong> {{SUBMITTED}}</span>
+      <span><strong>Attempt:</strong> {{ATTEMPT}}</span>
+    </div>
+    <div class="score">
+      <span class="pass-count">&#10003; {{PASS_COUNT}} passed</span>
+      <span class="fail-count">&#10007; {{FAIL_COUNT}} failed</span>
+    </div>
+    <div class="output">
+      <h2>Output</h2>
+      <pre>{{OUTPUT}}</pre>
+    </div>
+  </div>
+</body>
+</html>

{assignment_codeval-0.0.24.dist-info → assignment_codeval-0.0.26.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assignment-codeval
-Version: 0.0.24
+Version: 0.0.26
 Summary: CodEval for evaluating programming assignments
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
@@ -69,6 +69,7 @@ Tags used in a spec file (\<course name>.codeval)
 | CF | Check Function | Will be followed by a function name and a list of files to check to ensure that the function is used by one of those files. |
 | CC | Check Container | Will be followed by a function name and a list of files to check to ensure that a container is used by one of those files. Primarily supports C++ containers such as std::vector |
 | CO | Check Object | Will be followed by a function name and a list of files to check to ensure that an object is used by one of those files. Primarily support C++ stream operations |
+| PRINT | Print Label | Prints a label/message to stdout. Cleaner alternative to `CMD echo "..."` for section labels. |
 | CMD/TCMD | Run Command | Will be followed by a command to run. The TCMD will cause the evaluation to fail if the command exits with an error. |
 | CMP | Compare | Will be followed by two files to compare. |
 | T/HT | Test Case | Will be followed by the command to run to test the submission. |

{assignment_codeval-0.0.24.dist-info → assignment_codeval-0.0.26.dist-info}/RECORD

@@ -2,18 +2,20 @@ assignment_codeval/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
 assignment_codeval/ai_benchmark.py,sha256=ZGWWrBeUcaArkuQqS9wY42gxP8g7J2nivJMedYq9RFs,18045
 assignment_codeval/canvas_utils.py,sha256=MNtGXc93tHlqudEZsb_6r-5QzV9tSqf3AVxIVGEmk-U,5989
 assignment_codeval/check_grading.py,sha256=VFbkAG_PrBi6iFeyqHs7mLKPi0Xjccg6WtLwY9xrRIk,5880
-assignment_codeval/cli.py,sha256=CimNl-0urFsjSK-76WFtBjYEWpFngoqgR9g7TaTSRzE,1270
+assignment_codeval/cli.py,sha256=Vl27zNI7kI0QEZoXTIE2aTrraEJWj9vgJiyVWou3vgU,1357
 assignment_codeval/commons.py,sha256=rr09qHacHoLmmU8S9sBss4_utb0DH8-IyL9Dsj0ycE0,1412
 assignment_codeval/convertMD2Html.py,sha256=4yu41U4Up1oGI5vDZGelmuogwFmEZrRS55M4ZHbRfaE,6243
 assignment_codeval/create_assignment.py,sha256=oZ-0LadVb6R-rXVg0Lz5itziaalIvFoxd71cIxgMfow,10815
-assignment_codeval/evaluate.py,sha256=Wf9KnMqqgxXrodme_me8SDfX-QCpPqZTDP-XqB7B9sA,26796
+assignment_codeval/evaluate.py,sha256=er21lA-eohP1BfHPYPWRw9m0a_IOeljMNtcidOYacJY,36253
+assignment_codeval/export_tests.py,sha256=IDAlGAp3zk-K5AFwLr85v4w9_WcYr6HXVF0joi4eYTg,5467
 assignment_codeval/file_utils.py,sha256=RDgAgmyIOLKvRlxbAr-zILz2A24zSr-r3--p-dxl4G8,2132
 assignment_codeval/github_connect.py,sha256=AP7mZPhzKKK8fxS-3JMfG6AQ-1wUutfsq-Bt-KG9WUs,4270
 assignment_codeval/install_assignment.py,sha256=HsXzTex4G-U3t4tb-HyQ2AobBDBPcKeR6oAUDFqBP-A,4599
 assignment_codeval/recent_comments.py,sha256=e8UQL2_9dsPlf-NYFrGSaM6h6XYnTf_MW2jWhtEsb38,9097
-assignment_codeval/submissions.py,sha256=OW6wotV8oKcW4g5E0kSfybRbOL62_CD4FmKyiFyCH3E,20966
-assignment_codeval-0.0.24.dist-info/METADATA,sha256=_41zKTIp2QSibs_mQJGnXoVtKpMRCMhv3Upfw8n-6fY,15469
-assignment_codeval-0.0.24.dist-info/WHEEL,sha256=YCfwYGOYMi5Jhw2fU4yNgwErybb2IX5PEwBKV4ZbdBo,91
-assignment_codeval-0.0.24.dist-info/entry_points.txt,sha256=Pk0eAhsXBpCsgkv6HYLSHTRf5x_zUf-Lg8KK9KsK2B8,66
-assignment_codeval-0.0.24.dist-info/top_level.txt,sha256=a_P8W2IWjAELxKlTV2t4rLwWVoTxElxUMvWgWjR_1n4,19
-assignment_codeval-0.0.24.dist-info/RECORD,,
+assignment_codeval/submissions.py,sha256=YbwKynZwWsmcyytwRi3Kuxq3WN-V-vvshO9JrNJoxAs,28333
+assignment_codeval/test_template.html,sha256=lgWsin5sYE_Pwjp3xqa7_UT-yZ4dWrVqAvNlSBBiNMI,2584
+assignment_codeval-0.0.26.dist-info/METADATA,sha256=CvKtHeb1kR3x4Sz0ojGIKui1v_Q1ejiColTIShfMOfg,15589
+assignment_codeval-0.0.26.dist-info/WHEEL,sha256=YCfwYGOYMi5Jhw2fU4yNgwErybb2IX5PEwBKV4ZbdBo,91
+assignment_codeval-0.0.26.dist-info/entry_points.txt,sha256=Pk0eAhsXBpCsgkv6HYLSHTRf5x_zUf-Lg8KK9KsK2B8,66
+assignment_codeval-0.0.26.dist-info/top_level.txt,sha256=a_P8W2IWjAELxKlTV2t4rLwWVoTxElxUMvWgWjR_1n4,19
+assignment_codeval-0.0.26.dist-info/RECORD,,