assignment-codeval 0.0.24__tar.gz → 0.0.26__tar.gz

{assignment_codeval-0.0.24 → assignment_codeval-0.0.26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assignment-codeval
-Version: 0.0.24
+Version: 0.0.26
 Summary: CodEval for evaluating programming assignments
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
@@ -69,6 +69,7 @@ Tags used in a spec file (\<course name>.codeval)
 | CF | Check Function | Will be followed by a function name and a list of files to check to ensure that the function is used by one of those files. |
 | CC | Check Container | Will be followed by a function name and a list of files to check to ensure that a container is used by one of those files. Primarily supports C++ containers such as std::vector |
 | CO | Check Object | Will be followed by a function name and a list of files to check to ensure that an object is used by one of those files. Primarily support C++ stream operations |
+| PRINT | Print Label | Prints a label/message to stdout. Cleaner alternative to `CMD echo "..."` for section labels. |
 | CMD/TCMD | Run Command | Will be followed by a command to run. The TCMD will cause the evaluation to fail if the command exits with an error. |
 | CMP | Compare | Will be followed by two files to compare. |
 | T/HT | Test Case | Will be followed by the command to run to test the submission. |

{assignment_codeval-0.0.24 → assignment_codeval-0.0.26}/README.md

@@ -49,6 +49,7 @@ Tags used in a spec file (\<course name>.codeval)
 | CF | Check Function | Will be followed by a function name and a list of files to check to ensure that the function is used by one of those files. |
 | CC | Check Container | Will be followed by a function name and a list of files to check to ensure that a container is used by one of those files. Primarily supports C++ containers such as std::vector |
 | CO | Check Object | Will be followed by a function name and a list of files to check to ensure that an object is used by one of those files. Primarily support C++ stream operations |
+| PRINT | Print Label | Prints a label/message to stdout. Cleaner alternative to `CMD echo "..."` for section labels. |
 | CMD/TCMD | Run Command | Will be followed by a command to run. The TCMD will cause the evaluation to fail if the command exits with an error. |
 | CMP | Compare | Will be followed by two files to compare. |
 | T/HT | Test Case | Will be followed by the command to run to test the submission. |

{assignment_codeval-0.0.24 → assignment_codeval-0.0.26}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "assignment-codeval"
-version = "0.0.24"
+version = "0.0.26"
 description = "CodEval for evaluating programming assignments"
 readme = "README.md"
 requires-python = ">=3.12"
@@ -27,3 +27,6 @@ test = ["pytest>=7.0"]
 
 [project.scripts]
 assignment-codeval = "assignment_codeval.cli:cli"
+
+[tool.setuptools.package-data]
+assignment_codeval = ["*.html"]

{assignment_codeval-0.0.24 → assignment_codeval-0.0.26}/src/assignment_codeval/cli.py

@@ -5,6 +5,7 @@ from assignment_codeval.check_grading import check_grading
 from assignment_codeval.commons import set_config
 from assignment_codeval.create_assignment import create_assignment
 from assignment_codeval.evaluate import run_evaluation
+from assignment_codeval.export_tests import export_tests
 from assignment_codeval.github_connect import github_setup_repo
 from assignment_codeval.install_assignment import install_assignment
 from assignment_codeval.recent_comments import recent_comments
@@ -26,6 +27,7 @@ cli.add_command(list_codeval_assignments)
 cli.add_command(install_assignment)
 cli.add_command(recent_comments)
 cli.add_command(check_grading)
+cli.add_command(export_tests)
 cli.add_command(get_benchmark_command())
 
 if __name__ == "__main__":

{assignment_codeval-0.0.24 → assignment_codeval-0.0.26}/src/assignment_codeval/evaluate.py

@@ -1,5 +1,6 @@
 #! /usr/bin/python3
 
+import ast
 import os
 import re
 import subprocess
@@ -27,6 +28,7 @@ def get_testing_path(filename: str) -> str:
 test_args = ""
 cmps = []
 timeout_val = 10
+output_length_limit = 4096
 expected_exit_code = -1
 test_case_count = 0
 test_case_hint = ""
@@ -80,52 +82,178 @@ def compile_code(compile_command):
         sys.exit(1)
 
 
-def check_function(args):
-    """Will be followed by a function name and a list of files to check to ensure that the function
-    is used by one of those files.
+###########################################################
+# Function detection helpers
+###########################################################
 
-    Arguments:
-        function_name: the function name to check files for usage of
-        *files: the files to check for the function name
 
-    Returns:
-        None
+def _detect_language(files):
+    """Return 'c_cpp', 'java', 'python', or 'unknown' based on file extensions."""
+    for f in files:
+        ext = os.path.splitext(f)[1].lower()
+        if ext in ('.c', '.cpp', '.cc', '.cxx', '.h', '.hpp'):
+            return 'c_cpp'
+        if ext == '.java':
+            return 'java'
+        if ext == '.py':
+            return 'python'
+    return 'unknown'
+
+
+def _find_compiled_artifact(source_file):
+    """Return the compiled artifact for a source file, or None if not found.
+
+    Checks (in order): same base name with .o, .exe, and no extension.
     """
-    check_test()
-    args = args.split()
-    function_name = args[0]
-    files = args[1:]
+    base = os.path.splitext(source_file)[0]
+    for candidate in (base + '.o', base + '.exe', base):
+        if os.path.exists(candidate):
+            return candidate
+    return None
 
-    # Match:
-    #  [1] start of line OR non-identifier character
-    #  [2] function name
-    #  [3] optional whitespace
-    #  [4] opening parenthesis (function call)
-    regex = rf'(^|[^[:alnum:]_]){function_name}[[:space:]]*\('
 
-    # Strip comments, then search
+def _function_used_in_c_cpp(function_name, files):
+    """Use objdump on compiled artifacts to detect C/C++ function usage.
+
+    Returns True if found, False if artifact exists but function absent,
+    None if no compiled artifact was found (caller should fall back).
+    """
+    found_artifact = False
+    for source_file in files:
+        artifact = _find_compiled_artifact(source_file)
+        if artifact is None:
+            continue
+        found_artifact = True
+        # objdump -t dumps the symbol table; pipe through c++filt to demangle C++ names.
+        # Mangled C++ names embed the original identifier, so grep -w finds exact matches.
+        cmd = f"objdump -t '{artifact}' | c++filt | grep -w '{function_name}'"
+        result = subprocess.run(
+            ["bash", "-c", cmd],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        if result.returncode == 0:
+            return True
+    if not found_artifact:
+        return None
+    return False
+
+
+def _function_used_in_java(function_name, files):
+    """Use javap on compiled .class files to detect Java method usage.
+
+    Returns True if found, False if class file exists but method absent,
+    None if no .class file was found (caller should fall back).
+    """
+    found_class = False
+    for source_file in files:
+        base = os.path.splitext(source_file)[0]
+        class_file = base + '.class'
+        if not os.path.exists(class_file):
+            continue
+        found_class = True
+        # javap -c disassembles bytecode; method invocations reference the method name.
+        result = subprocess.run(
+            ["javap", "-c", class_file],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        if function_name.encode() in result.stdout:
+            return True
+    if not found_class:
+        return None
+    return False
+
+
+def _function_used_in_python(function_name, files):
+    """Use the ast module to detect function calls in Python source files.
+
+    Returns True if a matching call is found, False otherwise.
+    Handles both direct calls (func()) and attribute calls (obj.func()).
+    """
+    for source_file in files:
+        try:
+            with open(source_file, 'r') as f:
+                source = f.read()
+            tree = ast.parse(source)
+            for node in ast.walk(tree):
+                if isinstance(node, ast.Call):
+                    func = node.func
+                    if isinstance(func, ast.Name) and func.id == function_name:
+                        return True
+                    if isinstance(func, ast.Attribute) and func.attr == function_name:
+                        return True
+        except (SyntaxError, OSError):
+            continue
+    return False
+
+
+def _function_used_regex(function_name, files):
+    """Legacy regex-based fallback used when no compiled artifact is available."""
+    regex = rf'(^|[^[:alnum:]_]){function_name}[[:space:]]*\('
     cmd = (
         "sed -E "
         "'"
-        "s://.*$::g; " # C/C++ single-line comments
-        "s:#.*$::g; " # Python single-line comments
+        "s://.*$::g; "
+        "s:#.*$::g; "
         ":a; /\\/\\*/{N; s:/\\*.*?\\*/::g; ba}"
         "' "
         + " ".join(files)
         + f" | grep -E '{regex}'"
     )
-
-    function_popen = subprocess.Popen(
+    result = subprocess.run(
         ["bash", "-c", cmd],
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
     )
+    return result.returncode == 0
 
-    function_popen.communicate()
-    if function_popen.returncode:
-        print(f"Not using {function_name} FAILED")
-    else:
+
+def _is_function_used(function_name, files):
+    """Detect whether function_name is used in the given files.
+
+    Dispatches to the appropriate tool based on file language:
+      - C/C++:  objdump on compiled artifact (falls back to regex if none found)
+      - Java:   javap on .class file (falls back to regex if none found)
+      - Python: ast module on source
+      - Other:  regex on source
+    """
+    lang = _detect_language(files)
+    if lang == 'python':
+        return _function_used_in_python(function_name, files)
+    if lang == 'java':
+        result = _function_used_in_java(function_name, files)
+        if result is None:
+            return _function_used_regex(function_name, files)
+        return result
+    if lang == 'c_cpp':
+        result = _function_used_in_c_cpp(function_name, files)
+        if result is None:
+            return _function_used_regex(function_name, files)
+        return result
+    return _function_used_regex(function_name, files)
+
+
+def check_function(args):
+    """Will be followed by a function name and a list of files to check to ensure that the function
+    is used by one of those files.
+
+    Arguments:
+        function_name: the function name to check files for usage of
+        *files: the files to check for the function name
+
+    Returns:
+        None
+    """
+    check_test()
+    args = args.split()
+    function_name = args[0]
+    files = args[1:]
+
+    if _is_function_used(function_name, files):
         print(f"Used {function_name} PASSED")
+    else:
+        print(f"Not using {function_name} FAILED")
 
 def check_object(args):
     """Will be followed by an object and a list of files to ensure that the function is
@@ -228,7 +356,7 @@ def check_not_function(args):
     is not used by any of those files.
 
     Arguments:
-        function_name: the funcion name to check files for usage of
+        function_name: the function name to check files for usage of
         *files: the files to check for the function name
 
     Returns:
@@ -239,18 +367,23 @@ def check_not_function(args):
     function_name = args[0]
     files = args[1:]
 
-    # Surpress output
-    function_popen = subprocess.Popen(
-        ["grep", f"[^[:alpha:]]{function_name}[[:space:]]*("] + files,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-    )
-
-    function_popen.communicate()
-    if function_popen.returncode:
-        print(f"used{function_name} PASSED")
+    if _is_function_used(function_name, files):
+        print(f"Used {function_name} FAILED")
     else:
-        print(f"not using {function_name} FAILED")
+        print(f"Not using {function_name} PASSED")
+
+
+def print_label(text):
+    """Print a label/message to stdout.
+
+    Arguments:
+        text: the text to print
+
+    Returns:
+        None
+    """
+    check_test()
+    print(text)
 
 
 def run_command(command):
@@ -511,6 +644,19 @@ def timeout(timeout_sec):
     timeout_val = float(timeout_sec)
 
 
+def output_length(length):
+    """Specifies the maximum number of bytes of diff output to render. Defaults to 4096.
+
+    Arguments:
+        length: maximum output length in bytes
+
+    Returns:
+        None
+    """
+    global output_length_limit
+    output_length_limit = int(length)
+
+
 def exit_code(test_case_exit_code):
     """Specifies the expected exit code for a test case. Defaults to zero.
 
@@ -578,6 +724,7 @@ tag_func_map = {
     "CO" : check_object,
     "CC": check_container,
     "NCF": check_not_function,
+    "PRINT": print_label,
     "CMD": run_command,
     "TCMD": run_command_noerror,
     "CMP": compare,
@@ -593,6 +740,7 @@ tag_func_map = {
     "EB": check_error_bare,
     "HINT": hint,
     "TO": timeout,
+    "OLEN": output_length,
     "X": exit_code,
     "SS": start_server,
 }
@@ -774,6 +922,93 @@ def parse_diff(diff_lines: list[str], testing_dir: str):
                     outfile.write(line)
 
 
+def _render_diff_output(raw_bytes: bytes) -> str:
+    """Render raw bytes for diff output, making unprintable characters visible.
+
+    Replaces the shell pipeline `cat -te | head -22` with pure Python.
+    """
+    result = []
+    i = 0
+    n = len(raw_bytes)
+
+    while i < n:
+        b = raw_bytes[i]
+
+        if b == 0x0A:  # newline
+            result.append("$\n")
+            i += 1
+        elif b == 0x20 or (0x21 <= b <= 0x7E):  # space or printable ASCII
+            result.append(chr(b))
+            i += 1
+        elif b <= 0x1F:  # control chars (except newline handled above)
+            result.append("^" + chr(b + 0x40))
+            i += 1
+        elif b == 0x7F:  # DEL
+            result.append("^?")
+            i += 1
+        elif b >= 0x80:
+            # Try to decode as UTF-8 multi-byte sequence
+            seq_len = 0
+            if (b & 0xE0) == 0xC0:
+                seq_len = 2
+            elif (b & 0xF0) == 0xE0:
+                seq_len = 3
+            elif (b & 0xF8) == 0xF0:
+                seq_len = 4
+
+            decoded_char = None
+            if seq_len >= 2 and i + seq_len <= n:
+                try:
+                    decoded_char = raw_bytes[i:i + seq_len].decode("utf-8")
+                except UnicodeDecodeError:
+                    pass
+
+            if decoded_char and len(decoded_char) == 1 and decoded_char.isprintable():
+                result.append(decoded_char)
+                i += seq_len
+            elif decoded_char and len(decoded_char) == 1:
+                # Valid UTF-8 but non-printable: render each byte as \xCC
+                for j in range(seq_len):
+                    result.append(f"\\x{raw_bytes[i + j]:02X}")
+                i += seq_len
+            else:
+                # Invalid UTF-8 byte
+                result.append(f"\\x{b:02X}")
+                i += 1
+        else:
+            result.append(chr(b))
+            i += 1
+
+    output = "".join(result)
+
+    # If output doesn't end with \n, append $ at the very end
+    if output and not output.endswith("\n"):
+        output += "$"
+
+    return output
+
+
+def _find_touched_files(pre_run_timestamp):
+    """Find files touched (modified or accessed) since pre_run_timestamp.
+
+    Recursively scans the current working directory, ignoring TESTING_DIR.
+    """
+    touched = []
+    for dirpath, dirnames, filenames in os.walk("."):
+        # Prune TESTING_DIR from traversal
+        if TESTING_DIR in dirnames:
+            dirnames.remove(TESTING_DIR)
+        for filename in filenames:
+            filepath = os.path.join(dirpath, filename)
+            try:
+                st = os.stat(filepath)
+                if st.st_mtime > pre_run_timestamp or st.st_atime > pre_run_timestamp:
+                    touched.append(filepath)
+            except OSError:
+                continue
+    return touched
+
+
 def check_test():
     global test_args
     if test_args == "":
@@ -782,6 +1017,9 @@ def check_test():
     print(f"Test case {test_case_count} of {test_case_total}")
     passed = True
 
+    # Record timestamp before running student code for file tracking
+    pre_run_timestamp = time.time()
+
     with open(get_testing_path("fileinput"), "rb") as fileinput, open(
         get_testing_path("youroutput"), "w"
     ) as youroutput, open(get_testing_path("yourerror"), "w") as yourerror:
@@ -797,27 +1035,28 @@ def check_test():
         print(f"Took more than {timeout_val} seconds to run. FAIL")
         passed = False
 
+    # Find files touched by student's program
+    touched_files = _find_touched_files(pre_run_timestamp)
+
     # Difflog handling
     with open(get_testing_path("difflog"), "w") as outfile:
         diff_popen = subprocess.Popen(
-            f"diff -U1 -a ./{TESTING_DIR}/youroutput ./{TESTING_DIR}/expectedoutput | cat -te | head -22",
-            shell=True,
-            stdout=outfile,
-            stderr=outfile,
-            text=True,
+            ["diff", "-U1", "-a",
+             f"./{TESTING_DIR}/youroutput", f"./{TESTING_DIR}/expectedoutput"],
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
         )
-        diff_popen.communicate()
+        raw_output, _ = diff_popen.communicate()
+        outfile.write(_render_diff_output(raw_output[:output_length_limit]))
 
     # Append to difflog second time around
     with open(get_testing_path("difflog"), "a") as outfile:
         diff_popen = subprocess.Popen(
-            f"diff -U1 -a ./{TESTING_DIR}/yourerror ./{TESTING_DIR}/expectederror | cat -te | head -22",
-            shell=True,
-            stdout=outfile,
-            stderr=outfile,
-            text=True,
+            ["diff", "-U1", "-a",
+             f"./{TESTING_DIR}/yourerror", f"./{TESTING_DIR}/expectederror"],
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
         )
-        diff_popen.communicate()
+        raw_output, _ = diff_popen.communicate()
+        outfile.write(_render_diff_output(raw_output[:output_length_limit]))
 
     # Now read all the lines to accumulate both diffs
     with open(get_testing_path("difflog"), "r") as infile:
@@ -872,6 +1111,33 @@ def check_test():
                     # Print entire file
                     print("".join(file_lines))
 
+            # Show diffs for files touched by student's program
+            if touched_files and cmps:
+                expected_map = {}
+                for file_pair in cmps:
+                    if len(file_pair) >= 2:
+                        expected_map[os.path.normpath(file_pair[0])] = file_pair[1]
+
+                for filepath in touched_files:
+                    normalized = os.path.normpath(filepath)
+                    if normalized in expected_map:
+                        expected_file = expected_map[normalized]
+                        if os.path.exists(expected_file):
+                            diff_proc = subprocess.Popen(
+                                ["diff", "-U1", "-a", filepath, expected_file],
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE,
+                                text=True,
+                            )
+                            diff_output, _ = diff_proc.communicate()
+                            if diff_output:
+                                print(f"    File output diff ({os.path.basename(filepath)}):")
+                                lines = diff_output.splitlines()
+                                for line in lines[:22]:
+                                    print(f"    {line}")
+                                if len(lines) > 22:
+                                    print(f"    ... ({len(lines) - 22} more lines)")
+
         cleanup()
 
         # Exit program after failed test case