assignment-codeval 0.0.24__tar.gz → 0.0.25__tar.gz

{assignment_codeval-0.0.24 → assignment_codeval-0.0.25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assignment-codeval
-Version: 0.0.24
+Version: 0.0.25
 Summary: CodEval for evaluating programming assignments
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
@@ -69,6 +69,7 @@ Tags used in a spec file (\<course name>.codeval)
 | CF | Check Function | Will be followed by a function name and a list of files to check to ensure that the function is used by one of those files. |
 | CC | Check Container | Will be followed by a function name and a list of files to check to ensure that a container is used by one of those files. Primarily supports C++ containers such as std::vector |
 | CO | Check Object | Will be followed by a function name and a list of files to check to ensure that an object is used by one of those files. Primarily support C++ stream operations |
+| PRINT | Print Label | Prints a label/message to stdout. Cleaner alternative to `CMD echo "..."` for section labels. |
 | CMD/TCMD | Run Command | Will be followed by a command to run. The TCMD will cause the evaluation to fail if the command exits with an error. |
 | CMP | Compare | Will be followed by two files to compare. |
 | T/HT | Test Case | Will be followed by the command to run to test the submission. |

{assignment_codeval-0.0.24 → assignment_codeval-0.0.25}/README.md

@@ -49,6 +49,7 @@ Tags used in a spec file (\<course name>.codeval)
 | CF | Check Function | Will be followed by a function name and a list of files to check to ensure that the function is used by one of those files. |
 | CC | Check Container | Will be followed by a function name and a list of files to check to ensure that a container is used by one of those files. Primarily supports C++ containers such as std::vector |
 | CO | Check Object | Will be followed by a function name and a list of files to check to ensure that an object is used by one of those files. Primarily support C++ stream operations |
+| PRINT | Print Label | Prints a label/message to stdout. Cleaner alternative to `CMD echo "..."` for section labels. |
 | CMD/TCMD | Run Command | Will be followed by a command to run. The TCMD will cause the evaluation to fail if the command exits with an error. |
 | CMP | Compare | Will be followed by two files to compare. |
 | T/HT | Test Case | Will be followed by the command to run to test the submission. |

{assignment_codeval-0.0.24 → assignment_codeval-0.0.25}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "assignment-codeval"
-version = "0.0.24"
+version = "0.0.25"
 description = "CodEval for evaluating programming assignments"
 readme = "README.md"
 requires-python = ">=3.12"
@@ -27,3 +27,6 @@ test = ["pytest>=7.0"]
 
 [project.scripts]
 assignment-codeval = "assignment_codeval.cli:cli"
+
+[tool.setuptools.package-data]
+assignment_codeval = ["*.html"]

{assignment_codeval-0.0.24 → assignment_codeval-0.0.25}/src/assignment_codeval/cli.py

@@ -5,6 +5,7 @@ from assignment_codeval.check_grading import check_grading
 from assignment_codeval.commons import set_config
 from assignment_codeval.create_assignment import create_assignment
 from assignment_codeval.evaluate import run_evaluation
+from assignment_codeval.export_tests import export_tests
 from assignment_codeval.github_connect import github_setup_repo
 from assignment_codeval.install_assignment import install_assignment
 from assignment_codeval.recent_comments import recent_comments
@@ -26,6 +27,7 @@ cli.add_command(list_codeval_assignments)
 cli.add_command(install_assignment)
 cli.add_command(recent_comments)
 cli.add_command(check_grading)
+cli.add_command(export_tests)
 cli.add_command(get_benchmark_command())
 
 if __name__ == "__main__":

{assignment_codeval-0.0.24 → assignment_codeval-0.0.25}/src/assignment_codeval/evaluate.py

@@ -1,5 +1,6 @@
 #! /usr/bin/python3
 
+import ast
 import os
 import re
 import subprocess
@@ -80,52 +81,178 @@ def compile_code(compile_command):
         sys.exit(1)
 
 
-def check_function(args):
-    """Will be followed by a function name and a list of files to check to ensure that the function
-    is used by one of those files.
+###########################################################
+# Function detection helpers
+###########################################################
 
-    Arguments:
-        function_name: the function name to check files for usage of
-        *files: the files to check for the function name
 
-    Returns:
-        None
+def _detect_language(files):
+    """Return 'c_cpp', 'java', 'python', or 'unknown' based on file extensions."""
+    for f in files:
+        ext = os.path.splitext(f)[1].lower()
+        if ext in ('.c', '.cpp', '.cc', '.cxx', '.h', '.hpp'):
+            return 'c_cpp'
+        if ext == '.java':
+            return 'java'
+        if ext == '.py':
+            return 'python'
+    return 'unknown'
+
+
+def _find_compiled_artifact(source_file):
+    """Return the compiled artifact for a source file, or None if not found.
+
+    Checks (in order): same base name with .o, .exe, and no extension.
     """
-    check_test()
-    args = args.split()
-    function_name = args[0]
-    files = args[1:]
+    base = os.path.splitext(source_file)[0]
+    for candidate in (base + '.o', base + '.exe', base):
+        if os.path.exists(candidate):
+            return candidate
+    return None
 
-    # Match:
-    #  [1] start of line OR non-identifier character
-    #  [2] function name
-    #  [3] optional whitespace
-    #  [4] opening parenthesis (function call)
-    regex = rf'(^|[^[:alnum:]_]){function_name}[[:space:]]*\('
 
-    # Strip comments, then search
+def _function_used_in_c_cpp(function_name, files):
+    """Use objdump on compiled artifacts to detect C/C++ function usage.
+
+    Returns True if found, False if artifact exists but function absent,
+    None if no compiled artifact was found (caller should fall back).
+    """
+    found_artifact = False
+    for source_file in files:
+        artifact = _find_compiled_artifact(source_file)
+        if artifact is None:
+            continue
+        found_artifact = True
+        # objdump -t dumps the symbol table; pipe through c++filt to demangle C++ names.
+        # Mangled C++ names embed the original identifier, so grep -w finds exact matches.
+        cmd = f"objdump -t '{artifact}' | c++filt | grep -w '{function_name}'"
+        result = subprocess.run(
+            ["bash", "-c", cmd],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        if result.returncode == 0:
+            return True
+    if not found_artifact:
+        return None
+    return False
+
+
+def _function_used_in_java(function_name, files):
+    """Use javap on compiled .class files to detect Java method usage.
+
+    Returns True if found, False if class file exists but method absent,
+    None if no .class file was found (caller should fall back).
+    """
+    found_class = False
+    for source_file in files:
+        base = os.path.splitext(source_file)[0]
+        class_file = base + '.class'
+        if not os.path.exists(class_file):
+            continue
+        found_class = True
+        # javap -c disassembles bytecode; method invocations reference the method name.
+        result = subprocess.run(
+            ["javap", "-c", class_file],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        if function_name.encode() in result.stdout:
+            return True
+    if not found_class:
+        return None
+    return False
+
+
+def _function_used_in_python(function_name, files):
+    """Use the ast module to detect function calls in Python source files.
+
+    Returns True if a matching call is found, False otherwise.
+    Handles both direct calls (func()) and attribute calls (obj.func()).
+    """
+    for source_file in files:
+        try:
+            with open(source_file, 'r') as f:
+                source = f.read()
+            tree = ast.parse(source)
+            for node in ast.walk(tree):
+                if isinstance(node, ast.Call):
+                    func = node.func
+                    if isinstance(func, ast.Name) and func.id == function_name:
+                        return True
+                    if isinstance(func, ast.Attribute) and func.attr == function_name:
+                        return True
+        except (SyntaxError, OSError):
+            continue
+    return False
+
+
+def _function_used_regex(function_name, files):
+    """Legacy regex-based fallback used when no compiled artifact is available."""
+    regex = rf'(^|[^[:alnum:]_]){function_name}[[:space:]]*\('
     cmd = (
         "sed -E "
         "'"
-        "s://.*$::g; " # C/C++ single-line comments
-        "s:#.*$::g; " # Python single-line comments
+        "s://.*$::g; "
+        "s:#.*$::g; "
         ":a; /\\/\\*/{N; s:/\\*.*?\\*/::g; ba}"
         "' "
         + " ".join(files)
         + f" | grep -E '{regex}'"
     )
-
-    function_popen = subprocess.Popen(
+    result = subprocess.run(
         ["bash", "-c", cmd],
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
     )
+    return result.returncode == 0
 
-    function_popen.communicate()
-    if function_popen.returncode:
-        print(f"Not using {function_name} FAILED")
-    else:
+
+def _is_function_used(function_name, files):
+    """Detect whether function_name is used in the given files.
+
+    Dispatches to the appropriate tool based on file language:
+      - C/C++:  objdump on compiled artifact (falls back to regex if none found)
+      - Java:   javap on .class file (falls back to regex if none found)
+      - Python: ast module on source
+      - Other:  regex on source
+    """
+    lang = _detect_language(files)
+    if lang == 'python':
+        return _function_used_in_python(function_name, files)
+    if lang == 'java':
+        result = _function_used_in_java(function_name, files)
+        if result is None:
+            return _function_used_regex(function_name, files)
+        return result
+    if lang == 'c_cpp':
+        result = _function_used_in_c_cpp(function_name, files)
+        if result is None:
+            return _function_used_regex(function_name, files)
+        return result
+    return _function_used_regex(function_name, files)
+
+
+def check_function(args):
+    """Will be followed by a function name and a list of files to check to ensure that the function
+    is used by one of those files.
+
+    Arguments:
+        function_name: the function name to check files for usage of
+        *files: the files to check for the function name
+
+    Returns:
+        None
+    """
+    check_test()
+    args = args.split()
+    function_name = args[0]
+    files = args[1:]
+
+    if _is_function_used(function_name, files):
         print(f"Used {function_name} PASSED")
+    else:
+        print(f"Not using {function_name} FAILED")
 
 def check_object(args):
     """Will be followed by an object and a list of files to ensure that the function is
@@ -228,7 +355,7 @@ def check_not_function(args):
     is not used by any of those files.
 
     Arguments:
-        function_name: the funcion name to check files for usage of
+        function_name: the function name to check files for usage of
         *files: the files to check for the function name
 
     Returns:
@@ -239,18 +366,23 @@ def check_not_function(args):
     function_name = args[0]
     files = args[1:]
 
-    # Surpress output
-    function_popen = subprocess.Popen(
-        ["grep", f"[^[:alpha:]]{function_name}[[:space:]]*("] + files,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-    )
-
-    function_popen.communicate()
-    if function_popen.returncode:
-        print(f"used{function_name} PASSED")
+    if _is_function_used(function_name, files):
+        print(f"Used {function_name} FAILED")
     else:
-        print(f"not using {function_name} FAILED")
+        print(f"Not using {function_name} PASSED")
+
+
+def print_label(text):
+    """Print a label/message to stdout.
+
+    Arguments:
+        text: the text to print
+
+    Returns:
+        None
+    """
+    check_test()
+    print(text)
 
 
 def run_command(command):
@@ -578,6 +710,7 @@ tag_func_map = {
     "CO" : check_object,
     "CC": check_container,
     "NCF": check_not_function,
+    "PRINT": print_label,
     "CMD": run_command,
     "TCMD": run_command_noerror,
     "CMP": compare,
@@ -774,6 +907,27 @@ def parse_diff(diff_lines: list[str], testing_dir: str):
                     outfile.write(line)
 
 
+def _find_touched_files(pre_run_timestamp):
+    """Find files touched (modified or accessed) since pre_run_timestamp.
+
+    Recursively scans the current working directory, ignoring TESTING_DIR.
+    """
+    touched = []
+    for dirpath, dirnames, filenames in os.walk("."):
+        # Prune TESTING_DIR from traversal
+        if TESTING_DIR in dirnames:
+            dirnames.remove(TESTING_DIR)
+        for filename in filenames:
+            filepath = os.path.join(dirpath, filename)
+            try:
+                st = os.stat(filepath)
+                if st.st_mtime > pre_run_timestamp or st.st_atime > pre_run_timestamp:
+                    touched.append(filepath)
+            except OSError:
+                continue
+    return touched
+
+
 def check_test():
     global test_args
     if test_args == "":
@@ -782,6 +936,9 @@ def check_test():
     print(f"Test case {test_case_count} of {test_case_total}")
     passed = True
 
+    # Record timestamp before running student code for file tracking
+    pre_run_timestamp = time.time()
+
     with open(get_testing_path("fileinput"), "rb") as fileinput, open(
         get_testing_path("youroutput"), "w"
     ) as youroutput, open(get_testing_path("yourerror"), "w") as yourerror:
@@ -797,6 +954,9 @@ def check_test():
         print(f"Took more than {timeout_val} seconds to run. FAIL")
         passed = False
 
+    # Find files touched by student's program
+    touched_files = _find_touched_files(pre_run_timestamp)
+
     # Difflog handling
     with open(get_testing_path("difflog"), "w") as outfile:
         diff_popen = subprocess.Popen(
@@ -872,6 +1032,33 @@ def check_test():
                     # Print entire file
                     print("".join(file_lines))
 
+            # Show diffs for files touched by student's program
+            if touched_files and cmps:
+                expected_map = {}
+                for file_pair in cmps:
+                    if len(file_pair) >= 2:
+                        expected_map[os.path.normpath(file_pair[0])] = file_pair[1]
+
+                for filepath in touched_files:
+                    normalized = os.path.normpath(filepath)
+                    if normalized in expected_map:
+                        expected_file = expected_map[normalized]
+                        if os.path.exists(expected_file):
+                            diff_proc = subprocess.Popen(
+                                ["diff", "-U1", "-a", filepath, expected_file],
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE,
+                                text=True,
+                            )
+                            diff_output, _ = diff_proc.communicate()
+                            if diff_output:
+                                print(f"    File output diff ({os.path.basename(filepath)}):")
+                                lines = diff_output.splitlines()
+                                for line in lines[:22]:
+                                    print(f"    {line}")
+                                if len(lines) > 22:
+                                    print(f"    ... ({len(lines) - 22} more lines)")
+
         cleanup()
 
         # Exit program after failed test case

assignment_codeval-0.0.25/src/assignment_codeval/export_tests.py

@@ -0,0 +1,187 @@
+import os
+import re
+import zipfile
+
+import click
+
+
+def _read_referenced_file(filename, zip_archives):
+    """Read a file referenced by IF/OF/EF from zip archives.
+
+    Args:
+        filename: the filename from the tag
+        zip_archives: list of opened ZipFile objects
+
+    Returns:
+        file contents as string
+    """
+    for zf in zip_archives:
+        try:
+            return zf.read(filename).decode("utf-8")
+        except KeyError:
+            continue
+
+    raise FileNotFoundError(
+        f"Referenced file '{filename}' not found in zip archives"
+    )
+
+
+def parse_codeval_tests(codeval_file):
+    """Parse a codeval file and extract test cases.
+
+    Returns a list of dicts with keys:
+        num, command, input, output, error, exit_code, hidden
+    """
+    codeval_dir = os.path.dirname(os.path.abspath(codeval_file))
+
+    with open(codeval_file, "r") as f:
+        lines = f.readlines()
+
+    # First pass: collect Z tags to open zip archives
+    tag_pattern = r"([A-Z_]+) (.*)"
+    zip_paths = []
+    in_crt_hw = False
+    for line in lines:
+        stripped = line.strip()
+        if stripped.startswith("CRT_HW"):
+            in_crt_hw = not in_crt_hw
+            continue
+        if in_crt_hw:
+            continue
+        m = re.match(tag_pattern, line)
+        if m and m.group(1) == "Z":
+            zip_paths.append(os.path.join(codeval_dir, m.group(2).strip()))
+
+    # Open zip archives for the duration of parsing
+    zip_archives = []
+    try:
+        for zp in zip_paths:
+            if os.path.exists(zp):
+                zip_archives.append(zipfile.ZipFile(zp, "r"))
+
+        tests = _parse_codeval_lines(lines, zip_archives)
+    finally:
+        for zf in zip_archives:
+            zf.close()
+
+    return tests
+
+
+def _parse_codeval_lines(lines, zip_archives):
+    """Parse codeval lines into test case dicts."""
+    tag_pattern = r"([A-Z_]+) (.*)"
+    in_crt_hw_block = False
+    tests = []
+    current = None
+    test_num = 0
+
+    def finalize():
+        if current is not None:
+            tests.append(current)
+
+    for line in lines:
+        stripped = line.strip()
+        if not stripped or stripped.startswith("#"):
+            continue
+
+        if stripped.startswith("CRT_HW"):
+            in_crt_hw_block = not in_crt_hw_block
+            continue
+
+        if in_crt_hw_block:
+            continue
+
+        tag_match = re.match(tag_pattern, line)
+        if not tag_match:
+            continue
+
+        tag = tag_match.group(1)
+        args = tag_match.group(2)
+
+        if tag in ("T", "HT"):
+            finalize()
+            test_num += 1
+            current = {
+                "num": test_num,
+                "command": args,
+                "input": "",
+                "output": "",
+                "error": "",
+                "exit_code": -1,
+                "hidden": tag == "HT",
+            }
+        elif tag == "TCMD":
+            finalize()
+            test_num += 1
+            current = {
+                "num": test_num,
+                "command": args,
+                "input": "",
+                "output": "",
+                "error": "",
+                "exit_code": -1,
+                "hidden": False,
+            }
+        elif current is not None:
+            if tag == "I":
+                current["input"] += args + "\n"
+            elif tag == "IB":
+                current["input"] += args
+            elif tag == "IF":
+                current["input"] += _read_referenced_file(
+                    args.strip(), zip_archives
+                )
+            elif tag == "O":
+                current["output"] += args + "\n"
+            elif tag == "OB":
+                current["output"] += args
+            elif tag == "OF":
+                current["output"] += _read_referenced_file(
+                    args.strip(), zip_archives
+                )
+            elif tag == "E":
+                current["error"] += args + "\n"
+            elif tag == "EB":
+                current["error"] += args
+            elif tag == "X":
+                current["exit_code"] = int(args.strip())
+
+    finalize()
+    return tests
+
+
+@click.command()
+@click.argument("codeval_file", type=click.Path(exists=True))
+@click.option("--include-hidden", is_flag=True, help="Include hidden test cases")
+@click.option("--output", "-o", default=None, help="Output zip file path")
+def export_tests(codeval_file, include_hidden, output):
+    """Export test cases from a codeval file into a zip archive.
+
+    Creates a zip with in.X, out.X, err.X files for each test case
+    and a TESTS.md summary.
+    """
+    tests = parse_codeval_tests(codeval_file)
+
+    if output is None:
+        basename = os.path.splitext(os.path.basename(codeval_file))[0]
+        output = f"{basename}_tests.zip"
+
+    filtered = [t for t in tests if include_hidden or not t["hidden"]]
+
+    with zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as zf:
+        tests_md = ""
+        for test in filtered:
+            num = test["num"]
+            zf.writestr(f"in.{num}", test["input"])
+            zf.writestr(f"out.{num}", test["output"])
+            zf.writestr(f"err.{num}", test["error"])
+
+            tests_md += f"# Test {num}\n"
+            tests_md += f"command: `{test['command']}`\n"
+            if test["exit_code"] != -1:
+                tests_md += f"expected exit code: {test['exit_code']}\n"
+            tests_md += "\n"
+
+        zf.writestr("TESTS.md", tests_md)
+
+    click.echo(f"Exported {len(filtered)} test(s) to {output}")