assignment-codeval 0.0.23__tar.gz → 0.0.25__tar.gz

{assignment_codeval-0.0.23 → assignment_codeval-0.0.25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assignment-codeval
-Version: 0.0.23
+Version: 0.0.25
 Summary: CodEval for evaluating programming assignments
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
@@ -69,6 +69,7 @@ Tags used in a spec file (\<course name>.codeval)
 | CF | Check Function | Will be followed by a function name and a list of files to check to ensure that the function is used by one of those files. |
 | CC | Check Container | Will be followed by a function name and a list of files to check to ensure that a container is used by one of those files. Primarily supports C++ containers such as std::vector |
 | CO | Check Object | Will be followed by a function name and a list of files to check to ensure that an object is used by one of those files. Primarily support C++ stream operations |
+| PRINT | Print Label | Prints a label/message to stdout. Cleaner alternative to `CMD echo "..."` for section labels. |
 | CMD/TCMD | Run Command | Will be followed by a command to run. The TCMD will cause the evaluation to fail if the command exits with an error. |
 | CMP | Compare | Will be followed by two files to compare. |
 | T/HT | Test Case | Will be followed by the command to run to test the submission. |

{assignment_codeval-0.0.23 → assignment_codeval-0.0.25}/README.md

@@ -49,6 +49,7 @@ Tags used in a spec file (\<course name>.codeval)
 | CF | Check Function | Will be followed by a function name and a list of files to check to ensure that the function is used by one of those files. |
 | CC | Check Container | Will be followed by a function name and a list of files to check to ensure that a container is used by one of those files. Primarily supports C++ containers such as std::vector |
 | CO | Check Object | Will be followed by a function name and a list of files to check to ensure that an object is used by one of those files. Primarily support C++ stream operations |
+| PRINT | Print Label | Prints a label/message to stdout. Cleaner alternative to `CMD echo "..."` for section labels. |
 | CMD/TCMD | Run Command | Will be followed by a command to run. The TCMD will cause the evaluation to fail if the command exits with an error. |
 | CMP | Compare | Will be followed by two files to compare. |
 | T/HT | Test Case | Will be followed by the command to run to test the submission. |

{assignment_codeval-0.0.23 → assignment_codeval-0.0.25}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "assignment-codeval"
-version = "0.0.23"
+version = "0.0.25"
 description = "CodEval for evaluating programming assignments"
 readme = "README.md"
 requires-python = ">=3.12"
@@ -27,3 +27,6 @@ test = ["pytest>=7.0"]
 
 [project.scripts]
 assignment-codeval = "assignment_codeval.cli:cli"
+
+[tool.setuptools.package-data]
+assignment_codeval = ["*.html"]

{assignment_codeval-0.0.23 → assignment_codeval-0.0.25}/src/assignment_codeval/canvas_utils.py

@@ -5,6 +5,7 @@ from functools import cache
 from typing import NamedTuple
 
 import click
+import requests
 from canvasapi import Canvas
 from canvasapi.current_user import CurrentUser
 
@@ -107,3 +108,77 @@ def get_assignment(course, assignment_name):
             sys.exit(2)
     assignment = assignments[0]
     return assignment
+
+
+SUBMISSIONS_QUERY = """
+query SubmissionsQuery($assignmentId: ID!, $cursor: String) {
+  assignment(id: $assignmentId) {
+    submissionsConnection(after: $cursor) {
+      pageInfo {
+        hasNextPage
+        endCursor
+      }
+      nodes {
+        _id
+        submittedAt
+        user {
+          name
+        }
+        commentsConnection(filter: {allComments: true}, sortOrder: desc) {
+          nodes {
+            comment
+            createdAt
+          }
+        }
+      }
+    }
+  }
+}
+"""
+
+
+def get_canvas_credentials():
+    """Read raw url and token from codeval.ini config file."""
+    parser = ConfigParser()
+    config_file = click.get_app_dir("codeval.ini")
+    parser.read(config_file)
+    if 'SERVER' not in parser:
+        error(f"did not find [SERVER] section in {config_file}.")
+        sys.exit(1)
+    for key in ['url', 'token']:
+        if key not in parser['SERVER']:
+            error(f"did not find {key} in [SERVER] in {config_file}.")
+            sys.exit(1)
+    return parser['SERVER']['url'], parser['SERVER']['token']
+
+
+def graphql_request(base_url, token, query, variables):
+    """POST a GraphQL query to Canvas and return the data."""
+    url = f"{base_url}/api/graphql"
+    headers = {"Authorization": f"Bearer {token}"}
+    payload = {"query": query, "variables": variables}
+    resp = requests.post(url, json=payload, headers=headers)
+    resp.raise_for_status()
+    result = resp.json()
+    if "errors" in result:
+        error(f"GraphQL errors: {result['errors']}")
+        sys.exit(1)
+    return result["data"]
+
+
+def fetch_all_submissions(base_url, token, assignment_id):
+    """Fetch all submissions for an assignment using relay pagination."""
+    all_nodes = []
+    cursor = None
+    while True:
+        variables = {"assignmentId": str(assignment_id)}
+        if cursor:
+            variables["cursor"] = cursor
+        data = graphql_request(base_url, token, SUBMISSIONS_QUERY, variables)
+        conn = data["assignment"]["submissionsConnection"]
+        all_nodes.extend(conn["nodes"])
+        if conn["pageInfo"]["hasNextPage"]:
+            cursor = conn["pageInfo"]["endCursor"]
+        else:
+            break
+    return all_nodes

{assignment_codeval-0.0.23 → assignment_codeval-0.0.25}/src/assignment_codeval/check_grading.py

@@ -1,14 +1,15 @@
 """Check which submissions are missing a codeval comment newer than the submission."""
 
 import sys
-from configparser import ConfigParser
 from datetime import datetime, timedelta, timezone
 from math import floor
 
 import click
-import requests
 
-from assignment_codeval.canvas_utils import connect_to_canvas, get_course
+from assignment_codeval.canvas_utils import (
+    connect_to_canvas, get_course,
+    get_canvas_credentials, graphql_request, fetch_all_submissions,
+)
 from assignment_codeval.commons import debug, error
 
 ASSIGNMENTS_QUERY = """
@@ -29,79 +30,6 @@ query AssignmentsQuery($courseId: ID!) {
 }
 """
 
-SUBMISSIONS_QUERY = """
-query SubmissionsQuery($assignmentId: ID!, $cursor: String) {
-  assignment(id: $assignmentId) {
-    submissionsConnection(after: $cursor) {
-      pageInfo {
-        hasNextPage
-        endCursor
-      }
-      nodes {
-        _id
-        submittedAt
-        user {
-          name
-        }
-        commentsConnection(filter: {allComments: true}) {
-          nodes {
-            comment
-            createdAt
-          }
-        }
-      }
-    }
-  }
-}
-"""
-
-
-def _get_canvas_credentials():
-    """Read raw url and token from codeval.ini config file."""
-    parser = ConfigParser()
-    config_file = click.get_app_dir("codeval.ini")
-    parser.read(config_file)
-    if 'SERVER' not in parser:
-        error(f"did not find [SERVER] section in {config_file}.")
-        sys.exit(1)
-    for key in ['url', 'token']:
-        if key not in parser['SERVER']:
-            error(f"did not find {key} in [SERVER] in {config_file}.")
-            sys.exit(1)
-    return parser['SERVER']['url'], parser['SERVER']['token']
-
-
-def _graphql_request(base_url, token, query, variables):
-    """POST a GraphQL query to Canvas and return the data."""
-    url = f"{base_url}/api/graphql"
-    headers = {"Authorization": f"Bearer {token}"}
-    payload = {"query": query, "variables": variables}
-    resp = requests.post(url, json=payload, headers=headers)
-    resp.raise_for_status()
-    result = resp.json()
-    if "errors" in result:
-        error(f"GraphQL errors: {result['errors']}")
-        sys.exit(1)
-    return result["data"]
-
-
-def _fetch_all_submissions(base_url, token, assignment_id):
-    """Fetch all submissions for an assignment using relay pagination."""
-    all_nodes = []
-    cursor = None
-    while True:
-        variables = {"assignmentId": str(assignment_id)}
-        if cursor:
-            variables["cursor"] = cursor
-        data = _graphql_request(base_url, token, SUBMISSIONS_QUERY, variables)
-        conn = data["assignment"]["submissionsConnection"]
-        all_nodes.extend(conn["nodes"])
-        if conn["pageInfo"]["hasNextPage"]:
-            cursor = conn["pageInfo"]["endCursor"]
-        else:
-            break
-    return all_nodes
-
 
 def _format_elapsed(submitted_at_str):
     """Format elapsed time since submission as a human-readable string."""
@@ -160,10 +88,10 @@ def check_grading(course_name, assignment_group, codeval_prefix, verbose, warn,
     """
     (canvas, user) = connect_to_canvas()
     course = get_course(canvas, course_name)
-    base_url, token = _get_canvas_credentials()
+    base_url, token = get_canvas_credentials()
 
     debug(f"fetching assignment groups for course {course.name} (id={course.id})")
-    data = _graphql_request(base_url, token, ASSIGNMENTS_QUERY, {"courseId": str(course.id)})
+    data = graphql_request(base_url, token, ASSIGNMENTS_QUERY, {"courseId": str(course.id)})
 
     groups = data["course"]["assignmentGroupsConnection"]["nodes"]
     target_group = None
@@ -192,7 +120,7 @@ def check_grading(course_name, assignment_group, codeval_prefix, verbose, warn,
         assignment_name = assignment["name"]
         debug(f"checking assignment: {assignment_name}")
 
-        submissions = _fetch_all_submissions(base_url, token, assignment_id)
+        submissions = fetch_all_submissions(base_url, token, assignment_id)
 
         missing = []
         warned = []

{assignment_codeval-0.0.23 → assignment_codeval-0.0.25}/src/assignment_codeval/cli.py

@@ -5,6 +5,7 @@ from assignment_codeval.check_grading import check_grading
 from assignment_codeval.commons import set_config
 from assignment_codeval.create_assignment import create_assignment
 from assignment_codeval.evaluate import run_evaluation
+from assignment_codeval.export_tests import export_tests
 from assignment_codeval.github_connect import github_setup_repo
 from assignment_codeval.install_assignment import install_assignment
 from assignment_codeval.recent_comments import recent_comments
@@ -26,6 +27,7 @@ cli.add_command(list_codeval_assignments)
 cli.add_command(install_assignment)
 cli.add_command(recent_comments)
 cli.add_command(check_grading)
+cli.add_command(export_tests)
 cli.add_command(get_benchmark_command())
 
 if __name__ == "__main__":

{assignment_codeval-0.0.23 → assignment_codeval-0.0.25}/src/assignment_codeval/evaluate.py

@@ -1,5 +1,6 @@
 #! /usr/bin/python3
 
+import ast
 import os
 import re
 import subprocess
@@ -80,52 +81,178 @@ def compile_code(compile_command):
         sys.exit(1)
 
 
-def check_function(args):
-    """Will be followed by a function name and a list of files to check to ensure that the function
-    is used by one of those files.
+###########################################################
+# Function detection helpers
+###########################################################
 
-    Arguments:
-        function_name: the function name to check files for usage of
-        *files: the files to check for the function name
 
-    Returns:
-        None
+def _detect_language(files):
+    """Return 'c_cpp', 'java', 'python', or 'unknown' based on file extensions."""
+    for f in files:
+        ext = os.path.splitext(f)[1].lower()
+        if ext in ('.c', '.cpp', '.cc', '.cxx', '.h', '.hpp'):
+            return 'c_cpp'
+        if ext == '.java':
+            return 'java'
+        if ext == '.py':
+            return 'python'
+    return 'unknown'
+
+
+def _find_compiled_artifact(source_file):
+    """Return the compiled artifact for a source file, or None if not found.
+
+    Checks (in order): same base name with .o, .exe, and no extension.
     """
-    check_test()
-    args = args.split()
-    function_name = args[0]
-    files = args[1:]
+    base = os.path.splitext(source_file)[0]
+    for candidate in (base + '.o', base + '.exe', base):
+        if os.path.exists(candidate):
+            return candidate
+    return None
 
-    # Match:
-    #  [1] start of line OR non-identifier character
-    #  [2] function name
-    #  [3] optional whitespace
-    #  [4] opening parenthesis (function call)
-    regex = rf'(^|[^[:alnum:]_]){function_name}[[:space:]]*\('
 
-    # Strip comments, then search
+def _function_used_in_c_cpp(function_name, files):
+    """Use objdump on compiled artifacts to detect C/C++ function usage.
+
+    Returns True if found, False if artifact exists but function absent,
+    None if no compiled artifact was found (caller should fall back).
+    """
+    found_artifact = False
+    for source_file in files:
+        artifact = _find_compiled_artifact(source_file)
+        if artifact is None:
+            continue
+        found_artifact = True
+        # objdump -t dumps the symbol table; pipe through c++filt to demangle C++ names.
+        # Mangled C++ names embed the original identifier, so grep -w finds exact matches.
+        cmd = f"objdump -t '{artifact}' | c++filt | grep -w '{function_name}'"
+        result = subprocess.run(
+            ["bash", "-c", cmd],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        if result.returncode == 0:
+            return True
+    if not found_artifact:
+        return None
+    return False
+
+
+def _function_used_in_java(function_name, files):
+    """Use javap on compiled .class files to detect Java method usage.
+
+    Returns True if found, False if class file exists but method absent,
+    None if no .class file was found (caller should fall back).
+    """
+    found_class = False
+    for source_file in files:
+        base = os.path.splitext(source_file)[0]
+        class_file = base + '.class'
+        if not os.path.exists(class_file):
+            continue
+        found_class = True
+        # javap -c disassembles bytecode; method invocations reference the method name.
+        result = subprocess.run(
+            ["javap", "-c", class_file],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        if function_name.encode() in result.stdout:
+            return True
+    if not found_class:
+        return None
+    return False
+
+
+def _function_used_in_python(function_name, files):
+    """Use the ast module to detect function calls in Python source files.
+
+    Returns True if a matching call is found, False otherwise.
+    Handles both direct calls (func()) and attribute calls (obj.func()).
+    """
+    for source_file in files:
+        try:
+            with open(source_file, 'r') as f:
+                source = f.read()
+            tree = ast.parse(source)
+            for node in ast.walk(tree):
+                if isinstance(node, ast.Call):
+                    func = node.func
+                    if isinstance(func, ast.Name) and func.id == function_name:
+                        return True
+                    if isinstance(func, ast.Attribute) and func.attr == function_name:
+                        return True
+        except (SyntaxError, OSError):
+            continue
+    return False
+
+
+def _function_used_regex(function_name, files):
+    """Legacy regex-based fallback used when no compiled artifact is available."""
+    regex = rf'(^|[^[:alnum:]_]){function_name}[[:space:]]*\('
     cmd = (
         "sed -E "
         "'"
-        "s://.*$::g; " # C/C++ single-line comments
-        "s:#.*$::g; " # Python single-line comments
+        "s://.*$::g; "
+        "s:#.*$::g; "
         ":a; /\\/\\*/{N; s:/\\*.*?\\*/::g; ba}"
         "' "
         + " ".join(files)
         + f" | grep -E '{regex}'"
     )
-
-    function_popen = subprocess.Popen(
+    result = subprocess.run(
         ["bash", "-c", cmd],
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
     )
+    return result.returncode == 0
 
-    function_popen.communicate()
-    if function_popen.returncode:
-        print(f"Not using {function_name} FAILED")
-    else:
+
+def _is_function_used(function_name, files):
+    """Detect whether function_name is used in the given files.
+
+    Dispatches to the appropriate tool based on file language:
+      - C/C++:  objdump on compiled artifact (falls back to regex if none found)
+      - Java:   javap on .class file (falls back to regex if none found)
+      - Python: ast module on source
+      - Other:  regex on source
+    """
+    lang = _detect_language(files)
+    if lang == 'python':
+        return _function_used_in_python(function_name, files)
+    if lang == 'java':
+        result = _function_used_in_java(function_name, files)
+        if result is None:
+            return _function_used_regex(function_name, files)
+        return result
+    if lang == 'c_cpp':
+        result = _function_used_in_c_cpp(function_name, files)
+        if result is None:
+            return _function_used_regex(function_name, files)
+        return result
+    return _function_used_regex(function_name, files)
+
+
+def check_function(args):
+    """Will be followed by a function name and a list of files to check to ensure that the function
+    is used by one of those files.
+
+    Arguments:
+        function_name: the function name to check files for usage of
+        *files: the files to check for the function name
+
+    Returns:
+        None
+    """
+    check_test()
+    args = args.split()
+    function_name = args[0]
+    files = args[1:]
+
+    if _is_function_used(function_name, files):
         print(f"Used {function_name} PASSED")
+    else:
+        print(f"Not using {function_name} FAILED")
 
 def check_object(args):
     """Will be followed by an object and a list of files to ensure that the function is
@@ -228,7 +355,7 @@ def check_not_function(args):
     is not used by any of those files.
 
     Arguments:
-        function_name: the funcion name to check files for usage of
+        function_name: the function name to check files for usage of
         *files: the files to check for the function name
 
     Returns:
@@ -239,18 +366,23 @@ def check_not_function(args):
     function_name = args[0]
     files = args[1:]
 
-    # Surpress output
-    function_popen = subprocess.Popen(
-        ["grep", f"[^[:alpha:]]{function_name}[[:space:]]*("] + files,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-    )
-
-    function_popen.communicate()
-    if function_popen.returncode:
-        print(f"used{function_name} PASSED")
+    if _is_function_used(function_name, files):
+        print(f"Used {function_name} FAILED")
     else:
-        print(f"not using {function_name} FAILED")
+        print(f"Not using {function_name} PASSED")
+
+
+def print_label(text):
+    """Print a label/message to stdout.
+
+    Arguments:
+        text: the text to print
+
+    Returns:
+        None
+    """
+    check_test()
+    print(text)
 
 
 def run_command(command):
@@ -578,6 +710,7 @@ tag_func_map = {
     "CO" : check_object,
     "CC": check_container,
     "NCF": check_not_function,
+    "PRINT": print_label,
     "CMD": run_command,
     "TCMD": run_command_noerror,
     "CMP": compare,
@@ -774,6 +907,27 @@ def parse_diff(diff_lines: list[str], testing_dir: str):
                     outfile.write(line)
 
 
+def _find_touched_files(pre_run_timestamp):
+    """Find files touched (modified or accessed) since pre_run_timestamp.
+
+    Recursively scans the current working directory, ignoring TESTING_DIR.
+    """
+    touched = []
+    for dirpath, dirnames, filenames in os.walk("."):
+        # Prune TESTING_DIR from traversal
+        if TESTING_DIR in dirnames:
+            dirnames.remove(TESTING_DIR)
+        for filename in filenames:
+            filepath = os.path.join(dirpath, filename)
+            try:
+                st = os.stat(filepath)
+                if st.st_mtime > pre_run_timestamp or st.st_atime > pre_run_timestamp:
+                    touched.append(filepath)
+            except OSError:
+                continue
+    return touched
+
+
 def check_test():
     global test_args
     if test_args == "":
@@ -782,6 +936,9 @@ def check_test():
     print(f"Test case {test_case_count} of {test_case_total}")
     passed = True
 
+    # Record timestamp before running student code for file tracking
+    pre_run_timestamp = time.time()
+
     with open(get_testing_path("fileinput"), "rb") as fileinput, open(
         get_testing_path("youroutput"), "w"
     ) as youroutput, open(get_testing_path("yourerror"), "w") as yourerror:
@@ -797,6 +954,9 @@ def check_test():
         print(f"Took more than {timeout_val} seconds to run. FAIL")
         passed = False
 
+    # Find files touched by student's program
+    touched_files = _find_touched_files(pre_run_timestamp)
+
     # Difflog handling
     with open(get_testing_path("difflog"), "w") as outfile:
         diff_popen = subprocess.Popen(
@@ -872,6 +1032,33 @@ def check_test():
                     # Print entire file
                     print("".join(file_lines))
 
+            # Show diffs for files touched by student's program
+            if touched_files and cmps:
+                expected_map = {}
+                for file_pair in cmps:
+                    if len(file_pair) >= 2:
+                        expected_map[os.path.normpath(file_pair[0])] = file_pair[1]
+
+                for filepath in touched_files:
+                    normalized = os.path.normpath(filepath)
+                    if normalized in expected_map:
+                        expected_file = expected_map[normalized]
+                        if os.path.exists(expected_file):
+                            diff_proc = subprocess.Popen(
+                                ["diff", "-U1", "-a", filepath, expected_file],
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE,
+                                text=True,
+                            )
+                            diff_output, _ = diff_proc.communicate()
+                            if diff_output:
+                                print(f"    File output diff ({os.path.basename(filepath)}):")
+                                lines = diff_output.splitlines()
+                                for line in lines[:22]:
+                                    print(f"    {line}")
+                                if len(lines) > 22:
+                                    print(f"    ... ({len(lines) - 22} more lines)")
+
         cleanup()
 
         # Exit program after failed test case