assignment-codeval 0.0.31__tar.gz → 0.0.32__tar.gz

{assignment_codeval-0.0.31 → assignment_codeval-0.0.32}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assignment-codeval
-Version: 0.0.31
+Version: 0.0.32
 Summary: CodEval for evaluating programming assignments
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
@@ -131,13 +131,13 @@ Codeval files (`.codeval` extension) define how to build, run, and test student
 | Tag | Meaning | Function |
 |---|---|---|
 | C | Compile Code | Specifies the command to compile the submission code |
-| CTO | Compile Timeout | Timeout in seconds for the compile command to run |
+| CTO | Compile Timeout | Timeout in seconds for the whole evaluation (compile **and** all test cases) to run |
 | T/HT | Test Case | Command to run to test the submission (HT = hidden test) |
 | I/IB/IF | Supply Input | Input for a test case. I adds a newline, IB does not, IF reads from a file |
 | O/OB/OF | Check Output | Expected output. O adds a newline, OB does not, OF reads from a file |
 | E/EB | Check Error | Expected error output. E adds a newline, EB does not |
-| TO | Timeout | Time limit in seconds for a test case (default: 20) |
-| X | Exit Code | Expected exit code for a test case (default: 0) |
+| TO | Timeout | Time limit in seconds for a test case (default: 10) |
+| X | Exit Code | Expected exit code for a test case (not checked unless X is specified) |
 | TEMP | Temp File | Registers a file to be deleted before and after the next test run |
 | CF | Check Function | Checks that a function is used in the submission |
 | NCF | Check Not Function | Checks that a function is **not** used |
@@ -145,7 +145,6 @@ Codeval files (`.codeval` extension) define how to build, run, and test student
 | PRINT | Print Label | Prints a label/message to stdout |
 | CMP | Compare | Compares two files |
 | Z | Download Zip | Zip files to download from Canvas for test cases |
-| SS | Start Server | Starts a server with timeout and kill-timeout settings |
 
 #### Assignment Description Macros
 

{assignment_codeval-0.0.31 → assignment_codeval-0.0.32}/README.md

@@ -110,13 +110,13 @@ Codeval files (`.codeval` extension) define how to build, run, and test student
 | Tag | Meaning | Function |
 |---|---|---|
 | C | Compile Code | Specifies the command to compile the submission code |
-| CTO | Compile Timeout | Timeout in seconds for the compile command to run |
+| CTO | Compile Timeout | Timeout in seconds for the whole evaluation (compile **and** all test cases) to run |
 | T/HT | Test Case | Command to run to test the submission (HT = hidden test) |
 | I/IB/IF | Supply Input | Input for a test case. I adds a newline, IB does not, IF reads from a file |
 | O/OB/OF | Check Output | Expected output. O adds a newline, OB does not, OF reads from a file |
 | E/EB | Check Error | Expected error output. E adds a newline, EB does not |
-| TO | Timeout | Time limit in seconds for a test case (default: 20) |
-| X | Exit Code | Expected exit code for a test case (default: 0) |
+| TO | Timeout | Time limit in seconds for a test case (default: 10) |
+| X | Exit Code | Expected exit code for a test case (not checked unless X is specified) |
 | TEMP | Temp File | Registers a file to be deleted before and after the next test run |
 | CF | Check Function | Checks that a function is used in the submission |
 | NCF | Check Not Function | Checks that a function is **not** used |
@@ -124,7 +124,6 @@ Codeval files (`.codeval` extension) define how to build, run, and test student
 | PRINT | Print Label | Prints a label/message to stdout |
 | CMP | Compare | Compares two files |
 | Z | Download Zip | Zip files to download from Canvas for test cases |
-| SS | Start Server | Starts a server with timeout and kill-timeout settings |
 
 #### Assignment Description Macros
 

{assignment_codeval-0.0.31 → assignment_codeval-0.0.32}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "assignment-codeval"
-version = "0.0.31"
+version = "0.0.32"
 description = "CodEval for evaluating programming assignments"
 readme = "README.md"
 requires-python = ">=3.12"

{assignment_codeval-0.0.31 → assignment_codeval-0.0.32}/src/assignment_codeval/ai_benchmark.py

@@ -75,8 +75,10 @@ def extract_assignment_from_codeval(codeval_path: str) -> tuple[str, str, str]:
     with open(codeval_path, "r", encoding="utf-8") as f:
         content = f.read()
 
-    # Extract content between ASSIGNMENT START / CRT_HW START and ASSIGNMENT END / CRT_HW END
-    match = re.search(r"(?:ASSIGNMENT START|CRT_HW START) \S+\n(.*?)(?:ASSIGNMENT END|CRT_HW END)", content, re.DOTALL)
+    # Extract content between ASSIGNMENT START / CRT_HW START and ASSIGNMENT END / CRT_HW END.
+    # The title after the marker may contain spaces (e.g. "CRT_HW START Place Boats")
+    # or be absent entirely, so match the rest of the marker line, not a single token.
+    match = re.search(r"(?:ASSIGNMENT START|CRT_HW START)[^\n]*\n(.*?)(?:ASSIGNMENT END|CRT_HW END)", content, re.DOTALL)
     if match:
         description = match.group(1).strip()
     else:
@@ -223,10 +225,17 @@ def call_openai(model_id: str, prompt: str, api_key: str) -> Optional[str]:
     try:
         client = openai.OpenAI(api_key=api_key)
 
+        # Reasoning models (o1/o3/o4 families, gpt-5) reject `max_tokens` and
+        # require `max_completion_tokens` instead.
+        if model_id.startswith(("o1", "o3", "o4", "gpt-5")):
+            token_kwargs = {"max_completion_tokens": 8192}
+        else:
+            token_kwargs = {"max_tokens": 8192}
+
         response = client.chat.completions.create(
             model=model_id,
             messages=[{"role": "user", "content": prompt}],
-            max_tokens=8192,
+            **token_kwargs,
         )
 
         return response.choices[0].message.content

{assignment_codeval-0.0.31 → assignment_codeval-0.0.32}/src/assignment_codeval/convertMD2Html.py

@@ -1,6 +1,7 @@
 import os
 import re
 import zipfile
+from html import escape as html_escape
 
 import markdown
 
@@ -21,13 +22,17 @@ _ANSI_STYLES = {
 
 
 def ansi_to_html(text):
-    """Convert ANSI escape codes in text to HTML span tags."""
+    """Convert ANSI escape codes in text to HTML span tags.
+
+    The text parts are HTML-escaped so characters like < and & render
+    literally instead of being parsed as markup.
+    """
     parts = re.split(r'\x1b\[([\d;]*)m', text)
     result = []
     open_spans = 0
     for i, part in enumerate(parts):
         if i % 2 == 0:
-            result.append(part)
+            result.append(html_escape(part, quote=False))
         else:
             codes = part.split(';') if part else ['0']
             if '0' in codes:
@@ -66,47 +71,48 @@ def _read_file_content(filename, spec_dir, zip_archives=None):
 def sampleTestCases(listOfTC, numOfTC, spec_dir=None, zip_archives=None):
     counter = 0
     samples = "<pre><code>"
+    esc = lambda s: html_escape(s, quote=False)
     for line in listOfTC:
         if line.startswith('T ', 0, 2):
             counter = counter + 1
             if counter > numOfTC:
                 break
-            samples = samples + "\n" + "Command to RUN: " + line[2:]
+            samples = samples + "\n" + "Command to RUN: " + esc(line[2:])
         elif line.startswith('IF '):
             filename = line[3:].strip()
             content = _read_file_content(filename, spec_dir, zip_archives)
             if content is not None:
                 samples = samples + "<span style=\"color:green\">" + ansi_to_html(content) + "</span>"
             else:
-                samples = samples + "<span style=\"color:green\">Input from file: " + filename + "\n</span>"
+                samples = samples + "<span style=\"color:green\">Input from file: " + esc(filename) + "\n</span>"
         elif line.startswith('IB '):
-            samples = samples + "<span style=\"color:green\">" + line[3:] + "</span>"
+            samples = samples + "<span style=\"color:green\">" + esc(line[3:]) + "</span>"
         elif line.startswith('I '):
-            samples = samples + "<span style=\"color:green\">" + line[2:] + "</span>"
+            samples = samples + "<span style=\"color:green\">" + esc(line[2:]) + "</span>"
         elif line.startswith('OF '):
             filename = line[3:].strip()
             content = _read_file_content(filename, spec_dir, zip_archives)
             if content is not None:
                 samples = samples + "<span style=\"color:blue\">" + ansi_to_html(content) + "</span>"
             else:
-                samples = samples + "<span style=\"color:blue\">Expected output from file: " + filename + "\n</span>"
+                samples = samples + "<span style=\"color:blue\">Expected output from file: " + esc(filename) + "\n</span>"
         elif line.startswith('OB '):
-            samples = samples + "<span style=\"color:blue\">" + line[3:] + "</span>"
+            samples = samples + "<span style=\"color:blue\">" + esc(line[3:]) + "</span>"
         elif line.startswith('O '):
-            samples = samples + "<span style=\"color:blue\">" + line[2:] + "</span>"
+            samples = samples + "<span style=\"color:blue\">" + esc(line[2:]) + "</span>"
         elif line.startswith('X '):
-            samples = samples + "Expected Exit Code: " + line[2:]
+            samples = samples + "Expected Exit Code: " + esc(line[2:])
         elif line.startswith('EF '):
             filename = line[3:].strip()
             content = _read_file_content(filename, spec_dir, zip_archives)
             if content is not None:
                 samples = samples + "<span style=\"color:Tomato\">" + ansi_to_html(content) + "</span>"
             else:
-                samples = samples + "<span style=\"color:Tomato\">Expected error from file: " + filename + "\n</span>"
+                samples = samples + "<span style=\"color:Tomato\">Expected error from file: " + esc(filename) + "\n</span>"
         elif line.startswith('EB '):
-            samples = samples + "<span style=\"color:Tomato\">" + line[3:] + "</span>"
+            samples = samples + "<span style=\"color:Tomato\">" + esc(line[3:]) + "</span>"
         elif line.startswith('E '):
-            samples = samples + "<span style=\"color:Tomato\">" + line[2:] + "</span>"
+            samples = samples + "<span style=\"color:Tomato\">" + esc(line[2:]) + "</span>"
         else:
             continue
     samples = samples + "</code></pre>"
@@ -140,8 +146,9 @@ def mdToHtml(file_name, files_resolver=None):
                     compile_command = line[2:].strip()
                 if past_crt_hw and line.startswith('Z '):
                     zip_paths.append(line[2:].strip())
-                if 'EXMPLS ' in line:
-                    numOfSampleTC = int(line[7:])
+                exmpls_match = re.search(r'EXMPLS\s+([0-9]+)', line)
+                if exmpls_match:
+                    numOfSampleTC = int(exmpls_match.group(1))
                 text = text + line
         spec_dir = os.path.dirname(os.path.abspath(file_name))
 
@@ -158,7 +165,10 @@ def mdToHtml(file_name, files_resolver=None):
             for zf in zip_archives:
                 zf.close()
 
-        assignment = re.sub('EXMPLS [0-9]+', samples, assignment)
+        # Use a function replacement so backslash sequences in the sample data
+        # (e.g. "\d" or "\1" in expected output) are inserted literally rather
+        # than interpreted as regex escapes.
+        assignment = re.sub('EXMPLS [0-9]+', lambda m: samples, assignment)
         if compile_command:
             assignment = assignment.replace('COMPILE', compile_command)
         # Handle FILE macros in two passes:

{assignment_codeval-0.0.31 → assignment_codeval-0.0.32}/src/assignment_codeval/create_assignment.py

@@ -95,7 +95,7 @@ def files_resolver(filename):
             if not matches:
                 warn(f"could not find {filename} in {z}")
             elif len(matches) > 1:
-                warn("found multiple matches for {filename} in {z}: {matches}")
+                warn(f"found multiple matches for {filename} in {z}: {matches}")
             if matches:
                 zfname = matches[0]
                 with TemporaryDirectory("ce") as tmpdir:
@@ -115,7 +115,9 @@ def files_resolver(filename):
 @click.option("--verbose/--no-verbose", default=False, show_default=True, help="Verbose actions")
 @click.option("--group_name", default="Assignments", show_default=True,
               help="Group name in which assignments needs to be created.")
-def create_assignment(dryrun, verbose, course_name, group_name, specname, extra):
+@click.option("--active/--inactive", default=True, show_default=True,
+              help="Only look at active courses.")
+def create_assignment(dryrun, verbose, course_name, group_name, specname, extra, active):
     """
         Create the assignment in the given course.
     """
@@ -128,7 +130,7 @@ def create_assignment(dryrun, verbose, course_name, group_name, specname, extra)
     (canvas, user) = connect_to_canvas()
 
     try:
-        course = get_course(canvas, course_name)
+        course = get_course(canvas, course_name, is_active=active)
     except Exception as e:
         errorWithException(f'get_course api failed with following error : {e}')
     else:
@@ -152,16 +154,17 @@ def create_assignment(dryrun, verbose, course_name, group_name, specname, extra)
         canvas_folder = get_assignment_subfolder(course, codeval_folder, assign_name_early)
     if extra:
         upload_assignment_files(canvas_folder, extra)
-    # find zipfiles in spec
+    # find zipfiles in spec; resolve them relative to the spec directory so the
+    # command works when run from any working directory
+    spec_dir = os.path.dirname(os.path.abspath(specname))
     with open(specname) as f:
         for line in f:
             if line.startswith("Z "):
-                zipfile = line[2:].strip()
-                if zipfile not in zip_files:
-                    zip_files.append(zipfile)
+                zip_name = os.path.join(spec_dir, line[2:].strip())
+                if zip_name not in zip_files:
+                    zip_files.append(zip_name)
 
     # Extract FILE macros and upload local files that exist
-    spec_dir = os.path.dirname(os.path.abspath(specname))
     file_macros = extract_file_macros(specname)
     for filename in file_macros:
         if filename not in file_dict:

{assignment_codeval-0.0.31 → assignment_codeval-0.0.32}/src/assignment_codeval/evaluate.py

@@ -3,10 +3,10 @@
 import ast
 import os
 import re
+import signal
 import subprocess
 import sys
 import traceback
-import threading
 import time
 
 import click
@@ -295,6 +295,22 @@ def _is_function_used(function_name, files, allow_regex_fallback=True):
     return False
 
 
+def _resolve_function_check_files():
+    """Derive source files for a CF/NCF tag that gave no explicit filenames.
+
+    Source files come from the most recent compile command. Commands that don't
+    name their sources (e.g. ``make``) yield nothing, in which case the check
+    cannot be trusted, so print a visible warning rather than silently passing a
+    forbidden-function ban or failing a required-function check for everyone.
+    """
+    files = _extract_source_files_from_compile(last_compile_command)
+    if not files:
+        print(f"    WARNING: could not determine source files from compile command "
+              f"'{last_compile_command}'; the function-usage check is unreliable. "
+              f"List the source file(s) explicitly after the function name in the CF/NCF tag.")
+    return files
+
+
 def check_function(args):
     """Will be followed by a function name and an optional list of files to check to ensure that
     the function is used by one of those files.
@@ -323,7 +339,7 @@ def check_function(args):
     if not files:
         # No filename provided — derive source files from the compile command and
         # disable the regex fallback (compiled-artifact check only).
-        files = _extract_source_files_from_compile(last_compile_command)
+        files = _resolve_function_check_files()
         if _is_function_used(function_name, files, allow_regex_fallback=False):
             print(f"Used {function_name} PASSED")
         else:
@@ -457,7 +473,7 @@ def check_not_function(args):
     if not files:
         # No filename provided — derive source files from the compile command and
         # disable the regex fallback (compiled-artifact check only).
-        files = _extract_source_files_from_compile(last_compile_command)
+        files = _resolve_function_check_files()
         used = _is_function_used(function_name, files, allow_regex_fallback=False)
     else:
         used = _is_function_used(function_name, files)
@@ -683,7 +699,6 @@ def check_output_file(output_file):
     Returns:
         None
     """
-    output_length(os.path.getsize(output_file)) 
     with open(output_file, "r") as infile:
         output_lines = infile.readlines()
         
@@ -773,7 +788,7 @@ def _post_test_temp_cleanup():
 
 
 def timeout(timeout_sec):
-    """Specifies the time limit in seconds for a test case to run. Defaults to 20 seconds.
+    """Specifies the time limit in seconds for a test case to run. Defaults to 10 seconds.
 
     Arguments:
         timeout_sec: time limit in seconds for a test case to run
@@ -799,7 +814,9 @@ def output_length(length):
 
 
 def exit_code(test_case_exit_code):
-    """Specifies the expected exit code for a test case. Defaults to zero.
+    """Specifies the expected exit code for a test case.
+
+    If no X tag is given, the exit code is not checked.
 
     Arguments:
         test_case_exit_code: the expected exit code for a test case
@@ -811,48 +828,6 @@ def exit_code(test_case_exit_code):
     expected_exit_code = int(test_case_exit_code)
 
 
-def start_server(timeout_sec, kill_timeout_sec, *server_cmd):
-    """Command containing timeout (wait until server starts), kill timeout (wait to kill the server),
-    and the command to start a server
-
-    Arguments:
-        timeout_sec: timeout in seconds to wait for server to start
-        kill_timeout_sec: timeout in seconds to wait until killing the server
-        server_cmd: command to run to start the server
-
-    Returns:
-        None
-    """
-
-    print(
-        f'Starting server with command: {" ".join(server_cmd)} and sleeping for: {timeout_sec}. Will kill server '
-        f'after {kill_timeout_sec} seconds.'
-    )
-
-    # Send output to compile log in background
-    with open(get_testing_path("compilelog"), "w") as outfile:
-        server_popen = subprocess.Popen(
-            server_cmd, shell=True, stdout=outfile, stderr=outfile, text=True
-        )
-
-    print(f"Server pid: {server_popen.pid}. Sleeping for {timeout_sec} seconds.")
-    # Block for timeout_sec so that server can start
-    time.sleep(float(timeout_sec))
-
-    # Kill the server after the timeout
-    def kill_server(pid):
-        print(f"Killing {pid}")
-        subprocess.Popen(
-            ["kill", "-9", f"{pid}"], stdout=subprocess.PIPE, stderr=subprocess.PIPE
-        )
-
-    kill_timer = threading.Timer(
-        float(kill_timeout_sec), kill_server, *[server_popen.pid]
-    )
-    kill_timer.daemon = True
-    kill_timer.start()
-
-
 """
 Here is where the tags are mapped to functions.
 Any tags that are added or changed must be modified here.
@@ -883,7 +858,6 @@ tag_func_map = {
     "TO": timeout,
     "OLEN": output_length,
     "X": exit_code,
-    "SS": start_server,
     "TEMP": register_temp_file,
 }
 
@@ -921,8 +895,10 @@ def parse_tags(tags: list[str]):
         None
     """
     # Pattern matches: TAG arguments (arguments required)
-    # Use single space separator to preserve leading whitespace in values
-    tag_pattern = r"([A-Z_]+) (.*)"
+    # A single space or tab separates the tag from its value; any further
+    # whitespace is preserved as part of the value. Accepting a tab here keeps a
+    # tab-separated tag (e.g. "T\t./prog") from being silently dropped.
+    tag_pattern = r"([A-Z_]+)[ \t](.*)"
     # Pattern for tag with optional arguments
     tag_only_pattern = r"([A-Z_]+)\s*$"
 
@@ -1173,7 +1149,8 @@ def check_test():
         get_testing_path("youroutput"), "w"
     ) as youroutput, open(get_testing_path("yourerror"), "w") as yourerror:
         test_exec = subprocess.Popen(
-            test_args, shell=True, stdin=fileinput, stdout=youroutput, stderr=yourerror
+            test_args, shell=True, stdin=fileinput, stdout=youroutput, stderr=yourerror,
+            start_new_session=True,
         )
 
     # Timeout handling
@@ -1183,11 +1160,21 @@ def check_test():
     except subprocess.TimeoutExpired:
         print(f"Took more than {timeout_val} seconds to run. FAIL")
         passed = False
+        # Kill the whole process group so orphaned children (the shell spawned
+        # by shell=True and anything it launched) don't keep running.
+        try:
+            os.killpg(os.getpgid(test_exec.pid), signal.SIGKILL)
+        except (ProcessLookupError, PermissionError):
+            test_exec.kill()
+        test_exec.communicate()
 
     # Find files touched by student's program
     touched_files = _find_touched_files(pre_run_timestamp)
 
-    # Difflog handling
+    # Difflog handling. Pass/fail is decided by the diff exit status (0 == files
+    # identical), NOT by how much of the diff we render: OLEN/OF control only the
+    # amount of rendered text and must never be able to hide a real mismatch.
+    output_differs = False
     with open(get_testing_path("difflog"), "w") as outfile:
         diff_popen = subprocess.Popen(
             ["diff", "-U1", "-a",
@@ -1195,6 +1182,8 @@ def check_test():
             stdout=subprocess.PIPE, stderr=subprocess.PIPE,
         )
         raw_output, _ = diff_popen.communicate()
+        if diff_popen.returncode != 0:
+            output_differs = True
         outfile.write(_render_diff_output(raw_output[:output_length_limit]))
 
     # Append to difflog second time around
@@ -1205,14 +1194,17 @@ def check_test():
             stdout=subprocess.PIPE, stderr=subprocess.PIPE,
         )
         raw_output, _ = diff_popen.communicate()
+        if diff_popen.returncode != 0:
+            output_differs = True
         outfile.write(_render_diff_output(raw_output[:output_length_limit]))
 
     # Now read all the lines to accumulate both diffs
     with open(get_testing_path("difflog"), "r") as infile:
         diff_lines = infile.readlines()
 
-    if len(diff_lines):
+    if output_differs:
         passed = False
+    if len(diff_lines):
         parse_diff(diff_lines, TESTING_DIR)
 
     # Exit code handling
@@ -1354,7 +1346,9 @@ def run_evaluation(codeval_file):
                 continue
             if in_crt_hw_block:
                 continue
-            parts = testcase.split(" ", 1)
+            parts = stripped.split(None, 1)
+            if not parts:
+                continue
             tag = parts[0]
             if tag == "T" or tag == "HT" or tag == "TCMD":
                 test_case_total += 1

{assignment_codeval-0.0.31 → assignment_codeval-0.0.32}/src/assignment_codeval/file_utils.py

@@ -5,7 +5,7 @@ import subprocess
 
 import requests
 import zipfile
-from assignment_codeval.commons import debug, error
+from assignment_codeval.commons import debug, errorWithException
 
 
 def download_attachment(directory, attachment):
@@ -13,26 +13,31 @@ def download_attachment(directory, attachment):
         curPath = os.getcwd()
         directory = os.path.join(curPath, directory)
 
-    fname = attachment['display_name']
-    prefix = os.path.splitext(fname)[0]
-    suffix = os.path.splitext(fname)[1]
+    # display_name is attacker-controlled; collapse it to a bare filename so it
+    # cannot escape `directory` via path separators or "..".
+    fname = os.path.basename(attachment['display_name'])
+    if fname in ('', '.', '..'):
+        errorWithException(f"refusing to download attachment with unsafe name: {attachment['display_name']!r}")
     durl = attachment['url']
+    dest = os.path.join(directory, fname)
     with requests.get(durl) as response:
         if response.status_code != 200:
-            error(f'error {response.status_code} fetching {durl}')
-        with open(os.path.join(directory, f"{prefix}{suffix}"), "wb") as fd:
-            for chunk in response.iter_content():
+            # Don't write the error page body as if it were the attachment.
+            errorWithException(f'error {response.status_code} fetching {durl}')
+        with open(dest, "wb") as fd:
+            for chunk in response.iter_content(chunk_size=8192):
                 fd.write(chunk)
 
-    return os.path.join(directory, fname)
+    return dest
 
 
 def unzip(filepath, dir, delete=False):
     with zipfile.ZipFile(filepath) as file:
         for zi in file.infolist():
-            file.extract(zi.filename, path=dir)
+            # extract() sanitizes "../" and absolute members and returns the
+            # real path it wrote; stat/chmod that, not the raw archive name.
+            fname = file.extract(zi.filename, path=dir)
             debug(f"extracting {zi.filename}")
-            fname = os.path.join(dir, zi.filename)
             s = os.stat(fname)
             # the user executable bit is set
             perms = (s.st_mode | (zi.external_attr >> 16)) & 0o777

{assignment_codeval-0.0.31 → assignment_codeval-0.0.32}/src/assignment_codeval/github_connect.py

@@ -1,5 +1,6 @@
 import os
 import re
+import shutil
 import subprocess
 from time import sleep
 
@@ -10,6 +11,27 @@ from assignment_codeval.commons import error, info
 HEX_DIGITS = "0123456789abcdefABCDEF"
 
 
+def _is_safe_repo_url(url):
+    """Reject repo URLs that git would treat as an option or a code-executing transport.
+
+    A clone URL is partly derived from a student-controlled GitHub id, so a value
+    beginning with '-' (argument injection) or using git's ext::/file::/fd:: transport
+    helpers (arbitrary command execution) must never reach `git clone`.
+    """
+    if not url or url.startswith('-'):
+        return False
+    authority = url.split('/', 1)[0]
+    # ext::sh -c ... and friends use a "<helper>::" prefix before the first slash.
+    if '::' in authority:
+        return False
+    # Require a recognized remote form: scheme://... or scp-like git@host:path.
+    if re.match(r'^(https?|git|ssh)://', url, re.IGNORECASE):
+        return True
+    if re.match(r'^[\w.-]+@[\w.-]+:', url):
+        return True
+    return False
+
+
 def _read_metadata(ssid_dir):
     """Read metadata.txt and return a dict of key=value pairs."""
     metadata_path = os.path.join(ssid_dir, "metadata.txt")
@@ -24,6 +46,22 @@ def _read_metadata(ssid_dir):
     return metadata
 
 
+def _read_desired_commit(content_path):
+    """Return the validated (hex) commit hash from content.txt, or None.
+
+    content.txt holds the commit the student submitted, possibly wrapped in HTML
+    tags/entities from Canvas; strip those and accept it only if it is all hex.
+    """
+    if not os.path.exists(content_path):
+        return None
+    with open(content_path, "r") as cfd:
+        content = re.sub(r"<.*?>", "", cfd.readline().strip()).strip()
+        content = re.sub(r"&[a-z]+;", "", content).strip()
+    if content and all(c in HEX_DIGITS for c in content):
+        return content
+    return None
+
+
 @click.command()
 @click.argument("target_dir", metavar="SUBMISSIONS_DIR", default='./submissions')
 @click.option("--clone-delay",
@@ -65,8 +103,21 @@ def _setup_repos_for_assignment(assignment_path, clone_delay):
         submission_path = os.path.join(ssid_dir, "submission")
 
         if os.path.exists(submission_path):
-            info(f"skipping {ssid_dir}, repo already exists at {submission_path}")
-            continue
+            # A clone already exists. If the student resubmitted a new commit,
+            # the old checkout would grade the wrong attempt — re-clone in that
+            # case. If we can't determine the desired commit, leave it as-is.
+            desired_commit = _read_desired_commit(content_path)
+            recorded_commit = None
+            if os.path.exists(success_path):
+                with open(success_path, "r") as sfd:
+                    recorded_commit = sfd.readline().strip()
+            if not desired_commit or recorded_commit == desired_commit:
+                info(f"skipping {ssid_dir}, repo already at {recorded_commit or 'existing checkout'}")
+                continue
+            info(f"re-cloning {ssid_dir}: checked-out {recorded_commit} != submitted {desired_commit}")
+            # Guard against removing anything but the clone directory.
+            if os.path.basename(submission_path.rstrip("/")) == "submission":
+                shutil.rmtree(submission_path)
 
         metadata = _read_metadata(ssid_dir)
         repo_url = metadata.get('github_repo', '')
@@ -76,20 +127,22 @@ def _setup_repos_for_assignment(assignment_path, clone_delay):
 
         click.echo(f"Setting up repo for {ssid_dir}")
 
+        if not _is_safe_repo_url(repo_url):
+            error(f"❌ refusing to clone unsafe repo url for {ssid}: {repo_url!r}")
+            with open(result_path, "w") as fd:
+                print(f"❌ refusing to clone unsafe repo url: {repo_url}", file=fd)
+            continue
+
         with open(result_path, "w") as fd:
             # Read commit hash from content.txt
-            content = None
-            if os.path.exists(content_path):
-                with open(content_path, "r") as cfd:
-                    content = re.sub(r"<.*?>", "", cfd.readline().strip()).strip()
-                    content = re.sub(r"&[a-z]+;", "", content).strip()
-            if not content or not all(c in HEX_DIGITS for c in content):
-                print(f"❌ an invalid git digest was found: {content}", file=fd)
+            content = _read_desired_commit(content_path)
+            if not content:
+                print(f"❌ an invalid git digest was found in {content_path}", file=fd)
                 continue
 
             click.echo(f"Cloning {repo_url} to {ssid_dir}")
             print(f"cloning {repo_url}", file=fd)
-            rc = subprocess.run(['git', 'clone', repo_url, submission_path], stdout=fd, stderr=subprocess.STDOUT)
+            rc = subprocess.run(['git', 'clone', '--', repo_url, submission_path], stdout=fd, stderr=subprocess.STDOUT)
             if rc.returncode != 0:
                 error(f"❌ error {rc.returncode} connecting to github repo for {ssid} using {repo_url}")
                 continue

{assignment_codeval-0.0.31 → assignment_codeval-0.0.32}/src/assignment_codeval/recent_comments.py

@@ -38,7 +38,8 @@ def format_comment_preview(comment: str, prefix: str) -> str:
     Returns:
         Formatted preview string
     """
-    # Strip the prefix
+    # Strip the prefix (stored comments use the rstripped prefix + newline)
+    prefix = prefix.rstrip()
     text = comment[len(prefix):] if comment.startswith(prefix) else comment
     lines = text.strip().split('\n')
 
@@ -183,7 +184,10 @@ def recent_comments(course_name, active, time_period, codeval_prefix, verbose, s
                 codeval_comments = []
                 for comment in comments:
                     comment_text = comment.get("comment", "")
-                    if not comment_text.startswith(codeval_prefix):
+                    # Stored comments begin with the rstripped prefix + newline
+                    # (see upload_submission_comments), so match without the
+                    # trailing space the default prefix carries.
+                    if not comment_text.startswith(codeval_prefix.rstrip()):
                         continue
 
                     created_at_str = comment.get("createdAt", "")

{assignment_codeval-0.0.31 → assignment_codeval-0.0.32}/src/assignment_codeval/submissions.py

@@ -3,6 +3,7 @@ import re
 import shutil
 import subprocess
 import time
+import unicodedata
 from configparser import ConfigParser
 from datetime import datetime, timedelta, timezone
 from zoneinfo import ZoneInfo
@@ -16,6 +17,43 @@ import requests
 from assignment_codeval.canvas_utils import connect_to_canvas, get_course, get_courses, get_assignment
 from assignment_codeval.commons import debug, error, info, warn, despace
 
+# Bookkeeping files the tool writes into each student directory. A student must
+# never be able to clobber these via an attachment filename — in particular
+# SUBSTITUTIONS.txt is applied to their own grading comment, which would let a
+# student forge their grade.
+_RESERVED_SUBMISSION_FILES = {
+    "metadata.txt", "content.txt", "comments.txt", "comments.txt.sent",
+    "SUBSTITUTIONS.txt", "codeval_path.txt", "codeval.txt", "last-comment.txt",
+    "results.html", "gh_success.txt",
+}
+
+
+def _is_within_tree(tree_root, path):
+    """True if ``path``'s parent directory resolves to a location inside ``tree_root``.
+
+    Resolves symlinks so a student-planted symlink in a cloned repo cannot
+    redirect our file writes outside the submission directory.
+    """
+    real_root = os.path.realpath(tree_root)
+    real_parent = os.path.realpath(os.path.dirname(path))
+    return real_parent == real_root or real_parent.startswith(real_root + os.sep)
+
+
+def _safe_copy_into_tree(src, dst, tree_root):
+    """Copy ``src`` to ``dst``, refusing to follow symlinks out of ``tree_root``.
+
+    Returns True if the file was copied, False if the destination was rejected
+    as escaping the tree (possible symlink attack).
+    """
+    if not _is_within_tree(tree_root, dst):
+        error(f"refusing to write {dst}: resolves outside the submission tree")
+        return False
+    # Don't write *through* a student-planted symlink at the final path.
+    if os.path.islink(dst):
+        os.unlink(dst)
+    shutil.copy(src, dst)
+    return True
+
 
 def _parse_codeval_test_info(codeval_file):
     """Parse a codeval file and return a mapping from test case number to test metadata.
@@ -140,12 +178,31 @@ def find_codeval_file(codeval_dir, assignment_name):
     return None
 
 
-def get_github_repo_url(course, user_id, config_parser, github_field="github"):
+def github_safe_name(name):
+    """a person's name as a valid github repo-name fragment.
+
+    accented letters lose their accents, anything github won't take in a
+    repo name becomes "-", runs collapse, and the edges are trimmed.
+    Matches the repo naming used by gh-class-sak.
+    """
+    ascii_name = unicodedata.normalize("NFKD", name)
+    ascii_name = "".join(ch for ch in ascii_name if not unicodedata.combining(ch))
+    safe = "".join(ch if (ch.isascii() and ch.isalnum()) or ch in "._-" else "-"
+                   for ch in ascii_name)
+    while "--" in safe:
+        safe = safe.replace("--", "-")
+    return safe.strip("-._") or "x"
+
+
+def get_github_repo_url(course, user_id, config_parser, github_field="github", from_name=None):
     """
     Get the GitHub repo URL for a user in a course.
 
     Returns the repo URL string if found, or None if GitHub is not configured
     for this course or the user doesn't have a GitHub link in their profile.
+
+    If from_name is given, the repo URL is built from that name using
+    gh-class-sak's repo naming instead of the profile's GitHub link.
     """
     gh_key = course.name.replace(":", "").replace("=", "")
     if 'GITHUB' not in config_parser or gh_key not in config_parser['GITHUB']:
@@ -153,6 +210,9 @@ def get_github_repo_url(course, user_id, config_parser, github_field="github"):
 
     gh_repo_prefix = config_parser['GITHUB'][gh_key]
 
+    if from_name is not None:
+        return f"{gh_repo_prefix}-{github_safe_name(from_name)}.git"
+
     try:
         user = course.get_user(user_id)
         profile = user.get_profile(include=["links", "link"])
@@ -428,9 +488,14 @@ def upload_submission_comments(submissions_dir, codeval_prefix, delete):
                         substitutions = _parse_substitutions_file(subs_file)
                         comment = _apply_substitutions(comment, substitutions)
                     if delete:
-                        all_comments = sorted(submission.submission_comments, key=lambda c: c['created_at'])
-                        if all_comments:
-                            last_comment_id = all_comments[-1]["id"]
+                        # Only delete a previous codeval comment, never a
+                        # student's or TA's comment that happens to be newest.
+                        codeval_comments = sorted(
+                            (c for c in submission.submission_comments
+                             if c.get('comment', '').startswith(codeval_prefix.rstrip())),
+                            key=lambda c: c['created_at'])
+                        if codeval_comments:
+                            last_comment_id = codeval_comments[-1]["id"]
                             delete_submission_comment(canvas, course.id, assignment.id, student_id, last_comment_id)
                     canvas_url, canvas_token = _get_canvas_config()
                     requests.put(
@@ -441,10 +506,12 @@ def upload_submission_comments(submissions_dir, codeval_prefix, delete):
                             'comment[file_ids][]': file_id,
                         }
                     ).raise_for_status()
+                    # Only mark as sent once we have actually uploaded; otherwise
+                    # a transient "no submission" would skip this student forever.
+                    with open(f"{dirpath}/comments.txt.sent", "w") as fd:
+                        fd.write(datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'))
                 else:
                     warn(f"no submission found for {student_id} in {course_name}: {assignment_name}")
-                with open(f"{dirpath}/comments.txt.sent", "w") as fd:
-                    fd.write(datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'))
 
 
 @click.command()
@@ -472,8 +539,10 @@ def evaluate_submissions(codeval_dir, submissions_dir):
     raw_command = parser["RUN"]["command"]
     if not raw_command:
         warn(f"commands section under [RUN] in {parser.config_file} is empty")
-    for dirpath, dirnames, filenames in os.walk(submissions_dir):
-        match = re.match(fr'^{submissions_dir}/([^/]+)/([^/]+)/([^/]+)$', dirpath)
+    clean_submissions_dir = submissions_dir.rstrip('/').replace('\\', '/')
+    for dirpath, dirnames, filenames in os.walk(clean_submissions_dir):
+        dirpath = dirpath.replace('\\', '/')
+        match = re.match(fr'^{re.escape(clean_submissions_dir)}/([^/]+)/([^/]+)/([^/]+)$', dirpath)
         if not match:
             continue
 
@@ -496,17 +565,27 @@ def evaluate_submissions(codeval_dir, submissions_dir):
         has_cd_tag = False
         zip_files = []
         move_to_next_submission = False
+        in_crt_hw_block = False
         with open(codeval_file, "r") as fd:
             for line in fd:
                 line = line.strip()
-                if line.startswith("CTO"):
+                # Skip the Canvas description markdown; it is prose, not tags
+                # (e.g. "Zip your solution" must not be read as a Z tag).
+                if line.startswith("CRT_HW"):
+                    in_crt_hw_block = not in_crt_hw_block
+                    continue
+                if in_crt_hw_block or not line:
+                    continue
+                parts = line.split(None, 1)
+                tag = parts[0]
+                if tag == "CTO":
                     try:
-                        compile_timeout = int(line.split(None, 1)[1])
-                    except Exception:
+                        compile_timeout = int(parts[1])
+                    except (IndexError, ValueError):
                         warn(f"could not parse compile timeout from {line}, using default {compile_timeout}")
-                if line.startswith("CD"):
+                if tag == "CD" and len(parts) > 1:
                     has_cd_tag = True
-                    cd_dir = line.split()[1].strip()
+                    cd_dir = parts[1].split()[0].strip()
                     if cd_dir == "GITHUB_DIRECTORY":
                         cd_dir = assignment_name
                     assignment_working_dir = os.path.normpath(
@@ -515,8 +594,8 @@ def evaluate_submissions(codeval_dir, submissions_dir):
                         out = f"{assignment_working_dir} does not exist or is not a directory\n".encode('utf-8')
                         move_to_next_submission = True
                         break
-                if line.startswith("Z"):
-                    zip_files.append(line.split(None, 1)[1])
+                if tag == "Z" and len(parts) > 1:
+                    zip_files.append(parts[1].strip())
 
         # If no CD tag and this is a GitHub submission (has .git), use assignment name as working dir
         if not has_cd_tag and os.path.exists(os.path.join(submission_dir, ".git")):
@@ -549,8 +628,12 @@ def evaluate_submissions(codeval_dir, submissions_dir):
                 full_assignment_working_dir = os.path.join(submission_link, assignment_working_dir)
                 if not os.path.isdir(full_assignment_working_dir):
                     out = b"no submission directory found"
+                elif not _safe_copy_into_tree(
+                        codeval_file,
+                        os.path.join(full_assignment_working_dir, "codeval.txt"),
+                        submission_dir):
+                    out = b"refusing to write codeval.txt into the submission (possible symlink attack)"
                 else:
-                    shutil.copy(codeval_file, os.path.join(full_assignment_working_dir, "codeval.txt"))
                     codeval_source_dir = os.path.dirname(codeval_file)
                     with open(codeval_file, "r") as cf:
                         for cf_line in cf:
@@ -559,7 +642,10 @@ def evaluate_submissions(codeval_dir, submissions_dir):
                                 ref_file = parts[1].strip()
                                 src = os.path.join(codeval_source_dir, ref_file)
                                 if os.path.isfile(src):
-                                    shutil.copy(src, os.path.join(full_assignment_working_dir, ref_file))
+                                    _safe_copy_into_tree(
+                                        src,
+                                        os.path.join(full_assignment_working_dir, ref_file),
+                                        submission_dir)
 
                     command = command.replace("SUBMISSIONS", full_assignment_working_dir)
                     info(f"command to execute: {command}")
@@ -579,7 +665,9 @@ def evaluate_submissions(codeval_dir, submissions_dir):
                         info("finished executing docker")
 
         info("writing results")
-        with open(f"{dirpath}/comments.txt", "ab") as fd:
+        # Truncate, don't append: re-running evaluation must replace the prior
+        # result for this submission, not double it up.
+        with open(f"{dirpath}/comments.txt", "wb") as fd:
             fd.write(out)
         info("continuing")
 
@@ -599,8 +687,11 @@ def evaluate_submissions(codeval_dir, submissions_dir):
 @click.option("--codeval-prefix", help="prefix for codeval comments", default="codeval: ", show_default=True)
 @click.option("--include-empty", is_flag=True, help="include empty submissions")
 @click.option("--for-name", help="only download submissions for this student name")
+@click.option("--repo-from-name", is_flag=True,
+              help="construct the github repo url from the student name (gh-class-sak naming) "
+                   "instead of the github link in the canvas profile")
 def download_submissions(course_name, assignment_name, active, until_window, target_dir, include_commented,
-                         codeval_prefix, include_empty, uncommented_for, for_name):
+                         codeval_prefix, include_empty, uncommented_for, for_name, repo_from_name):
     """
     Download submissions for a given assignment in a course from Canvas.
 
@@ -656,7 +747,7 @@ def download_submissions(course_name, assignment_name, active, until_window, tar
                 info(f"downloading submissions for {course.name}: {assignment.name}")
                 _download_assignment_submissions(
                     canvas, course, assignment, target_dir, include_commented, codeval_prefix,
-                    include_empty, uncommented_for, for_name
+                    include_empty, uncommented_for, for_name, repo_from_name
                 )
         return
 
@@ -664,12 +755,12 @@ def download_submissions(course_name, assignment_name, active, until_window, tar
     assignment = get_assignment(course, assignment_name)
     _download_assignment_submissions(
         canvas, course, assignment, target_dir, include_commented, codeval_prefix,
-        include_empty, uncommented_for, for_name
+        include_empty, uncommented_for, for_name, repo_from_name
     )
 
 
 def _download_assignment_submissions(canvas, course, assignment, target_dir, include_commented, codeval_prefix,
-                                     include_empty, uncommented_for, for_name):
+                                     include_empty, uncommented_for, for_name, repo_from_name=False):
     """Download submissions for a single assignment."""
     submission_dir = os.path.join(target_dir, despace(course.name), despace(assignment.name))
     os.makedirs(submission_dir, exist_ok=True)
@@ -716,7 +807,8 @@ def _download_assignment_submissions(canvas, course, assignment, target_dir, inc
         os.makedirs(student_submission_dir, exist_ok=True)
 
         metapath = os.path.join(student_submission_dir, "metadata.txt")
-        github_repo = get_github_repo_url(course, submission.user_id, parser)
+        github_repo = get_github_repo_url(course, submission.user_id, parser,
+                                          from_name=student_name if repo_from_name else None)
         with open(metapath, "w") as fd:
             print(f"""id={student_id}
 name={student_name}
@@ -748,11 +840,17 @@ github_repo={github_repo or ''}""", file=fd)
 
         if hasattr(submission, "attachments"):
             for attachment in submission.attachments:
-                fname = attachment.filename
+                # attachment.filename is student-controlled: collapse to a bare
+                # name so it can't traverse out of the dir, and never let it
+                # overwrite one of the tool's control files.
+                fname = os.path.basename(attachment.filename)
+                if fname in ('', '.', '..') or fname in _RESERVED_SUBMISSION_FILES:
+                    warn(f"skipping attachment with reserved/unsafe name {attachment.filename!r} "
+                         f"for student {student_id}")
+                    continue
                 filepath = os.path.join(student_submission_dir, fname)
                 attachment.download(filepath)
-
-            debug(f"Downloaded submission for student {student_id} to {filepath}")
+                debug(f"Downloaded attachment {fname} for student {student_id} to {filepath}")
 
     return submission_dir
 

{assignment_codeval-0.0.31 → assignment_codeval-0.0.32}/src/assignment_codeval.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assignment-codeval
-Version: 0.0.31
+Version: 0.0.32
 Summary: CodEval for evaluating programming assignments
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
@@ -131,13 +131,13 @@ Codeval files (`.codeval` extension) define how to build, run, and test student
 | Tag | Meaning | Function |
 |---|---|---|
 | C | Compile Code | Specifies the command to compile the submission code |
-| CTO | Compile Timeout | Timeout in seconds for the compile command to run |
+| CTO | Compile Timeout | Timeout in seconds for the whole evaluation (compile **and** all test cases) to run |
 | T/HT | Test Case | Command to run to test the submission (HT = hidden test) |
 | I/IB/IF | Supply Input | Input for a test case. I adds a newline, IB does not, IF reads from a file |
 | O/OB/OF | Check Output | Expected output. O adds a newline, OB does not, OF reads from a file |
 | E/EB | Check Error | Expected error output. E adds a newline, EB does not |
-| TO | Timeout | Time limit in seconds for a test case (default: 20) |
-| X | Exit Code | Expected exit code for a test case (default: 0) |
+| TO | Timeout | Time limit in seconds for a test case (default: 10) |
+| X | Exit Code | Expected exit code for a test case (not checked unless X is specified) |
 | TEMP | Temp File | Registers a file to be deleted before and after the next test run |
 | CF | Check Function | Checks that a function is used in the submission |
 | NCF | Check Not Function | Checks that a function is **not** used |
@@ -145,7 +145,6 @@ Codeval files (`.codeval` extension) define how to build, run, and test student
 | PRINT | Print Label | Prints a label/message to stdout |
 | CMP | Compare | Compares two files |
 | Z | Download Zip | Zip files to download from Canvas for test cases |
-| SS | Start Server | Starts a server with timeout and kill-timeout settings |
 
 #### Assignment Description Macros
 

{assignment_codeval-0.0.31 → assignment_codeval-0.0.32}/src/assignment_codeval.egg-info/SOURCES.txt

@@ -15,7 +15,6 @@ src/assignment_codeval/github_connect.py
 src/assignment_codeval/install_assignment.py
 src/assignment_codeval/recent_comments.py
 src/assignment_codeval/submissions.py
-src/assignment_codeval/t.html
 src/assignment_codeval/test_template.html
 src/assignment_codeval.egg-info/PKG-INFO
 src/assignment_codeval.egg-info/SOURCES.txt

assignment_codeval-0.0.31/src/assignment_codeval/t.html

@@ -1,2 +0,0 @@
-<a href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==" alt="Red dot">hello</a>
-