assignment-codeval 0.0.1__py3-none-any.whl

assignment_codeval/canvas_utils.py

@@ -0,0 +1,75 @@
+import datetime
+import sys
+from functools import cache
+from typing import NamedTuple
+from configparser import ConfigParser
+
+import click
+from canvasapi import Canvas
+from canvasapi.current_user import CurrentUser
+
+from assignment_codeval.commons import error, info, errorWithException
+
+CanvasConnection = NamedTuple('CanvasConnection', [('canvas', Canvas), ('user', CurrentUser)])
+
+
+def _check_config(parser, section, key):
+    if section not in parser:
+        error(f"did not find [{section}] section in {parser.config_file}.")
+        sys.exit(1)
+    if key not in parser[section]:
+        error(f"did not find {key} in [{section}] in {parser.config_file}.")
+        sys.exit(1)
+
+
+def connect_to_canvas():
+    parser = ConfigParser()
+    config_file = click.get_app_dir("codeval.ini")
+    parser.read(config_file)
+    parser.config_file = config_file
+
+    for key in ['url', 'token']:
+        _check_config(parser, 'SERVER', key)
+    try:
+        canvas = Canvas(parser['SERVER']['url'], parser['SERVER']['token'])
+        user = canvas.get_current_user()
+        info(f"connected to canvas as {user.name} ({user.id})")
+        return CanvasConnection(canvas, user)
+    except:
+        errorWithException(f"there was a problem accessing canvas.")
+
+
+@cache
+def get_course(canvas, name, is_active=True):
+    ''' find one course based on partial match '''
+    course_list = get_courses(canvas, name, is_active)
+    if len(course_list) == 0:
+        error(f'no courses found that contain {name}. options are:')
+        for c in get_courses(canvas, "", is_active):
+            error(fr"    {c.name}")
+        sys.exit(2)
+    elif len(course_list) > 1:
+        error(f"multiple courses found for {name}: {[c.name for c in course_list]}")
+        for c in course_list:
+            error(f"    {c.name}")
+        sys.exit(2)
+    return course_list[0]
+
+
+def get_courses(canvas, name: str, is_active=True, is_finished=False):
+    ''' find the courses based on partial match '''
+    courses = canvas.get_courses(enrollment_type="teacher")
+    now = datetime.datetime.now(datetime.timezone.utc)
+    course_list = []
+    for c in courses:
+        start = c.start_at_date if hasattr(c, "start_at_date") else now
+        end = c.end_at_date if hasattr(c, "end_at_date") else now
+        if is_active and (start > now or end < now):
+            continue
+        if is_finished and end < now:
+            continue
+        if name in c.name:
+            c.start = start
+            c.end = end
+            course_list.append(c)
+    return course_list

assignment_codeval/cli.py

@@ -0,0 +1,17 @@
+import click
+from assignment_codeval.evaluate import run_evaluation
+from assignment_codeval.github_connect import github_setup_repo
+from assignment_codeval.submissions import download_submissions, upload_submission_comments
+
+
+@click.group()
+def cli():
+    pass
+
+cli.add_command(run_evaluation)
+cli.add_command(download_submissions)
+cli.add_command(upload_submission_comments)
+cli.add_command(github_setup_repo)
+
+if __name__ == "__main__":
+    cli()

assignment_codeval/commons.py

@@ -0,0 +1,51 @@
+from typing import NoReturn
+
+import click
+import time
+import dataclasses
+
+
+@dataclasses.dataclass(init=True, repr=True, frozen=True)
+class _Config():
+    """Global configuration object for the CLI"""
+    show_debug: bool
+    dry_run: bool
+    force: bool
+    copy_tmpdir: bool
+
+    # static global config instance
+    _instance: '_Config' = None
+
+
+def get_config():
+    if _Config._instance is None:
+        _Config._instance = _Config(False, True, False, False)
+    return _Config._instance
+
+
+def set_config(show_debug, dry_run, force, copy_tmpdir):
+    _Config._instance = _Config(show_debug, dry_run, force, copy_tmpdir)
+    return _Config._instance
+
+
+def _now():
+    return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())
+
+
+def debug(message):
+    if get_config().show_debug:
+        click.echo(click.style(f"{_now()} D {message}", fg='magenta'))
+
+def error(message):
+    click.echo(click.style(f"{_now()} E {message}", fg='red'))
+
+def errorWithException(message) -> NoReturn:
+    error(message)
+    raise EnvironmentError(message)
+
+def info(message):
+    click.echo(click.style(f"{_now()} I {message}", fg='blue'))
+
+
+def warn(message):
+    click.echo(click.style(f"{_now()} W {message}", fg='yellow'))

assignment_codeval/evaluate.py

@@ -0,0 +1,676 @@
+#! /usr/bin/python3
+
+import os
+import re
+import subprocess
+import sys
+import traceback
+import threading
+import time
+
+import click
+
+from assignment_codeval.file_utils import unzip
+
+###########################################################
+# Globals
+###########################################################
+
+
+test_args = ""
+cmps = []
+timeout_val = 10
+expected_exit_code = -1
+test_case_count = 0
+test_case_hint = ""
+test_case_total = 0
+num_passed = 0
+num_failed = 0
+is_hidden_testcase = False
+is_verbose = False
+compilelog = []
+
+###########################################################
+# Specification Tags to Function Mapping
+###########################################################
+
+
+def compile_code(compile_command):
+    """Specifies the command to compile the submission code
+
+    Arguments:
+        compile_command: the command to compile the submission code with
+
+    Returns:
+        None
+    """
+    if test_case_count != 0:
+        check_test()
+
+    # Run compile command
+    with open("compilelog", "w") as outfile:
+        compile_popen = subprocess.Popen(
+            compile_command, shell=True, stdout=outfile, stderr=outfile, text=True
+        )
+
+    compile_popen.communicate(compile_popen)
+
+    if compile_popen.returncode:
+        with open("compilelog", "r") as infile:
+            compile_log = infile.readlines()
+
+        # Print head of compile log
+        for line in compile_log[:10]:
+            print(line, end="")
+
+        if len(compile_log) > 10:
+            print("...", end="")
+
+            # Print tail of compile log
+            for line in compile_log[-10:]:
+                print(line, end="")
+
+        sys.exit(1)
+
+
+def check_function(args):
+    """Will be followed by a function name and a list of files to check to ensure that the function
+    is used by one of those files.
+
+    Arguments:
+        function_name: the function name to check files for usage of
+        *files: the files to check for the function name
+
+    Returns:
+        None
+    """
+    check_test()
+    args = args.split()
+    function_name = args[0]
+    files = args[1:]
+
+    # Surpress output
+    function_popen = subprocess.Popen(
+        ["grep", f"[^[:alpha:]]{function_name}[[:space:]]*("] + files,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+
+    function_popen.communicate()
+    if function_popen.returncode:
+        print(f"Not using {function_name} FAILED")
+    else:
+        print(f"Used {function_name} PASSED")
+
+
+def check_not_function(args):
+    """Will be followed by a function name and a list of files to check to ensure that the function
+    is not used by any of those files.
+
+    Arguments:
+        function_name: the funcion name to check files for usage of
+        *files: the files to check for the function name
+
+    Returns:
+        None
+    """
+    check_test()
+    args = args.split()
+    function_name = args[0]
+    files = args[1:]
+
+    # Surpress output
+    function_popen = subprocess.Popen(
+        ["grep", f"[^[:alpha:]]{function_name}[[:space:]]*("] + files,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+
+    function_popen.communicate()
+    if function_popen.returncode:
+        print(f"used{function_name} PASSED")
+    else:
+        print(f"not using {function_name} FAILED")
+
+
+def run_command(command):
+    """Will be followed by a command to run.
+
+    Arguments:
+        command: the command to run
+
+    Returns:
+        None
+    """
+    check_test()
+
+    # Execute without surpressing output
+    print(command)
+    command_popen = subprocess.Popen(command, shell=True)
+    command_popen.communicate()
+
+
+def run_command_noerror(command):
+    """Will be followed by a command to run, evaluation fails if the command exits with an error.
+
+    Arguments:
+        command: the command to run
+
+    Returns:
+        None
+    """
+    check_test()
+
+    # Run as test case
+    global test_case_count
+    test_case_count += 1
+    print(f"Test case count {test_case_count} of {test_case_total}")
+
+    # Execute without surpressing output
+    command_popen = subprocess.Popen(command, shell=True)
+    command_popen.communicate()
+
+    if command_popen.returncode:
+        print("FAILED")
+        for file in os.listdir("evaluationLogs"):
+            with open(file, "r") as infile:
+                file_lines = infile.readlines()
+            # Print entire file
+            print("\n".join(file_lines))
+
+        # Exit entire program with error
+        sys.exit(1)
+    else:
+        print("PASSED")
+
+
+def compare(file1, file2):
+    """Will be followed by two files to compare.
+
+    Arguments:
+        file1: The first file to compare
+        file2: The second file to compare
+
+    Returns:
+        None
+    """
+    cmps.append(file1)
+    cmps.append(file2)
+
+
+def test_case(test_case_command):
+    """Will be followed by the command to run to test the submission.
+
+    Arguments:
+        test_case_command: the command to run the submission
+
+    Returns:
+        None
+    """
+    check_test()
+
+    # Clear hint
+    global hint
+    hint = ""
+
+    # Set new test case command
+    global test_args
+    test_args = test_case_command
+
+    # Increment test cases
+    global test_case_count
+    test_case_count += 1
+
+    # Set test case hidden
+    global test_case_hidden
+    test_case_hidden = False
+
+
+def test_case_hidden(test_case_command):
+    """Will be followed by the command to run to test the submission. Test case is hidden.
+
+    Arguments:
+        test_case_command: the command to run the submission
+
+    Returns:
+        None
+    """
+    check_test()
+
+    # Clear hint
+    global hint
+    hint = ""
+
+    # Set new test case command
+    global test_args
+    test_args = test_case_command
+
+    # Increment test cases
+    global test_case_count
+    test_case_count += 1
+
+    # Set hidden test case
+    global test_case_hidden
+    test_case_hidden = True
+
+
+def supply_input(inputs):
+    """Specifies the input for a test case.
+
+    Arguments:
+        *inputs: inputs to be used for test case
+
+    Returns:
+        None
+    """
+    with open("fileinput", "a") as outfile:
+        outfile.write(inputs)
+
+
+def supply_input_file(input_file):
+    """Specifies the input for a test case read from a file.
+
+    Arguments:
+        input_file: file to get input for test case from
+
+    Returns:
+        None
+    """
+    with open(input_file, "r") as infile:
+        input_lines = infile.readlines()
+
+    with open("fileinput", "a") as outfile:
+        outfile.writelines(input_lines)
+
+
+def check_output(outputs):
+    """Specifies the expected output for a test case.
+
+    Arguments:
+        *outputs: outputs to be used for test case
+
+    Returns:
+        None
+    """
+
+    with open("expectedoutput", "a") as outfile:
+        outfile.write(outputs + "\n")
+
+
+def check_output_file(output_file):
+    """Specifies the expected output for a test case read from a file.
+
+    Arguments:
+        output_file: file to get output for test case from
+
+    Returns:
+        None
+    """
+    with open(output_file, "r") as infile:
+        output_lines = infile.readlines()
+
+    with open("expectedoutput", "a") as outfile:
+        outfile.writelines(output_lines)
+
+
+def check_error(error_output):
+    """Specifies the expected error output for a test case.
+
+    Arguments:
+        error_output: expected error output for a test case
+
+    Returns:
+        None
+    """
+    with open("expectederror", "a") as outfile:
+        outfile.write(error_output)
+
+
+def hint(hints):
+    """Hint
+
+    Arguments:
+        *hints: hints to be associated with test case
+
+    Returns:
+        None
+    """
+    global test_case_hint
+    test_case_hint = hints
+
+
+def timeout(timeout_sec):
+    """Specifies the time limit in seconds for a test case to run. Defaults to 20 seconds.
+
+    Arguments:
+        timeout_sec: time limit in seconds for a test case to run
+
+    Returns:
+        None
+    """
+    global timeout_val
+    timeout_val = float(timeout_sec)
+
+
+def exit_code(test_case_exit_code):
+    """Specifies the expected exit code for a test case. Defaults to zero.
+
+    Arguments:
+        test_case_exit_code: the expected exit code for a test case
+
+    Returns:
+        None
+    """
+    global expected_exit_code
+    expected_exit_code = float(test_case_exit_code)
+
+
+def start_server(timeout_sec, kill_timeout_sec, *server_cmd):
+    """Command containing timeout (wait until server starts), kill timeout (wait to kill the server),
+    and the command to start a server
+
+    Arguments:
+        timeout_sec: timeout in seconds to wait for server to start
+        kill_timeout_sec: timeout in seconds to wait until killing the server
+        server_cmd: command to run to start the server
+
+    Returns:
+        None
+    """
+
+    print(
+        f'Starting server with command: {" ".join(server_cmd)} and sleeping for: {timeout_sec}. Will kill server '
+        f'after {kill_timeout_sec} seconds.'
+    )
+
+    # Send output to compile log in background
+    with open("compilelog", "w") as outfile:
+        server_popen = subprocess.Popen(
+            server_cmd, shell=True, stdout=outfile, stderr=outfile, text=True
+        )
+
+    print(f"Server pid: {server_popen.pid}. Sleeping for {timeout_sec} seconds.")
+    # Block for timeout_sec so that server can start
+    time.sleep(float(timeout_sec))
+
+    # Kill the server after the timeout
+    def kill_server(pid):
+        print(f"Killing {pid}")
+        subprocess.Popen(
+            ["kill", "-9", f"{pid}"], stdout=subprocess.PIPE, stderr=subprocess.PIPE
+        )
+
+    kill_timer = threading.Timer(
+        float(kill_timeout_sec), kill_server, *[server_popen.pid]
+    )
+    kill_timer.daemon = True
+    kill_timer.start()
+
+
+"""
+Here is where the tags are mapped to functions.
+Any tags that are added or changed must be modified here.
+Assume that everything will be passed to these functions as a string, account for
+this in the function itself.
+"""
+tag_func_map = {
+    "C": compile_code,
+    "CF": check_function,
+    "NCF": check_not_function,
+    "CMD": run_command,
+    "TCMD": run_command_noerror,
+    "CMP": compare,
+    "T": test_case,
+    "HT": test_case_hidden,
+    "I": supply_input,
+    "IF": supply_input_file,
+    "O": check_output,
+    "OF": check_output_file,
+    "E": check_error,
+    "HINT": hint,
+    "TO": timeout,
+    "X": exit_code,
+    "SS": start_server,
+}
+
+
+def setup():
+    files = [
+        "compilelog",
+        "difflog",
+        "expectedoutput",
+        "expectederror",
+        "fileinput",
+        "yourerror",
+        "youroutput",
+    ]
+    cleanup()
+    # Create files
+    for file in files:
+        open(file, "w").close()
+
+
+def parse_tags(tags: list[str]):
+    """Given list of strings, parses and executes tags
+
+    Arguments:
+        tags (list[str]): list of tags and arguments to be parsed and executed
+
+    Returns:
+        None
+    """
+    tag_pattern = r"([A-Z]+) (.*)"
+    for tag_line in tags:
+        tag_match = re.match(tag_pattern, tag_line)
+
+        # If line does not match tag format
+        if not tag_match:
+            continue
+
+        tag = tag_match.group(1)
+        args = tag_match.group(2)
+
+        # Execute function based on tag-function mapping
+        try:
+            tag_func_map[tag](args)
+        except KeyError:
+            # Tag was not found in dictionary
+            continue
+        except (TypeError, ValueError):
+            traceback.print_exc()
+            print(f"Invalid arguments for tag {tag} {args}")
+
+
+def parse_diff(diff_lines: list[str]):
+    """Given output from diff command, parse lines into console
+
+    Arguments:
+        parse_diff (list[str]): list of lines as output from diff command
+
+    Returns:
+        None
+    """
+    os.makedirs("evaluationLogs", exist_ok=True)
+    # Directly write into logOfDiff rather than use redirection
+    with open("evaluationLogs/logOfDiff", "w") as outfile:
+        for line in diff_lines:
+            first_word = line.split(" ")[:2]
+            first_character = first_word[0]
+
+            if first_character != "@":
+                # Lines present in your output but not present in expected
+                if first_character == "-" and first_word[:3] == "---":
+                    student_output_file = line[3:-37]
+                    outfile.write(f"Your output file: {student_output_file}")
+
+                # Lines present in expected output but not in yours
+                elif first_character == "+" and first_word[:3] == "+++":
+                    expected_output_file = line[3:-37]
+                    outfile.write(f"Expected output file: {expected_output_file}")
+
+                # Catch rest
+                else:
+                    outfile.write(line)
+
+
+def check_test():
+    global test_args
+    if test_args == "":
+        return
+
+    print(f"Test case {test_case_count} of {test_case_total}")
+    passed = True
+
+    with open("fileinput", "r") as fileinput, open(
+        "youroutput", "w"
+    ) as youroutput, open("yourerror", "w") as yourerror:
+        test_exec = subprocess.Popen(
+            test_args, shell=True, stdin=fileinput, stdout=youroutput, stderr=yourerror
+        )
+
+    # Timeout handling
+    try:
+        test_exec.communicate(timeout=timeout_val)
+
+    except TimeoutError:
+        print(f"Took more than {timeout_val} seconds to run. FAIL")
+        passed = False
+
+    # Difflog handling
+    with open("difflog", "w") as outfile:
+        diff_popen = subprocess.Popen(
+            "diff -U1 -a ./youroutput ./expectedoutput | cat -te | head -22",
+            shell=True,
+            stdout=outfile,
+            stderr=outfile,
+            text=True,
+        )
+        diff_popen.communicate()
+
+    # Append to difflog second time around
+    with open("difflog", "a") as outfile:
+        diff_popen = subprocess.Popen(
+            "diff -U1 -a ./yourerror ./expectederror | cat -te | head -22",
+            shell=True,
+            stdout=outfile,
+            stderr=outfile,
+            text=True,
+        )
+        diff_popen.communicate()
+
+    # Now read all the lines to accumulate both diffs
+    with open("difflog", "r") as infile:
+        diff_lines = infile.readlines()
+
+    if len(diff_lines):
+        passed = False
+        parse_diff(diff_lines)
+
+    # Exit code handling
+    if expected_exit_code != -1 and test_exec.returncode != expected_exit_code:
+        passed = False
+        print(
+            f"    Exit Code failure: expected {expected_exit_code} got {test_exec.returncode}"
+        )
+
+    # Compare files handling, do not surpress output
+    for files in cmps:
+        cmd_popen = subprocess.Popen(["cmp", files])
+        cmd_popen.communicate()
+        if cmd_popen.returncode:
+            passed = False
+            break
+
+    test_args = ""
+
+    # Pass fail handling
+    if passed:
+        global num_passed
+        num_passed += 1
+        print("Passed")
+    else:
+        global num_failed
+        num_failed += 1
+        print("FAILED")
+
+        # Hidden test case handling
+        if test_case_hidden:
+            print("    Test Case is Hidden")
+            if hint:
+                print(f"HINT: {hint}")
+        else:
+            if hint:
+                print(f"HINT: {hint}")
+
+            # Cleanup
+            print(f"    Command ran: {test_args}")
+            for file in os.listdir("evaluationLogs"):
+                with open("evaluationLogs/" + file, "r") as infile:
+                    file_lines = infile.readlines()
+
+                # Print entire file
+                print("".join(file_lines))
+
+        cleanup()
+
+        # Exit program after failed test case
+        sys.exit(2)
+
+    # reinitialize test variables and files here
+    setup()
+
+
+def cleanup():
+    files = [
+        "compilelog",
+        "difflog",
+        "expectedoutput",
+        "expectederror",
+        "fileinput",
+        "yourerror",
+        "youroutput",
+    ]
+
+    if os.path.exists("evaluationLogs"):
+        if os.path.exists("evaluationLogs/logOfDiff"):
+            os.remove("evaluationLogs/logOfDiff")
+            os.rmdir("evaluationLogs")
+
+    for name in files:
+        if os.path.exists(name):
+            os.remove(name)
+
+
+@click.command()
+@click.argument("codeval_file", type=click.Path(exists=True))
+def run_evaluation(codeval_file):
+    start_time_seconds = time.time()
+
+    setup()
+
+    # Count test case total
+    global test_case_total
+    with open(codeval_file, "r") as infile:
+        testcases = infile.readlines()
+        for testcase in testcases:
+            parts = testcase.split(" ", 1)
+            tag = parts[0]
+            if tag == "T" or tag == "HT":
+                test_case_total += 1
+            elif tag == "Z":
+                filename=parts[1].strip()
+                file = os.path.join(os.path.dirname(codeval_file), filename) if filename.startswith("/") else filename
+                unzip(file, ".")
+
+    # Read testcases
+    with open(codeval_file, "r") as infile:
+        testcases = infile.readlines()
+        parse_tags(testcases)
+
+    check_test()
+
+    # cleanup
+    cleanup()
+
+    end_time_seconds = time.time()
+    print(f"took {end_time_seconds - start_time_seconds} seconds")

assignment_codeval/file_utils.py

@@ -0,0 +1,62 @@
+import os
+import shutil
+import sys
+import subprocess
+
+import requests
+import zipfile
+from assignment_codeval.commons import debug, error
+
+
+def download_attachment(directory, attachment):
+    curPath = os.getcwd()
+    os.chdir(os.path.join(curPath, directory))
+
+    fname = attachment['display_name']
+    prefix = os.path.splitext(fname)[0]
+    suffix = os.path.splitext(fname)[1]
+    durl = attachment['url']
+    with requests.get(durl) as response:
+        if response.status_code != 200:
+            error(f'error {response.status_code} fetching {durl}')
+        with open(f"{prefix}{suffix}", "wb") as fd:
+            for chunk in response.iter_content():
+                fd.write(chunk)
+
+    os.chdir(curPath)
+    return os.path.join(directory, fname)
+
+
+def unzip(filepath, dir, delete=False):
+    with zipfile.ZipFile(filepath) as file:
+        for zi in file.infolist():
+            file.extract(zi.filename, path=dir)
+            debug(f"extracting {zi.filename}")
+            fname = os.path.join(dir, zi.filename)
+            s = os.stat(fname)
+            # the user executable bit is set
+            perms = (s.st_mode | (zi.external_attr >> 16)) & 0o777
+            os.chmod(fname, perms)
+
+        debug(f"{filepath} extracted to {dir}.")
+    if delete:
+        os.remove(filepath)
+        debug(f"{filepath} deleted.")
+
+
+def set_acls(temp_dir):
+    """Set ACLs for the temporary directory"""
+    if sys.platform == 'darwin':
+        subprocess.call(["chmod", "-R", "o+rwx", temp_dir])
+    else:
+        subprocess.call(["setfacl", "-d", "-m", "o::rwx", temp_dir])
+
+
+def copy_files_to_submission_dir(temp_fixed, temp_dir):
+    shutil.copytree(temp_fixed, temp_dir, dirs_exist_ok=True)
+    shutil.copy("../../evaluate.sh", f"{temp_dir}/evaluate.sh")
+    shutil.copy("evaluate.py", f"{temp_dir}/evaluate.py")
+    shutil.copy("../../runvalgrind.sh", f"{temp_dir}/runvalgrind.sh")
+    shutil.copy("../../parsediff", f"{temp_dir}/parsediff")
+    shutil.copy("../../parsevalgrind", f"{temp_dir}/parsevalgrind")
+    shutil.copy("../../checksql.sh", f"{temp_dir}/checksql.sh")

assignment_codeval/github_connect.py

@@ -0,0 +1,63 @@
+import os.path
+import subprocess
+from configparser import ConfigParser
+
+import click
+
+from assignment_codeval.canvas_utils import get_course, connect_to_canvas
+from assignment_codeval.commons import error, debug
+
+
+@click.command()
+@click.argument("course_name", metavar="COURSE")
+@click.option("--github-field", help="GitHub field name in canvas profile", default="github", show_default=True)
+def github_setup_repo(course_name, github_field):
+    """
+    Connect to a GitHub repository for a given course and assignment.
+
+    COURSE can be a unique substring of the actual course name.
+
+    GITHUB_PREFIX is the prefix of the GitHub repository name, it should have a form similar to:
+
+    git@github.com:<gh_classroom_account>/<gh_classroom_assignment>
+    """
+    canvas, user = connect_to_canvas()
+    parser = ConfigParser()
+    config_file = click.get_app_dir("codeval.ini")
+    parser.read(config_file)
+    parser.config_file = config_file
+    course = get_course(canvas, course_name, True)
+
+    gh_key = course.name.replace(":", "").replace("=", "")
+    if 'GITHUB' not in parser or gh_key not in parser['GITHUB']:
+        error(f"did not find mapping for {gh_key} in GITHUB section of {parser.config_file}.")
+        return
+
+    gh_repo_prefix = parser['GITHUB'][gh_key]
+
+    users = course.get_users(include=["enrollments"])
+    for user in users:
+        ssid = user.login_id
+        os.makedirs(ssid, exist_ok=True)
+        with open(f"{ssid}/gh_result.txt", "w") as fd:
+            profile = user.get_profile(include=["links", "link"])
+            gh_links = None
+            if 'links' in profile:
+                gh_links = [m['url'] for m in profile['links'] if m['title'].lower() == github_field.lower()]
+            if not gh_links:
+                print(f"no {github_field} link found in canvas profile", file=fd)
+                continue
+            if len(gh_links) != 1:
+                print(f"multiple {github_field} links found in canvas profile", file=fd)
+                continue
+            gh_url = gh_links[0]
+            gh_id = gh_url.rstrip('/').rsplit('/', 1)[-1]
+            repo_url = f"{gh_repo_prefix}-{gh_id}.git"
+            if os.path.exists(f'{ssid}/repo'):
+                print(f"pulling {repo_url}", file=fd)
+                rc = subprocess.run(['git', 'pull'], cwd=f'{ssid}/repo', stdout=fd, stderr=subprocess.STDOUT)
+            else:
+                print(f"cloning {repo_url}", file=fd)
+                rc = subprocess.run(['git', 'clone', repo_url, f'{ssid}/repo'], stdout=fd, stderr=subprocess.STDOUT)
+            if rc.returncode != 0:
+                error(f"error {rc.returncode} connecting to github repo for {ssid} using {repo_url}")

assignment_codeval/submissions.py

@@ -0,0 +1,156 @@
+import os
+import re
+import sys
+from datetime import datetime, timezone
+from functools import cache
+
+import click
+import requests
+
+from assignment_codeval.canvas_utils import connect_to_canvas, get_course
+from assignment_codeval.commons import debug, error, info, warn
+
+
+@click.command()
+@click.argument("submissions_dir", metavar="SUBMISSIONS_DIR")
+@click.option("--codeval-prefix", help="prefix for codeval comments", default="codeval: ", show_default=True)
+def upload_submission_comments(submissions_dir, codeval_prefix):
+    """
+    Upload comments for submissions from a directory.
+
+    the submissions_dir specifies a directory that has comments to upload stored as:
+
+    COURSE/ASSIGNMENT/STUDENT_ID/comments.txt
+
+    if the file comments.txt.sent exts, the comment has already been uploaded and will be skipped.
+    """
+    (canvas, user) = connect_to_canvas()
+    clean_submissions_dir = submissions_dir.rstrip('/')
+    for dirpath, dirnames, filenames in os.walk(clean_submissions_dir):
+        match = re.match(fr'^{clean_submissions_dir}/([^/]+)/([^/]+)/([^/]+)$', dirpath)
+        if match:
+            course_name = match.group(1)
+            assignment_name = match.group(2)
+            student_id = match.group(3)
+            if "comments.txt" in filenames:
+                if "comments.txt.sent" in filenames:
+                    info(f"skipping already uploaded comments for {student_id} in {course_name}: {assignment_name}")
+                else:
+                    info(f"uploading comments for {student_id} in {course_name}: {assignment_name}")
+                    course = get_course(canvas, course_name)
+                    assignment = get_assignment(course, assignment_name)
+                    with open(f"{dirpath}/comments.txt", "r") as fd:
+                        comment = fd.read()
+                        # nulls seem to be very problematic for canvas
+                        comment = comment.replace("\0", "\\0")
+                        comment.strip()
+                        submission = get_submissions_by_id(assignment).get(student_id)
+                        if submission:
+                            submission.edit(comment={'text_comment': f'{codeval_prefix} {comment}'})
+                        else:
+                            warn(f"no submission found for {student_id} in {course_name}: {assignment_name}")
+                    with open(f"{dirpath}/comments.txt.sent", "w") as fd:
+                        fd.write(datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'))
+
+
+@click.command()
+@click.argument("course_name", metavar="COURSE")
+@click.argument("assignment_name", metavar="ASSIGNMENT")
+@click.option("--target-dir", help="directory to download submissions to", default='./submissions', show_default=True)
+@click.option("--only-uncommented", is_flag=True, help="only download submissions without codeval comments since last submission")
+@click.option("--codeval-prefix", help="prefix for codeval comments", default="codeval: ", show_default=True)
+@click.option("--include-empty", is_flag=True, help="include empty submissions")
+def download_submissions(course_name, assignment_name, target_dir, only_uncommented, codeval_prefix, include_empty):
+    """
+    Download submissions for a given assignment in a course from Canvas.
+
+    the COURSE and ASSIGNMENT arguments can be partial names.
+    """
+    (canvas, user) = connect_to_canvas()
+
+    course = get_course(canvas, course_name)
+    assignment = get_assignment(course, assignment_name)
+    submission_dir = os.path.join(target_dir, course.name, assignment.name)
+    os.makedirs(submission_dir, exist_ok=True)
+
+    for submission in assignment.get_submissions(include=["submission_comments", "user"]):
+        if not submission.attempt and not include_empty:
+            continue
+        submission_comments = [c['created_at'] for c in submission.submission_comments if
+                               'comment' in c and c['comment'].startswith(codeval_prefix)]
+        submission_comments.sort()
+        if submission_comments:
+            last_comment_date = submission_comments[-1]
+        else:
+            last_comment_date = None
+        if only_uncommented and last_comment_date and submission.submitted_at <= last_comment_date:
+            continue
+
+        student_id = submission.user['login_id']
+        student_submission_dir = os.path.join(submission_dir, student_id)
+        os.makedirs(student_submission_dir, exist_ok=True)
+
+        metapath = os.path.join(student_submission_dir, "metadata.txt")
+        with open(metapath, "w") as fd:
+            print(f"""id={student_id}
+course={course.name}
+assignment={assignment.name}
+attempt={submission.attempt}
+late={submission.late}
+date={submission.submitted_at}
+last_comment={last_comment_date}""", file=fd)
+        body = submission.body
+        if body:
+            filepath = os.path.join(student_submission_dir, "content.txt")
+            with open(filepath, "w") as fd:
+                fd.write(body)
+            debug(f"Downloaded content for student {student_id} to {filepath}")
+
+        if hasattr(submission, "attachment"):
+            attachment = submission.attachment
+            fname = attachment['display_name']
+            prefix = os.path.splitext(fname)[0]
+            suffix = os.path.splitext(fname)[1]
+            durl = attachment['url']
+            filepath = os.path.join(student_submission_dir, f"{prefix}{suffix}")
+
+            with requests.get(durl) as response:
+                if response.status_code != 200:
+                    error(f'error {response.status_code} fetching {durl}')
+                with open(filepath, "wb") as fd:
+                    for chunk in response.iter_content():
+                        fd.write(chunk)
+
+            debug(f"Downloaded submission for student {student_id} to {filepath}")
+
+    return submission_dir
+
+
+@cache
+def get_assignment(course, assignment_name):
+    assignments = [a for a in course.get_assignments() if assignment_name.lower() in a.name.lower()]
+    if len(assignments) == 0:
+        error(f'no assignments found that contain {assignment_name}. options are:')
+        for a in course.get_assignments():
+            error(fr"    {a.name}")
+        sys.exit(2)
+    elif len(assignments) > 1:
+        strict_name_assignments = [a for a in assignments if a.name == assignment_name]
+        if len(strict_name_assignments) == 1:
+            assignments = strict_name_assignments
+        else:
+            error(f"multiple assignments found for {assignment_name}: {[a.name for a in assignments]}")
+            for a in assignments:
+                error(f"    {a.name}")
+            sys.exit(2)
+    assignment = assignments[0]
+    return assignment
+
+
+@cache
+def get_submissions_by_id(assignment):
+    submissions_by_id = {}
+    for submission in assignment.get_submissions(include=["user"]):
+        student_id = submission.user['login_id']
+        submissions_by_id[student_id] = submission
+    return submissions_by_id

assignment_codeval-0.0.1.dist-info/METADATA

@@ -0,0 +1,241 @@
+Metadata-Version: 2.4
+Name: assignment-codeval
+Version: 0.0.1
+Summary: CodEval for evaluating programming assignments
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+Requires-Dist: canvasapi==3.3.0
+Requires-Dist: certifi==2021.10.8
+Requires-Dist: charset-normalizer==2.0.9
+Requires-Dist: click==8.2.1
+Requires-Dist: configparser==5.2.0
+Requires-Dist: idna==3.3
+Requires-Dist: pytz==2021.3
+Requires-Dist: requests==2.27.0
+Requires-Dist: urllib3==1.26.7
+Requires-Dist: pymongo==4.3.3
+Requires-Dist: markdown==3.4.1
+
+# CodEval
+
+Currently CodEval has 3 main components:
+## 1. Test Simple I/O Programming Assignments on Canvas
+### codeval.ini contents
+```
+[SERVER]
+url=<canvas API>
+token=<canvas token>
+[RUN]
+precommand=
+command=
+```
+
+Refer to a sample codeval.ini file [here](samples/codeval.ini)
+
+### Command to run:  
+`python3 codeval.py grade-submissions <a unique part of course name> [FLAGS]`
+Example:  
+If the course name on Canvas is CS 149 - Operating Systems, the command can be:  
+`python3 codeval.py CS\ 149`  
+or  
+`python3 codeval.py "Operating Systems"`  
+Use a part of the course name that can uniquely identify the course on Canvas.
+
+### Flags
+- **--dry-run/--no-dry-run** (Optional)
+  - Default: --dry-run
+  - Do not update the results on Canvas. Print the results to the terminal instead.
+- **--verbose/--no-verbose** (Optional)
+  - Default: --no-verbose
+  - Show detailed logs
+- **--force/--no-force** (Optional)
+  - Default: --no-force
+  - Grade submissions even if already graded
+- **--copytmpdir/--no-copytmpdir** (Optional)
+  - Default: --no-copytmpdir
+  - Copy temporary directory content to current directory for debugging
+
+### Specification Tags
+Tags used in a spec file (\<course name>.codeval)
+
+| Tag | Meaning | Function |
+|---|---|---|
+| C | Compile Code | Specifies the command to compile the submission code |
+| CTO | Compile Timeout | Timeout in seconds for the compile command to run |
+| RUN | Run Script | Specifies the script to use to evaluate the specification file. Defaults to evaluate.sh. |
+| Z | Download Zip | Will be followed by zip files to download from Canvas to use when running the test cases. |
+| CF | Check Function | Will be followed by a function name and a list of files to check to ensure that the function is used by one of those files. |
+| CMD/TCMD | Run Command | Will be followed by a command to run. The TCMD will cause the evaluation to fail if the command exits with an error. |
+| CMP | Compare | Will be followed by two files to compare. |
+| T/HT | Test Case | Will be followed by the command to run to test the submission. |
+| I/IF | Supply Input | Specifies the input for a test case. The IF version will read the input from a file. |
+| O/OF | Check Output | Specifies the expected output for a test case. The OF version will read from a file. |
+| E | Check Error | Specifies the expected error output for a test case. |
+| TO | Timeout | Specifies the time limit in seconds for a test case to run. Defaults to 20 seconds. |
+| X | Exit Code | Specifies the expected exit code for a test case. Defaults to zero. |
+| SS | Start Server | Command containing timeout (wait until server starts), kill timeout (wait to kill the server), and the command to start a server |
+
+Refer to a sample spec file [here](samples/assignment-name.codeval)
+
+## 2. Test Distributed Programming Assignments
+### (or complex non I/O programs)
+### codeval.ini contents
+```
+[SERVER]
+url=<canvas API>
+token=<canvas token>
+[RUN]
+precommand=
+command=
+dist_command=
+host_ip=
+[MONGO]
+url=
+db=
+```
+
+Refer to a sample codeval.ini file [here](samples/codeval.ini)
+
+### Command to run
+is the same as the [command in #1](#command-to-run):  
+`python3 codeval.py grade-submissions <a unique part of course name> [FLAGS]`
+
+### Distributed Specification Tags  
+
+| Tag | Meaning | Function |
+|---|---|---|
+| --DT-- | Distributed Tests Begin | Marks the beginning of distributed tests. Is used to determine if the spec file has distributed tests |
+| GTO | Global timeout | A total timeout for all distributed tests, for each of homogenous and heterogenous tests. Homogenous tests = GTO value. Heterogenous tests = 2 * GTO value |
+| PORTS | Exposed ports count | Maximum number of ports needed to expose per docker container |
+| ECMD/ECMDT SYNC/ASYNC | External Command | Command that runs in the a controller container, emulating a host machine. ECMDT: Evaluation fails if command returns an error. SYNC: CodEval waits for command to execute or fail. ASYNC: CodEval doesn't wait for command to execute, failure is checked if ECMDT |
+| DTC $int [HOM] [HET] | Distributed Test Config Group | Signifies the start of a new group of Distributed tests. Replace $int with the number of containers that needs to be started for the test group. HOM denotes homogenous tests, i.e., user's own submissions will be executed in the contianers. HET denotes heterogenous tests, i.e., a combination of $int - 1 other users' and current user's submissions will be executed in the containers. Can enter either HOM or HET or both |
+| ICMD/ICMDT SYNC/ASYNC */n1,n2,n3... | Internal Command | Command that runs in each of the containers. ICMDT: Evaluation fails if command returns an error. SYNC: wait for command to execute or fail. ASYNC: Don't wait for command to execute, failure is checked if ICMDT *: run command in all the containers. n1,n2..nx: Run command in containers indexed n1,n2..nx only. Containers follow zero-based indexing  |
+| TESTCMD | Test Command | Command run on the host machine to validate the submission(s) |
+| --DTCLEAN-- | Cleanup Commands | Commands to execute after the tests have completed or failed. Can contain only ECMD or ECMDT |  
+
+### Special placeholders in commands
+| Placeholder | Usage |
+| --- | --- |
+| TEMP_DIR | used in ECMD/ECMDT to be replaced by the temporary directory generated by CodEval during execution |
+| HOST_IP | used in ECMD/ECMDT/ICMD/ICMDT to be replaced by the host's IP specified in codeval.ini |
+| USERNAME | used in ICMD/ICMDT to be replaced by the user's username whose submission is being evaluated |
+| PORT_$int | used in ICMD/ICMDT to be replaced by a port number assigned to the running docker continer. $int needs to be < PORT value in the specification |
+
+Refer to a sample spec file [here](samples/assignment-name.codeval)
+
+### Notes
+- The config file `codeval.ini` needs to contain the extra entries only if the tag `--DT--` exists in the specification file
+- Distributed tests need a running mongodb service to persists the progress of students running heterogenous tests
+
+
+## 3. Test SQL Assignments
+### codeval.ini contents
+```
+[SERVER]
+url=<canvas API>
+token=<canvas token>
+[RUN]
+precommand=
+command=
+dist_command=
+host_ip=
+sql_command=
+```
+
+Refer to a sample codeval.ini file [here](SQL/samples/codeval.ini)
+
+### Command to run
+is the same as the [command in #1](#command-to-run):  
+`python3 codeval.py grade-submissions <a unique part of course name> [FLAGS]`
+
+### SQL Specification Tags  
+
+| Tag              | Meaning                 | Function                                                                                     |
+|------------------|-------------------------|----------------------------------------------------------------------------------------------|
+| --SQL--          | SQL Tests Begin         | Marks the beginning of SQL tests. Is used to determine if the spec file has SQL based tests  |
+| INSERT           | Insert rows in DB       | Insert rows in the SQL database using files/ individual insert queries.                      |
+| CONDITIONPRESENT | Check condition in file | Validate submission files for a required condition to be present in submissions.             |
+| SCHEMACHECK      | External Command        | Validate submission files for database related checks like constraints.                      |
+| TSQL             | SQL Test                | Marks the SQL test, take input as a file or individual query and run it on submission files. |
+
+Refer to a sample spec file [here](SQL/samples/ASSIGNMENT:CREATE.codeval)
+
+### Notes
+- The config file `codeval.ini` needs to contain the extra entries only if the tag `--SQL--` exists in the specification file
+- SQL tests need a separate container image to run SQL tests in MYSQL.
+
+
+## 4. Create an assignment on Canvas
+
+### Command to create the assignment:
+**Syntax:** `python3 codeval.py create-assignment <course_name> <specification_file> [ --dry-run/--no-dry-run ] [ --verbose/--no-verbose ] [ --group_name ]`  
+**Example:** `python3 codeval.py create-assignment "Practice1" 'a_big_bag_of_strings.txt' --no-dry-run --verbose --group_name "exam 2"`
+
+### Command to grade the assignment:
+**Syntax:** `python3 codeval.py grade-submissions <course_name> [ --dry-run/--no-dry-run ] [ --verbose/--no-verbose ] [ --force/--no-force][--copytmpdir/--no-copytmpdir]`  
+**Example:** `python3 codeval.py grade-submissions "Practice1" --no-dry-run --force --verbose`
+
+**New tags introduced are :** 
+
+CRT_HW START <Assignment_name>
+
+CRT_HW END
+
+DISCSN_URL
+
+EXMPLS <no_of_test_cases>
+
+URL_OF_HW "file_name"
+
+### MODIFICATIONS REQUIRED IN THE SPECIFICATION FILE.
+1) Start the specification file with the tag CRT_HW START followed by a space followed by the name of assignment.
+	```  For ex: CRT_HW START Hello World```
+2) The following lines after the first line will contain the description of the assignment in Markdown format.
+3) The description ends with the last line containing just the tag CRT_HW END .
+	``` For ex: CRT_HW END ```
+4) After this tag, the content for grading the submission begins.
+
+	Addition of the Discussion Topic in the assignment description.
+	1) Insert the tag DISCUSSION_LINK wherever you want the corresponding discussion topic's link to appear.
+		```For ex: To access the discussion topic for this assignment you go here DISCUSSION_LINK```
+
+	#### Addition of sample examples in the assignment description.
+	1) Insert the tag EXMPLS followed by single space followed by the value. 
+	   Here value is the number of test cases to be displayed as sample examples. 
+	   At maximum it will print all the non hidden test cases.
+	   For ex: EXMPLS 5
+	#### Addition of the links to the files uploaded in the Codeval folder in the assignment description.
+	1) In order to add hyperlink to a file the markdown format is as follows:
+	   [file_name_to_be_displayed](Url_of_the_file)
+	   Here in the parenthesis where the Url is required,insert the tag
+	   URL_OF_HW followed by space followed by the file name of the file required to be linked in double quotes.
+	   For ex: URL_OF_HW "file name.extension"
+	   Note: The file should be present in the Codeval folder.
+	   
+### UPLOAD THE REQUIRED FILES IN CODEVAL FOLDER IN FILES SECTION.
+1) Create a folder called `assignmentFiles` which should conatin all the necessary files including
+	the specification file.
+	   
+### EXAMPLE OF THE SPECIFICATION FILE.	
+	
+	CRT_HW START Bag Of Strings
+	# Description
+	## Problem Statement
+	- This Is An Example For The Description Of The Assignment In Markdown.
+	- To Download The File [Hello_World](URL_OF_HW "Helloworld.Txt")
+
+	## Sample Examples
+	EXMPLS 3
+
+	## Discussion Topic
+	Here Is The Link To The Discussion Topic: DISCSN_URL
+
+	### Rubric 
+	| Cases | Points|
+	| ----- |----- |
+	| Base Points | 50 |
+
+	CRT_HW END  
+
+	C cc -o bigbag --std=gnu11 bigbag.c
+

assignment_codeval-0.0.1.dist-info/RECORD

@@ -0,0 +1,13 @@
+assignment_codeval/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+assignment_codeval/canvas_utils.py,sha256=LpNOYznMagA6t9_Dm8-liSgcDQE2BZpR_RdCazS2nco,2536
+assignment_codeval/cli.py,sha256=lfToqPg6s4txTKvFHhLTiSgp5QozgHtjojjmz22MiFc,448
+assignment_codeval/commons.py,sha256=iVCayl1AVeoJBRkF2ye2037qoM457eEdRd54F51ePlM,1209
+assignment_codeval/evaluate.py,sha256=CptkqpZXiy_mVLN_mt5pgGhEf2En7t9RwBVwSeQvyBc,17024
+assignment_codeval/file_utils.py,sha256=9wHuZACuU-1oMQqmwfFbN95WExVO2UPui7m_y9g9my8,2081
+assignment_codeval/github_connect.py,sha256=Xhgee4b-a-3iVpy0TKmXalJdhOW3bkeWnSgaAn9LN_U,2688
+assignment_codeval/submissions.py,sha256=_I1ylksbzrAfTfCT3KxjSMivmHdPE40SrABOykOTEN0,6968
+assignment_codeval-0.0.1.dist-info/METADATA,sha256=n0knrTyC_QfkEsHTE-7mLUOE4Ty7HWkC12bysfZ5COk,11328
+assignment_codeval-0.0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+assignment_codeval-0.0.1.dist-info/entry_points.txt,sha256=Pk0eAhsXBpCsgkv6HYLSHTRf5x_zUf-Lg8KK9KsK2B8,66
+assignment_codeval-0.0.1.dist-info/top_level.txt,sha256=a_P8W2IWjAELxKlTV2t4rLwWVoTxElxUMvWgWjR_1n4,19
+assignment_codeval-0.0.1.dist-info/RECORD,,

assignment_codeval-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

assignment_codeval-0.0.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+assignment-codeval = assignment_codeval.cli:cli

assignment_codeval-0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+assignment_codeval