assignment-codeval 0.0.30__py3-none-any.whl → 0.0.32__py3-none-any.whl

assignment_codeval/ai_benchmark.py

@@ -75,8 +75,10 @@ def extract_assignment_from_codeval(codeval_path: str) -> tuple[str, str, str]:
     with open(codeval_path, "r", encoding="utf-8") as f:
         content = f.read()
 
-    # Extract content between CRT_HW START and CRT_HW END
-    match = re.search(r"CRT_HW START \S+\n(.*?)CRT_HW END", content, re.DOTALL)
+    # Extract content between ASSIGNMENT START / CRT_HW START and ASSIGNMENT END / CRT_HW END.
+    # The title after the marker may contain spaces (e.g. "CRT_HW START Place Boats")
+    # or be absent entirely, so match the rest of the marker line, not a single token.
+    match = re.search(r"(?:ASSIGNMENT START|CRT_HW START)[^\n]*\n(.*?)(?:ASSIGNMENT END|CRT_HW END)", content, re.DOTALL)
     if match:
         description = match.group(1).strip()
     else:
@@ -223,10 +225,17 @@ def call_openai(model_id: str, prompt: str, api_key: str) -> Optional[str]:
     try:
         client = openai.OpenAI(api_key=api_key)
 
+        # Reasoning models (o1/o3/o4 families, gpt-5) reject `max_tokens` and
+        # require `max_completion_tokens` instead.
+        if model_id.startswith(("o1", "o3", "o4", "gpt-5")):
+            token_kwargs = {"max_completion_tokens": 8192}
+        else:
+            token_kwargs = {"max_tokens": 8192}
+
         response = client.chat.completions.create(
             model=model_id,
             messages=[{"role": "user", "content": prompt}],
-            max_tokens=8192,
+            **token_kwargs,
         )
 
         return response.choices[0].message.content

assignment_codeval/convertMD2Html.py

@@ -1,180 +1,192 @@
-import os
-import re
-import zipfile
-
-import markdown
-
-from assignment_codeval.commons import info, get_config
-
-
-_ANSI_STYLES = {
-    '1': 'font-weight:bold',
-    '4': 'text-decoration:underline',
-    '30': 'color:black', '31': 'color:red', '32': 'color:green',
-    '33': 'color:#a50', '34': 'color:blue', '35': 'color:magenta',
-    '36': 'color:cyan', '37': 'color:white',
-    '40': 'background-color:black', '41': 'background-color:red',
-    '42': 'background-color:green', '43': 'background-color:yellow',
-    '44': 'background-color:blue', '45': 'background-color:magenta',
-    '46': 'background-color:cyan', '47': 'background-color:white',
-}
-
-
-def ansi_to_html(text):
-    """Convert ANSI escape codes in text to HTML span tags."""
-    parts = re.split(r'\x1b\[([\d;]*)m', text)
-    result = []
-    open_spans = 0
-    for i, part in enumerate(parts):
-        if i % 2 == 0:
-            result.append(part)
-        else:
-            codes = part.split(';') if part else ['0']
-            if '0' in codes:
-                result.append('</span>' * open_spans)
-                open_spans = 0
-            else:
-                styles = [_ANSI_STYLES[c] for c in codes if c in _ANSI_STYLES]
-                if styles:
-                    result.append(f'<span style="{";".join(styles)}">')
-                    open_spans += 1
-    result.append('</span>' * open_spans)
-    return ''.join(result)
-
-
-def _read_file_content(filename, spec_dir, zip_archives=None):
-    """Read the content of a file referenced by an IF/OF/EF tag.
-
-    Searches local spec_dir first, then zip archives. Returns content as string.
-    """
-    if spec_dir:
-        filepath = os.path.join(spec_dir, filename)
-        try:
-            with open(filepath, 'r') as f:
-                return f.read()
-        except (FileNotFoundError, OSError):
-            pass
-    if zip_archives:
-        for zf in zip_archives:
-            try:
-                return zf.read(filename).decode('utf-8')
-            except KeyError:
-                continue
-    return None
-
-
-def sampleTestCases(listOfTC, numOfTC, spec_dir=None, zip_archives=None):
-    counter = 0
-    samples = "<pre><code>"
-    for line in listOfTC:
-        if line.startswith('T ', 0, 2):
-            counter = counter + 1
-            if counter > numOfTC:
-                break
-            samples = samples + "\n" + "Command to RUN: " + line[2:]
-        elif line.startswith('IF '):
-            filename = line[3:].strip()
-            content = _read_file_content(filename, spec_dir, zip_archives)
-            if content is not None:
-                samples = samples + "<span style=\"color:green\">" + ansi_to_html(content) + "</span>"
-            else:
-                samples = samples + "<span style=\"color:green\">Input from file: " + filename + "\n</span>"
-        elif line.startswith('IB '):
-            samples = samples + "<span style=\"color:green\">" + line[3:] + "</span>"
-        elif line.startswith('I '):
-            samples = samples + "<span style=\"color:green\">" + line[2:] + "</span>"
-        elif line.startswith('OF '):
-            filename = line[3:].strip()
-            content = _read_file_content(filename, spec_dir, zip_archives)
-            if content is not None:
-                samples = samples + "<span style=\"color:blue\">" + ansi_to_html(content) + "</span>"
-            else:
-                samples = samples + "<span style=\"color:blue\">Expected output from file: " + filename + "\n</span>"
-        elif line.startswith('OB '):
-            samples = samples + "<span style=\"color:blue\">" + line[3:] + "</span>"
-        elif line.startswith('O '):
-            samples = samples + "<span style=\"color:blue\">" + line[2:] + "</span>"
-        elif line.startswith('X '):
-            samples = samples + "Expected Exit Code: " + line[2:]
-        elif line.startswith('EF '):
-            filename = line[3:].strip()
-            content = _read_file_content(filename, spec_dir, zip_archives)
-            if content is not None:
-                samples = samples + "<span style=\"color:Tomato\">" + ansi_to_html(content) + "</span>"
-            else:
-                samples = samples + "<span style=\"color:Tomato\">Expected error from file: " + filename + "\n</span>"
-        elif line.startswith('EB '):
-            samples = samples + "<span style=\"color:Tomato\">" + line[3:] + "</span>"
-        elif line.startswith('E '):
-            samples = samples + "<span style=\"color:Tomato\">" + line[2:] + "</span>"
-        else:
-            continue
-    samples = samples + "</code></pre>"
-    return samples
-
-
-def mdToHtml(file_name, files_resolver=None):
-    with open(file_name, 'r') as f:
-        text = ""
-        examples = []
-        assignment = ""
-        compile_command = ""
-        past_crt_hw = False
-        numOfSampleTC = 1
-        zip_paths = []
-        for line in f:
-            if 'CRT_HW START' in line:
-                assignment_name = line[13:].strip()
-            elif 'CRT_HW END' in line:
-                assignment = text
-                past_crt_hw = True
-            elif line.startswith(('T ', 'I ', 'IB ', 'IF ', 'O ', 'OB ', 'OF ',
-                                  'EF ', 'X ', 'E ', 'EB ')):
-                examples.append(line)
-            elif line.startswith('HT '):
-                break
-            else:
-                if past_crt_hw and line.startswith('C '):
-                    compile_command = line[2:].strip()
-                if past_crt_hw and line.startswith('Z '):
-                    zip_paths.append(line[2:].strip())
-                if 'EXMPLS ' in line:
-                    numOfSampleTC = int(line[7:])
-                text = text + line
-        spec_dir = os.path.dirname(os.path.abspath(file_name))
-
-        # Open zip archives for IF/OF/EF file resolution
-        zip_archives = []
-        try:
-            for zp in zip_paths:
-                full_path = os.path.join(spec_dir, zp)
-                if os.path.exists(full_path):
-                    zip_archives.append(zipfile.ZipFile(full_path, 'r'))
-            samples = sampleTestCases(examples, numOfSampleTC, spec_dir,
-                                      zip_archives)
-        finally:
-            for zf in zip_archives:
-                zf.close()
-
-        assignment = re.sub('EXMPLS [0-9]+', samples, assignment)
-        if compile_command:
-            assignment = assignment.replace('COMPILE', compile_command)
-        # Handle FILE macros in two passes:
-        # 1. FILE inside markdown link syntax: [text](FILE[name]) -> [text](url)
-        assignment = re.sub(r'\]\(FILE\[([^]]+)]\)',
-                            lambda m: f']({files_resolver(m.group(1))})' if files_resolver else f'](FILE[{m.group(1)}])',
-                            assignment)
-        # 2. Standalone FILE macros: FILE[name] -> [name](url)
-        assignment = re.sub(r'FILE\[([^]]+)]',
-                            lambda m: f'[{m.group(1)}]({files_resolver(m.group(1))})' if files_resolver else f'FILE[{m.group(1)}]',
-                            assignment)
-        html = markdown.markdown(assignment,
-                                 extensions=['mdx_better_lists', 'extra'],
-                                 extension_configs={'mdx_better_lists': {'split_paragraph_lists': True}})
-
-    if get_config().dry_run:
-        html_file_name = file_name + '.html'
-        with open(html_file_name, 'w') as f:
-            f.write(html)
-            info(f'HTML preview created in the path : {html_file_name}')
-    return assignment_name, html
+import os
+import re
+import zipfile
+from html import escape as html_escape
+
+import markdown
+
+from assignment_codeval.commons import info, get_config
+
+
+_ANSI_STYLES = {
+    '1': 'font-weight:bold',
+    '4': 'text-decoration:underline',
+    '30': 'color:black', '31': 'color:red', '32': 'color:green',
+    '33': 'color:#a50', '34': 'color:blue', '35': 'color:magenta',
+    '36': 'color:cyan', '37': 'color:white',
+    '40': 'background-color:black', '41': 'background-color:red',
+    '42': 'background-color:green', '43': 'background-color:yellow',
+    '44': 'background-color:blue', '45': 'background-color:magenta',
+    '46': 'background-color:cyan', '47': 'background-color:white',
+}
+
+
+def ansi_to_html(text):
+    """Convert ANSI escape codes in text to HTML span tags.
+
+    The text parts are HTML-escaped so characters like < and & render
+    literally instead of being parsed as markup.
+    """
+    parts = re.split(r'\x1b\[([\d;]*)m', text)
+    result = []
+    open_spans = 0
+    for i, part in enumerate(parts):
+        if i % 2 == 0:
+            result.append(html_escape(part, quote=False))
+        else:
+            codes = part.split(';') if part else ['0']
+            if '0' in codes:
+                result.append('</span>' * open_spans)
+                open_spans = 0
+            else:
+                styles = [_ANSI_STYLES[c] for c in codes if c in _ANSI_STYLES]
+                if styles:
+                    result.append(f'<span style="{";".join(styles)}">')
+                    open_spans += 1
+    result.append('</span>' * open_spans)
+    return ''.join(result)
+
+
+def _read_file_content(filename, spec_dir, zip_archives=None):
+    """Read the content of a file referenced by an IF/OF/EF tag.
+
+    Searches local spec_dir first, then zip archives. Returns content as string.
+    """
+    if spec_dir:
+        filepath = os.path.join(spec_dir, filename)
+        try:
+            with open(filepath, 'r') as f:
+                return f.read()
+        except (FileNotFoundError, OSError):
+            pass
+    if zip_archives:
+        for zf in zip_archives:
+            try:
+                return zf.read(filename).decode('utf-8')
+            except KeyError:
+                continue
+    return None
+
+
+def sampleTestCases(listOfTC, numOfTC, spec_dir=None, zip_archives=None):
+    counter = 0
+    samples = "<pre><code>"
+    esc = lambda s: html_escape(s, quote=False)
+    for line in listOfTC:
+        if line.startswith('T ', 0, 2):
+            counter = counter + 1
+            if counter > numOfTC:
+                break
+            samples = samples + "\n" + "Command to RUN: " + esc(line[2:])
+        elif line.startswith('IF '):
+            filename = line[3:].strip()
+            content = _read_file_content(filename, spec_dir, zip_archives)
+            if content is not None:
+                samples = samples + "<span style=\"color:green\">" + ansi_to_html(content) + "</span>"
+            else:
+                samples = samples + "<span style=\"color:green\">Input from file: " + esc(filename) + "\n</span>"
+        elif line.startswith('IB '):
+            samples = samples + "<span style=\"color:green\">" + esc(line[3:]) + "</span>"
+        elif line.startswith('I '):
+            samples = samples + "<span style=\"color:green\">" + esc(line[2:]) + "</span>"
+        elif line.startswith('OF '):
+            filename = line[3:].strip()
+            content = _read_file_content(filename, spec_dir, zip_archives)
+            if content is not None:
+                samples = samples + "<span style=\"color:blue\">" + ansi_to_html(content) + "</span>"
+            else:
+                samples = samples + "<span style=\"color:blue\">Expected output from file: " + esc(filename) + "\n</span>"
+        elif line.startswith('OB '):
+            samples = samples + "<span style=\"color:blue\">" + esc(line[3:]) + "</span>"
+        elif line.startswith('O '):
+            samples = samples + "<span style=\"color:blue\">" + esc(line[2:]) + "</span>"
+        elif line.startswith('X '):
+            samples = samples + "Expected Exit Code: " + esc(line[2:])
+        elif line.startswith('EF '):
+            filename = line[3:].strip()
+            content = _read_file_content(filename, spec_dir, zip_archives)
+            if content is not None:
+                samples = samples + "<span style=\"color:Tomato\">" + ansi_to_html(content) + "</span>"
+            else:
+                samples = samples + "<span style=\"color:Tomato\">Expected error from file: " + esc(filename) + "\n</span>"
+        elif line.startswith('EB '):
+            samples = samples + "<span style=\"color:Tomato\">" + esc(line[3:]) + "</span>"
+        elif line.startswith('E '):
+            samples = samples + "<span style=\"color:Tomato\">" + esc(line[2:]) + "</span>"
+        else:
+            continue
+    samples = samples + "</code></pre>"
+    return samples
+
+
+def mdToHtml(file_name, files_resolver=None):
+    with open(file_name, 'r') as f:
+        text = ""
+        examples = []
+        assignment = ""
+        compile_command = ""
+        past_crt_hw = False
+        numOfSampleTC = 1
+        zip_paths = []
+        for line in f:
+            if 'ASSIGNMENT START' in line:
+                assignment_name = line[len('ASSIGNMENT START'):].strip()
+            elif 'CRT_HW START' in line:
+                assignment_name = line[len('CRT_HW START'):].strip()
+            elif 'ASSIGNMENT END' in line or 'CRT_HW END' in line:
+                assignment = text
+                past_crt_hw = True
+            elif line.startswith(('T ', 'I ', 'IB ', 'IF ', 'O ', 'OB ', 'OF ',
+                                  'EF ', 'X ', 'E ', 'EB ')):
+                examples.append(line)
+            elif line.startswith('HT '):
+                break
+            else:
+                if past_crt_hw and line.startswith('C '):
+                    compile_command = line[2:].strip()
+                if past_crt_hw and line.startswith('Z '):
+                    zip_paths.append(line[2:].strip())
+                exmpls_match = re.search(r'EXMPLS\s+([0-9]+)', line)
+                if exmpls_match:
+                    numOfSampleTC = int(exmpls_match.group(1))
+                text = text + line
+        spec_dir = os.path.dirname(os.path.abspath(file_name))
+
+        # Open zip archives for IF/OF/EF file resolution
+        zip_archives = []
+        try:
+            for zp in zip_paths:
+                full_path = os.path.join(spec_dir, zp)
+                if os.path.exists(full_path):
+                    zip_archives.append(zipfile.ZipFile(full_path, 'r'))
+            samples = sampleTestCases(examples, numOfSampleTC, spec_dir,
+                                      zip_archives)
+        finally:
+            for zf in zip_archives:
+                zf.close()
+
+        # Use a function replacement so backslash sequences in the sample data
+        # (e.g. "\d" or "\1" in expected output) are inserted literally rather
+        # than interpreted as regex escapes.
+        assignment = re.sub('EXMPLS [0-9]+', lambda m: samples, assignment)
+        if compile_command:
+            assignment = assignment.replace('COMPILE', compile_command)
+        # Handle FILE macros in two passes:
+        # 1. FILE inside markdown link syntax: [text](FILE[name]) -> [text](url)
+        assignment = re.sub(r'\]\(FILE\[([^]]+)]\)',
+                            lambda m: f']({files_resolver(m.group(1))})' if files_resolver else f'](FILE[{m.group(1)}])',
+                            assignment)
+        # 2. Standalone FILE macros: FILE[name] -> [name](url)
+        assignment = re.sub(r'FILE\[([^]]+)]',
+                            lambda m: f'[{m.group(1)}]({files_resolver(m.group(1))})' if files_resolver else f'FILE[{m.group(1)}]',
+                            assignment)
+        html = markdown.markdown(assignment,
+                                 extensions=['mdx_better_lists', 'extra'],
+                                 extension_configs={'mdx_better_lists': {'split_paragraph_lists': True}})
+
+    if get_config().dry_run:
+        html_file_name = file_name + '.html'
+        with open(html_file_name, 'w') as f:
+            f.write(html)
+            info(f'HTML preview created in the path : {html_file_name}')
+    return assignment_name, html

assignment_codeval/create_assignment.py

@@ -38,12 +38,14 @@ def get_assignment_subfolder(course, codeval_folder, assignment_name):
 
 
 def extract_assignment_name(specname):
-    """Extract the assignment name from the CRT_HW START line of a codeval spec file."""
+    """Extract the assignment name from the ASSIGNMENT START (or CRT_HW START) line of a codeval spec file."""
     with open(specname, 'r') as f:
         for line in f:
+            if 'ASSIGNMENT START' in line:
+                return line[len('ASSIGNMENT START'):].strip()
             if 'CRT_HW START' in line:
-                return line[13:].strip()
-    error(f"could not find CRT_HW START in {specname}")
+                return line[len('CRT_HW START'):].strip()
+    error(f"could not find ASSIGNMENT START in {specname}")
     exit(2)
 
 
@@ -53,9 +55,9 @@ def extract_file_macros(specname):
     in_assignment = False
     with open(specname, 'r') as f:
         for line in f:
-            if 'CRT_HW START' in line:
+            if 'ASSIGNMENT START' in line or 'CRT_HW START' in line:
                 in_assignment = True
-            elif 'CRT_HW END' in line:
+            elif 'ASSIGNMENT END' in line or 'CRT_HW END' in line:
                 in_assignment = False
             elif in_assignment:
                 # Find all FILE[filename] patterns in this line
@@ -93,7 +95,7 @@ def files_resolver(filename):
             if not matches:
                 warn(f"could not find {filename} in {z}")
             elif len(matches) > 1:
-                warn("found multiple matches for {filename} in {z}: {matches}")
+                warn(f"found multiple matches for {filename} in {z}: {matches}")
             if matches:
                 zfname = matches[0]
                 with TemporaryDirectory("ce") as tmpdir:
@@ -113,7 +115,9 @@ def files_resolver(filename):
 @click.option("--verbose/--no-verbose", default=False, show_default=True, help="Verbose actions")
 @click.option("--group_name", default="Assignments", show_default=True,
               help="Group name in which assignments needs to be created.")
-def create_assignment(dryrun, verbose, course_name, group_name, specname, extra):
+@click.option("--active/--inactive", default=True, show_default=True,
+              help="Only look at active courses.")
+def create_assignment(dryrun, verbose, course_name, group_name, specname, extra, active):
     """
         Create the assignment in the given course.
     """
@@ -126,7 +130,7 @@ def create_assignment(dryrun, verbose, course_name, group_name, specname, extra)
     (canvas, user) = connect_to_canvas()
 
     try:
-        course = get_course(canvas, course_name)
+        course = get_course(canvas, course_name, is_active=active)
     except Exception as e:
         errorWithException(f'get_course api failed with following error : {e}')
     else:
@@ -150,16 +154,17 @@ def create_assignment(dryrun, verbose, course_name, group_name, specname, extra)
         canvas_folder = get_assignment_subfolder(course, codeval_folder, assign_name_early)
     if extra:
         upload_assignment_files(canvas_folder, extra)
-    # find zipfiles in spec
+    # find zipfiles in spec; resolve them relative to the spec directory so the
+    # command works when run from any working directory
+    spec_dir = os.path.dirname(os.path.abspath(specname))
     with open(specname) as f:
         for line in f:
             if line.startswith("Z "):
-                zipfile = line[2:].strip()
-                if zipfile not in zip_files:
-                    zip_files.append(zipfile)
+                zip_name = os.path.join(spec_dir, line[2:].strip())
+                if zip_name not in zip_files:
+                    zip_files.append(zip_name)
 
     # Extract FILE macros and upload local files that exist
-    spec_dir = os.path.dirname(os.path.abspath(specname))
     file_macros = extract_file_macros(specname)
     for filename in file_macros:
         if filename not in file_dict: