assignment-codeval 0.0.30__tar.gz → 0.0.32__tar.gz

assignment_codeval-0.0.32/PKG-INFO

@@ -0,0 +1,242 @@
+Metadata-Version: 2.4
+Name: assignment-codeval
+Version: 0.0.32
+Summary: CodEval for evaluating programming assignments
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: canvasapi==3.3.0
+Requires-Dist: click==8.2.1
+Requires-Dist: configparser==5.2.0
+Requires-Dist: pytz==2021.3
+Requires-Dist: requests>=2.28.0
+Requires-Dist: pymongo==4.3.3
+Requires-Dist: markdown==3.4.1
+Requires-Dist: mdx-better-lists>=1.0.0
+Requires-Dist: anthropic>=0.39.0
+Requires-Dist: openai>=1.0.0
+Requires-Dist: google-generativeai>=0.8.0
+Provides-Extra: test
+Requires-Dist: pytest>=7.0; extra == "test"
+Requires-Dist: pytest-cov; extra == "test"
+
+# CodEval
+
+[![CI](https://github.com/SJSU-CMPE-195/group-project-team-29/actions/workflows/test.yml/badge.svg)](https://github.com/SJSU-CMPE-195/group-project-team-29/actions/workflows/test.yml)
+[![codecov](https://codecov.io/gh/SJSU-CMPE-195/group-project-team-29/branch/main/graph/badge.svg)](https://codecov.io/gh/SJSU-CMPE-195/group-project-team-29)
+[![PyPI](https://img.shields.io/pypi/v/assignment-codeval)](https://pypi.org/project/assignment-codeval/)
+
+A Python utility to download student submissions to programming assignments from Canvas and GitHub and evaluate them using codeval scripts.
+
+## Team
+- Sabira Abdolcader (sabdolc)
+- Chelsie Chen (cChe1z)
+- Aisha Syed (aisha-syed)
+- Zarah Taufique (zarahtau)
+
+## Prerequisites
+- Python 3.x
+- Docker (for running evaluations in containers)
+- A Canvas account with API access
+- A GitHub account (for GitHub-based assignments)
+- Optional AI provider packages: `anthropic`, `openai`, `google-generativeai`
+
+## Installation
+
+```bash
+# Install locally for development
+pip install -e .
+
+# Or install from PyPI
+pip install assignment-codeval
+
+# Install with AI provider support
+pip install assignment-codeval[ai]
+```
+
+## Configuration
+
+Create a `codeval.ini` file with your Canvas and run settings:
+
+```ini
+[SERVER]
+url=<canvas API>
+token=<canvas token>
+[RUN]
+precommand=
+command=
+```
+
+For distributed assignments, add:
+```ini
+[RUN]
+dist_command=
+host_ip=
+[MONGO]
+url=
+db=
+```
+
+For AI benchmarking, add:
+```ini
+[AI]
+anthropic_key=sk-ant-...
+openai_key=sk-...
+google_key=...
+```
+
+AI keys can also be set via environment variables: `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GOOGLE_API_KEY`.
+
+Refer to a sample config file [here](samples/codeval.ini)
+
+## Running the Application
+
+```bash
+# List all available commands
+assignment-codeval --help
+
+# Download submissions from Canvas/GitHub
+assignment-codeval download-submissions <course_name>
+
+# Evaluate downloaded submissions
+assignment-codeval evaluate-submissions <course_name>
+
+# Upload grading comments back to Canvas
+assignment-codeval upload-submission-comments <course_name>
+
+# Create an assignment on Canvas
+assignment-codeval create-assignment <course_name> <specification_file>
+
+# Check which submissions are missing grading
+assignment-codeval check-grading <course_name>
+
+# List recent codeval comments
+assignment-codeval recent-comments <course_name>
+
+# Test an assignment with AI models
+assignment-codeval test-with-ai <codeval_file> [OPTIONS]
+```
+
+## Usage
+
+### Codeval Specification Files
+
+Codeval files (`.codeval` extension) define how to build, run, and test student submissions.
+
+#### Assignment Description Tags
+- `ASSIGNMENT START <Assignment_name>` — begins the Canvas assignment description in markdown (`CRT_HW START` also accepted, but deprecated)
+- `ASSIGNMENT END` — ends the assignment description (`CRT_HW END` also accepted, but deprecated)
+
+#### Specification Tags
+
+| Tag | Meaning | Function |
+|---|---|---|
+| C | Compile Code | Specifies the command to compile the submission code |
+| CTO | Compile Timeout | Timeout in seconds for the whole evaluation (compile **and** all test cases) to run |
+| T/HT | Test Case | Command to run to test the submission (HT = hidden test) |
+| I/IB/IF | Supply Input | Input for a test case. I adds a newline, IB does not, IF reads from a file |
+| O/OB/OF | Check Output | Expected output. O adds a newline, OB does not, OF reads from a file |
+| E/EB | Check Error | Expected error output. E adds a newline, EB does not |
+| TO | Timeout | Time limit in seconds for a test case (default: 10) |
+| X | Exit Code | Expected exit code for a test case (not checked unless X is specified) |
+| TEMP | Temp File | Registers a file to be deleted before and after the next test run |
+| CF | Check Function | Checks that a function is used in the submission |
+| NCF | Check Not Function | Checks that a function is **not** used |
+| CMD/TCMD | Run Command | Runs a command. TCMD fails evaluation if the command errors |
+| PRINT | Print Label | Prints a label/message to stdout |
+| CMP | Compare | Compares two files |
+| Z | Download Zip | Zip files to download from Canvas for test cases |
+
+#### Assignment Description Macros
+
+| Macro | Replacement |
+|---|---|
+| `DISCSN_URL` | URL of the discussion created for the assignment |
+| `EXMPLS <n>` | First `n` non-hidden test cases formatted for display |
+| `FILE[file_name]` | Link to the specified file in the Codeval folder |
+| `COMPILE` | Compile command from the `C` tag |
+| `GITHUB_DIRECTORY` | GitHub directory for submission |
+
+#### Example Specification File
+
+```
+ASSIGNMENT START Hello World
+# Hello World
+
+## Problem Statement
+Write a C program that reads a name from stdin and prints `Hello, <name>!`
+
+## Submission
+Submit your code to GITHUB_DIRECTORY
+
+## Sample Examples
+EXMPLS 2
+
+## Discussion
+DISCSN_URL
+
+ASSIGNMENT END
+
+# Compile the submission
+C gcc -o hello hello.c
+
+# Test case 1: greet "World"
+T ./hello
+I World
+O Hello, World!
+
+# Test case 2: greet "Alice"
+T ./hello
+I Alice
+O Hello, Alice!
+```
+
+### AI Benchmarking
+
+Test assignments against multiple AI models:
+
+```bash
+# Test with all Anthropic models
+assignment-codeval test-with-ai my_assignment.codeval -p anthropic
+
+# Test with a specific model, 3 attempts
+assignment-codeval test-with-ai my_assignment.codeval -m "Claude Sonnet 4" -n 3
+```
+
+| Option | Description |
+|--------|-------------|
+| `-o, --output-dir` | Directory to store solutions and results (default: `ai_test_results`) |
+| `-n, --attempts` | Number of attempts per model (default: 1) |
+| `-m, --models` | Specific models to test (repeatable) |
+| `-p, --providers` | Filter by provider: `anthropic`, `openai`, `google` |
+
+## Project Structure
+
+```
+src/assignment_codeval/
+├── cli.py              # Click CLI entry point, registers all subcommands
+├── submissions.py      # Download/upload submissions, list assignments
+├── evaluate.py         # Run evaluation on submissions (run-evaluation)
+├── create_assignment.py # Create assignments on Canvas
+├── github_connect.py   # GitHub repository setup and integration
+├── canvas_utils.py     # Canvas API utilities
+├── ai_benchmark.py     # AI model testing (test-with-ai)
+├── install_assignment.py # Install codeval files to local/remote destinations
+├── recent_comments.py  # List recent codeval comments on Canvas
+├── check_grading.py    # Check which submissions are missing grading
+├── export_tests.py     # Export test cases from codeval files
+├── convertMD2Html.py   # Markdown to HTML conversion
+├── commons.py          # Shared utilities
+├── file_utils.py       # File handling utilities
+└── test_template.html  # HTML template for test results
+
+tests/
+├── unit/               # Unit tests and sample programs/codeval fixtures
+├── integration/
+│   └── test_codeval.py # Integration test suite
+└── e2e/
+    └── test_e2e.py     # End-to-end tests
+
+samples/
+├── codeval.ini               # Sample configuration file
+└── assignment-name.codeval   # Sample specification file
+```

assignment_codeval-0.0.32/README.md

@@ -0,0 +1,221 @@
+# CodEval
+
+[![CI](https://github.com/SJSU-CMPE-195/group-project-team-29/actions/workflows/test.yml/badge.svg)](https://github.com/SJSU-CMPE-195/group-project-team-29/actions/workflows/test.yml)
+[![codecov](https://codecov.io/gh/SJSU-CMPE-195/group-project-team-29/branch/main/graph/badge.svg)](https://codecov.io/gh/SJSU-CMPE-195/group-project-team-29)
+[![PyPI](https://img.shields.io/pypi/v/assignment-codeval)](https://pypi.org/project/assignment-codeval/)
+
+A Python utility to download student submissions to programming assignments from Canvas and GitHub and evaluate them using codeval scripts.
+
+## Team
+- Sabira Abdolcader (sabdolc)
+- Chelsie Chen (cChe1z)
+- Aisha Syed (aisha-syed)
+- Zarah Taufique (zarahtau)
+
+## Prerequisites
+- Python 3.x
+- Docker (for running evaluations in containers)
+- A Canvas account with API access
+- A GitHub account (for GitHub-based assignments)
+- Optional AI provider packages: `anthropic`, `openai`, `google-generativeai`
+
+## Installation
+
+```bash
+# Install locally for development
+pip install -e .
+
+# Or install from PyPI
+pip install assignment-codeval
+
+# Install with AI provider support
+pip install assignment-codeval[ai]
+```
+
+## Configuration
+
+Create a `codeval.ini` file with your Canvas and run settings:
+
+```ini
+[SERVER]
+url=<canvas API>
+token=<canvas token>
+[RUN]
+precommand=
+command=
+```
+
+For distributed assignments, add:
+```ini
+[RUN]
+dist_command=
+host_ip=
+[MONGO]
+url=
+db=
+```
+
+For AI benchmarking, add:
+```ini
+[AI]
+anthropic_key=sk-ant-...
+openai_key=sk-...
+google_key=...
+```
+
+AI keys can also be set via environment variables: `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GOOGLE_API_KEY`.
+
+Refer to a sample config file [here](samples/codeval.ini)
+
+## Running the Application
+
+```bash
+# List all available commands
+assignment-codeval --help
+
+# Download submissions from Canvas/GitHub
+assignment-codeval download-submissions <course_name>
+
+# Evaluate downloaded submissions
+assignment-codeval evaluate-submissions <course_name>
+
+# Upload grading comments back to Canvas
+assignment-codeval upload-submission-comments <course_name>
+
+# Create an assignment on Canvas
+assignment-codeval create-assignment <course_name> <specification_file>
+
+# Check which submissions are missing grading
+assignment-codeval check-grading <course_name>
+
+# List recent codeval comments
+assignment-codeval recent-comments <course_name>
+
+# Test an assignment with AI models
+assignment-codeval test-with-ai <codeval_file> [OPTIONS]
+```
+
+## Usage
+
+### Codeval Specification Files
+
+Codeval files (`.codeval` extension) define how to build, run, and test student submissions.
+
+#### Assignment Description Tags
+- `ASSIGNMENT START <Assignment_name>` — begins the Canvas assignment description in markdown (`CRT_HW START` also accepted, but deprecated)
+- `ASSIGNMENT END` — ends the assignment description (`CRT_HW END` also accepted, but deprecated)
+
+#### Specification Tags
+
+| Tag | Meaning | Function |
+|---|---|---|
+| C | Compile Code | Specifies the command to compile the submission code |
+| CTO | Compile Timeout | Timeout in seconds for the whole evaluation (compile **and** all test cases) to run |
+| T/HT | Test Case | Command to run to test the submission (HT = hidden test) |
+| I/IB/IF | Supply Input | Input for a test case. I adds a newline, IB does not, IF reads from a file |
+| O/OB/OF | Check Output | Expected output. O adds a newline, OB does not, OF reads from a file |
+| E/EB | Check Error | Expected error output. E adds a newline, EB does not |
+| TO | Timeout | Time limit in seconds for a test case (default: 10) |
+| X | Exit Code | Expected exit code for a test case (not checked unless X is specified) |
+| TEMP | Temp File | Registers a file to be deleted before and after the next test run |
+| CF | Check Function | Checks that a function is used in the submission |
+| NCF | Check Not Function | Checks that a function is **not** used |
+| CMD/TCMD | Run Command | Runs a command. TCMD fails evaluation if the command errors |
+| PRINT | Print Label | Prints a label/message to stdout |
+| CMP | Compare | Compares two files |
+| Z | Download Zip | Zip files to download from Canvas for test cases |
+
+#### Assignment Description Macros
+
+| Macro | Replacement |
+|---|---|
+| `DISCSN_URL` | URL of the discussion created for the assignment |
+| `EXMPLS <n>` | First `n` non-hidden test cases formatted for display |
+| `FILE[file_name]` | Link to the specified file in the Codeval folder |
+| `COMPILE` | Compile command from the `C` tag |
+| `GITHUB_DIRECTORY` | GitHub directory for submission |
+
+#### Example Specification File
+
+```
+ASSIGNMENT START Hello World
+# Hello World
+
+## Problem Statement
+Write a C program that reads a name from stdin and prints `Hello, <name>!`
+
+## Submission
+Submit your code to GITHUB_DIRECTORY
+
+## Sample Examples
+EXMPLS 2
+
+## Discussion
+DISCSN_URL
+
+ASSIGNMENT END
+
+# Compile the submission
+C gcc -o hello hello.c
+
+# Test case 1: greet "World"
+T ./hello
+I World
+O Hello, World!
+
+# Test case 2: greet "Alice"
+T ./hello
+I Alice
+O Hello, Alice!
+```
+
+### AI Benchmarking
+
+Test assignments against multiple AI models:
+
+```bash
+# Test with all Anthropic models
+assignment-codeval test-with-ai my_assignment.codeval -p anthropic
+
+# Test with a specific model, 3 attempts
+assignment-codeval test-with-ai my_assignment.codeval -m "Claude Sonnet 4" -n 3
+```
+
+| Option | Description |
+|--------|-------------|
+| `-o, --output-dir` | Directory to store solutions and results (default: `ai_test_results`) |
+| `-n, --attempts` | Number of attempts per model (default: 1) |
+| `-m, --models` | Specific models to test (repeatable) |
+| `-p, --providers` | Filter by provider: `anthropic`, `openai`, `google` |
+
+## Project Structure
+
+```
+src/assignment_codeval/
+├── cli.py              # Click CLI entry point, registers all subcommands
+├── submissions.py      # Download/upload submissions, list assignments
+├── evaluate.py         # Run evaluation on submissions (run-evaluation)
+├── create_assignment.py # Create assignments on Canvas
+├── github_connect.py   # GitHub repository setup and integration
+├── canvas_utils.py     # Canvas API utilities
+├── ai_benchmark.py     # AI model testing (test-with-ai)
+├── install_assignment.py # Install codeval files to local/remote destinations
+├── recent_comments.py  # List recent codeval comments on Canvas
+├── check_grading.py    # Check which submissions are missing grading
+├── export_tests.py     # Export test cases from codeval files
+├── convertMD2Html.py   # Markdown to HTML conversion
+├── commons.py          # Shared utilities
+├── file_utils.py       # File handling utilities
+└── test_template.html  # HTML template for test results
+
+tests/
+├── unit/               # Unit tests and sample programs/codeval fixtures
+├── integration/
+│   └── test_codeval.py # Integration test suite
+└── e2e/
+    └── test_e2e.py     # End-to-end tests
+
+samples/
+├── codeval.ini               # Sample configuration file
+└── assignment-name.codeval   # Sample specification file
+```

{assignment_codeval-0.0.30 → assignment_codeval-0.0.32}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "assignment-codeval"
-version = "0.0.30"
+version = "0.0.32"
 description = "CodEval for evaluating programming assignments"
 readme = "README.md"
 requires-python = ">=3.12"
@@ -23,7 +23,7 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-test = ["pytest>=7.0"]
+test = ["pytest>=7.0", "pytest-cov"]
 
 [project.scripts]
 assignment-codeval = "assignment_codeval.cli:cli"

{assignment_codeval-0.0.30 → assignment_codeval-0.0.32}/src/assignment_codeval/ai_benchmark.py

@@ -75,8 +75,10 @@ def extract_assignment_from_codeval(codeval_path: str) -> tuple[str, str, str]:
     with open(codeval_path, "r", encoding="utf-8") as f:
         content = f.read()
 
-    # Extract content between CRT_HW START and CRT_HW END
-    match = re.search(r"CRT_HW START \S+\n(.*?)CRT_HW END", content, re.DOTALL)
+    # Extract content between ASSIGNMENT START / CRT_HW START and ASSIGNMENT END / CRT_HW END.
+    # The title after the marker may contain spaces (e.g. "CRT_HW START Place Boats")
+    # or be absent entirely, so match the rest of the marker line, not a single token.
+    match = re.search(r"(?:ASSIGNMENT START|CRT_HW START)[^\n]*\n(.*?)(?:ASSIGNMENT END|CRT_HW END)", content, re.DOTALL)
     if match:
         description = match.group(1).strip()
     else:
@@ -223,10 +225,17 @@ def call_openai(model_id: str, prompt: str, api_key: str) -> Optional[str]:
     try:
         client = openai.OpenAI(api_key=api_key)
 
+        # Reasoning models (o1/o3/o4 families, gpt-5) reject `max_tokens` and
+        # require `max_completion_tokens` instead.
+        if model_id.startswith(("o1", "o3", "o4", "gpt-5")):
+            token_kwargs = {"max_completion_tokens": 8192}
+        else:
+            token_kwargs = {"max_tokens": 8192}
+
         response = client.chat.completions.create(
             model=model_id,
             messages=[{"role": "user", "content": prompt}],
-            max_tokens=8192,
+            **token_kwargs,
         )
 
         return response.choices[0].message.content