assignment-codeval 0.0.28__py3-none-any.whl → 0.0.30__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- assignment_codeval/canvas_utils.py +1 -1
- assignment_codeval/evaluate.py +176 -28
- assignment_codeval/recent_comments.py +2 -2
- assignment_codeval/submissions.py +51 -3
- {assignment_codeval-0.0.28.dist-info → assignment_codeval-0.0.30.dist-info}/METADATA +7 -2
- {assignment_codeval-0.0.28.dist-info → assignment_codeval-0.0.30.dist-info}/RECORD +9 -9
- {assignment_codeval-0.0.28.dist-info → assignment_codeval-0.0.30.dist-info}/WHEEL +1 -1
- {assignment_codeval-0.0.28.dist-info → assignment_codeval-0.0.30.dist-info}/entry_points.txt +0 -0
- {assignment_codeval-0.0.28.dist-info → assignment_codeval-0.0.30.dist-info}/top_level.txt +0 -0
|
@@ -98,7 +98,7 @@ def get_assignment(course, assignment_name):
|
|
|
98
98
|
error(fr" {a.name}")
|
|
99
99
|
sys.exit(2)
|
|
100
100
|
elif len(assignments) > 1:
|
|
101
|
-
strict_name_assignments = [a for a in assignments if a.name == assignment_name]
|
|
101
|
+
strict_name_assignments = [a for a in assignments if despace(a.name) == assignment_name]
|
|
102
102
|
if len(strict_name_assignments) == 1:
|
|
103
103
|
assignments = strict_name_assignments
|
|
104
104
|
else:
|
assignment_codeval/evaluate.py
CHANGED
|
@@ -38,11 +38,23 @@ num_failed = 0
|
|
|
38
38
|
is_hidden_testcase = False
|
|
39
39
|
is_verbose = False
|
|
40
40
|
compilelog = []
|
|
41
|
+
last_compile_command = ""
|
|
42
|
+
temp_files = []
|
|
43
|
+
_active_temp_files = []
|
|
41
44
|
|
|
42
45
|
###########################################################
|
|
43
46
|
# Specification Tags to Function Mapping
|
|
44
47
|
###########################################################
|
|
45
48
|
|
|
49
|
+
# Shell commands that likely indicate a missing CMD prefix when found untagged
|
|
50
|
+
_BARE_SHELL_COMMANDS = {
|
|
51
|
+
'echo', 'rm', 'cp', 'mv', 'mkdir', 'rmdir', 'cat', 'ls', 'grep',
|
|
52
|
+
'sed', 'awk', 'find', 'chmod', 'chown', 'touch', 'ln', 'diff',
|
|
53
|
+
'sort', 'head', 'tail', 'cut', 'tr', 'wc', 'bash', 'sh', 'python',
|
|
54
|
+
'python3', 'make', 'export', 'source', 'kill', 'pkill', 'sleep',
|
|
55
|
+
'printf', 'read', 'unzip', 'tar', 'curl', 'wget',
|
|
56
|
+
}
|
|
57
|
+
|
|
46
58
|
|
|
47
59
|
def compile_code(compile_command):
|
|
48
60
|
"""Specifies the command to compile the submission code
|
|
@@ -53,6 +65,9 @@ def compile_code(compile_command):
|
|
|
53
65
|
Returns:
|
|
54
66
|
None
|
|
55
67
|
"""
|
|
68
|
+
global last_compile_command
|
|
69
|
+
last_compile_command = compile_command
|
|
70
|
+
|
|
56
71
|
if test_case_count != 0:
|
|
57
72
|
check_test()
|
|
58
73
|
|
|
@@ -112,6 +127,44 @@ def _find_compiled_artifact(source_file):
|
|
|
112
127
|
return None
|
|
113
128
|
|
|
114
129
|
|
|
130
|
+
def _extract_executable_from_compile(compile_command):
|
|
131
|
+
"""Extract the output executable name from a compile command (looks for -o <name>).
|
|
132
|
+
|
|
133
|
+
Returns the executable name, or None if not found.
|
|
134
|
+
"""
|
|
135
|
+
parts = compile_command.split()
|
|
136
|
+
for i, part in enumerate(parts):
|
|
137
|
+
if part == '-o' and i + 1 < len(parts):
|
|
138
|
+
return parts[i + 1]
|
|
139
|
+
return None
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _extract_source_files_from_compile(compile_command):
|
|
143
|
+
"""Extract source file arguments from a compile command.
|
|
144
|
+
|
|
145
|
+
Recognises common source file extensions (.c, .cpp, .cc, .cxx, .java, .py)
|
|
146
|
+
and skips option arguments that consume the following token as a value
|
|
147
|
+
(e.g. -o, -I, -L, -l, -include, -isystem, -MF, -MT, -MQ).
|
|
148
|
+
|
|
149
|
+
Returns a list of source file paths found in the command.
|
|
150
|
+
"""
|
|
151
|
+
source_exts = ('.c', '.cpp', '.cc', '.cxx', '.java', '.py')
|
|
152
|
+
value_flags = {'-o', '-I', '-L', '-l', '-include', '-isystem', '-MF', '-MT', '-MQ'}
|
|
153
|
+
parts = compile_command.split()
|
|
154
|
+
files = []
|
|
155
|
+
skip_next = False
|
|
156
|
+
for part in parts:
|
|
157
|
+
if skip_next:
|
|
158
|
+
skip_next = False
|
|
159
|
+
continue
|
|
160
|
+
if part in value_flags:
|
|
161
|
+
skip_next = True
|
|
162
|
+
continue
|
|
163
|
+
if not part.startswith('-') and any(part.endswith(ext) for ext in source_exts):
|
|
164
|
+
files.append(part)
|
|
165
|
+
return files
|
|
166
|
+
|
|
167
|
+
|
|
115
168
|
def _function_used_in_c_cpp(function_name, files):
|
|
116
169
|
"""Use objdump on compiled artifacts to detect C/C++ function usage.
|
|
117
170
|
|
|
@@ -209,14 +262,20 @@ def _function_used_regex(function_name, files):
|
|
|
209
262
|
return result.returncode == 0
|
|
210
263
|
|
|
211
264
|
|
|
212
|
-
def _is_function_used(function_name, files):
|
|
265
|
+
def _is_function_used(function_name, files, allow_regex_fallback=True):
|
|
213
266
|
"""Detect whether function_name is used in the given files.
|
|
214
267
|
|
|
215
268
|
Dispatches to the appropriate tool based on file language:
|
|
216
|
-
- C/C++: objdump on compiled artifact (falls back to regex if none found
|
|
217
|
-
|
|
269
|
+
- C/C++: objdump on compiled artifact (falls back to regex if none found
|
|
270
|
+
and allow_regex_fallback is True)
|
|
271
|
+
- Java: javap on .class file (falls back to regex if none found
|
|
272
|
+
and allow_regex_fallback is True)
|
|
218
273
|
- Python: ast module on source
|
|
219
|
-
- Other: regex on source
|
|
274
|
+
- Other: regex on source (only when allow_regex_fallback is True)
|
|
275
|
+
|
|
276
|
+
When allow_regex_fallback is False (used when no filename was given in the
|
|
277
|
+
CF tag), the function relies solely on compiled-artifact inspection and
|
|
278
|
+
returns False rather than falling back to regex.
|
|
220
279
|
"""
|
|
221
280
|
lang = _detect_language(files)
|
|
222
281
|
if lang == 'python':
|
|
@@ -224,36 +283,56 @@ def _is_function_used(function_name, files):
|
|
|
224
283
|
if lang == 'java':
|
|
225
284
|
result = _function_used_in_java(function_name, files)
|
|
226
285
|
if result is None:
|
|
227
|
-
return _function_used_regex(function_name, files)
|
|
286
|
+
return _function_used_regex(function_name, files) if allow_regex_fallback else False
|
|
228
287
|
return result
|
|
229
288
|
if lang == 'c_cpp':
|
|
230
289
|
result = _function_used_in_c_cpp(function_name, files)
|
|
231
290
|
if result is None:
|
|
232
|
-
return _function_used_regex(function_name, files)
|
|
291
|
+
return _function_used_regex(function_name, files) if allow_regex_fallback else False
|
|
233
292
|
return result
|
|
234
|
-
|
|
293
|
+
if allow_regex_fallback:
|
|
294
|
+
return _function_used_regex(function_name, files)
|
|
295
|
+
return False
|
|
235
296
|
|
|
236
297
|
|
|
237
298
|
def check_function(args):
|
|
238
|
-
"""Will be followed by a function name and
|
|
239
|
-
is used by one of those files.
|
|
299
|
+
"""Will be followed by a function name and an optional list of files to check to ensure that
|
|
300
|
+
the function is used by one of those files.
|
|
301
|
+
|
|
302
|
+
When no filename is provided (preferred usage), the source files are derived from the most
|
|
303
|
+
recent C (compile) tag. The check then relies solely on compiled-artifact inspection
|
|
304
|
+
(objdump for C/C++, javap for Java, ast for Python) with no regex fallback.
|
|
305
|
+
|
|
306
|
+
When a filename is provided (legacy usage), the same compiled-artifact inspection is used
|
|
307
|
+
first, but a regex scan of the source file is available as a fallback when no compiled
|
|
308
|
+
artifact can be found.
|
|
240
309
|
|
|
241
310
|
Arguments:
|
|
242
311
|
function_name: the function name to check files for usage of
|
|
243
|
-
*files: the files to check for the function name
|
|
312
|
+
*files: (optional) the files to check for the function name. If omitted, source
|
|
313
|
+
files are inferred from the most recent C tag.
|
|
244
314
|
|
|
245
315
|
Returns:
|
|
246
316
|
None
|
|
247
317
|
"""
|
|
248
318
|
check_test()
|
|
249
|
-
|
|
250
|
-
function_name =
|
|
251
|
-
files =
|
|
252
|
-
|
|
253
|
-
if
|
|
254
|
-
|
|
319
|
+
args_list = args.split()
|
|
320
|
+
function_name = args_list[0]
|
|
321
|
+
files = args_list[1:]
|
|
322
|
+
|
|
323
|
+
if not files:
|
|
324
|
+
# No filename provided — derive source files from the compile command and
|
|
325
|
+
# disable the regex fallback (compiled-artifact check only).
|
|
326
|
+
files = _extract_source_files_from_compile(last_compile_command)
|
|
327
|
+
if _is_function_used(function_name, files, allow_regex_fallback=False):
|
|
328
|
+
print(f"Used {function_name} PASSED")
|
|
329
|
+
else:
|
|
330
|
+
print(f"Not using {function_name} FAILED")
|
|
255
331
|
else:
|
|
256
|
-
|
|
332
|
+
if _is_function_used(function_name, files):
|
|
333
|
+
print(f"Used {function_name} PASSED")
|
|
334
|
+
else:
|
|
335
|
+
print(f"Not using {function_name} FAILED")
|
|
257
336
|
|
|
258
337
|
def check_object(args):
|
|
259
338
|
"""Will be followed by an object and a list of files to ensure that the function is
|
|
@@ -352,22 +431,38 @@ def check_container(args):
|
|
|
352
431
|
|
|
353
432
|
|
|
354
433
|
def check_not_function(args):
|
|
355
|
-
"""Will be followed by a function name and
|
|
356
|
-
is not used by any of those files.
|
|
434
|
+
"""Will be followed by a function name and an optional list of files to check to ensure that
|
|
435
|
+
the function is not used by any of those files.
|
|
436
|
+
|
|
437
|
+
When no filename is provided (preferred usage), the source files are derived from the most
|
|
438
|
+
recent C (compile) tag. The check relies solely on compiled-artifact inspection with no
|
|
439
|
+
regex fallback.
|
|
440
|
+
|
|
441
|
+
When a filename is provided (legacy usage), compiled-artifact inspection is used first with
|
|
442
|
+
a regex fallback when no compiled artifact is found.
|
|
357
443
|
|
|
358
444
|
Arguments:
|
|
359
445
|
function_name: the function name to check files for usage of
|
|
360
|
-
*files: the files to check for the function name
|
|
446
|
+
*files: (optional) the files to check for the function name. If omitted, source
|
|
447
|
+
files are inferred from the most recent C tag.
|
|
361
448
|
|
|
362
449
|
Returns:
|
|
363
450
|
None
|
|
364
451
|
"""
|
|
365
452
|
check_test()
|
|
366
|
-
|
|
367
|
-
function_name =
|
|
368
|
-
files =
|
|
453
|
+
args_list = args.split()
|
|
454
|
+
function_name = args_list[0]
|
|
455
|
+
files = args_list[1:]
|
|
456
|
+
|
|
457
|
+
if not files:
|
|
458
|
+
# No filename provided — derive source files from the compile command and
|
|
459
|
+
# disable the regex fallback (compiled-artifact check only).
|
|
460
|
+
files = _extract_source_files_from_compile(last_compile_command)
|
|
461
|
+
used = _is_function_used(function_name, files, allow_regex_fallback=False)
|
|
462
|
+
else:
|
|
463
|
+
used = _is_function_used(function_name, files)
|
|
369
464
|
|
|
370
|
-
if
|
|
465
|
+
if used:
|
|
371
466
|
print(f"Used {function_name} FAILED")
|
|
372
467
|
else:
|
|
373
468
|
print(f"Not using {function_name} PASSED")
|
|
@@ -412,6 +507,7 @@ def run_command_noerror(command):
|
|
|
412
507
|
None
|
|
413
508
|
"""
|
|
414
509
|
check_test()
|
|
510
|
+
_pre_test_temp_cleanup()
|
|
415
511
|
|
|
416
512
|
# Run as test case
|
|
417
513
|
global test_case_count
|
|
@@ -460,6 +556,7 @@ def test_case(test_case_command):
|
|
|
460
556
|
None
|
|
461
557
|
"""
|
|
462
558
|
check_test()
|
|
559
|
+
_pre_test_temp_cleanup()
|
|
463
560
|
|
|
464
561
|
# Clear hint
|
|
465
562
|
global test_case_hint
|
|
@@ -488,6 +585,7 @@ def test_case_hidden(test_case_command):
|
|
|
488
585
|
None
|
|
489
586
|
"""
|
|
490
587
|
check_test()
|
|
588
|
+
_pre_test_temp_cleanup()
|
|
491
589
|
|
|
492
590
|
# Clear hint
|
|
493
591
|
global test_case_hint
|
|
@@ -585,9 +683,10 @@ def check_output_file(output_file):
|
|
|
585
683
|
Returns:
|
|
586
684
|
None
|
|
587
685
|
"""
|
|
686
|
+
output_length(os.path.getsize(output_file))
|
|
588
687
|
with open(output_file, "r") as infile:
|
|
589
688
|
output_lines = infile.readlines()
|
|
590
|
-
|
|
689
|
+
|
|
591
690
|
with open(get_testing_path("expectedoutput"), "a") as outfile:
|
|
592
691
|
outfile.writelines(output_lines)
|
|
593
692
|
|
|
@@ -631,6 +730,48 @@ def hint(hints):
|
|
|
631
730
|
test_case_hint = hints
|
|
632
731
|
|
|
633
732
|
|
|
733
|
+
def register_temp_file(filename):
|
|
734
|
+
"""Registers a file to be deleted before and after the next T, HT, or TCMD test.
|
|
735
|
+
|
|
736
|
+
Arguments:
|
|
737
|
+
filename: the file to delete before and after the next test
|
|
738
|
+
|
|
739
|
+
Returns:
|
|
740
|
+
None
|
|
741
|
+
"""
|
|
742
|
+
global temp_files
|
|
743
|
+
temp_files.append(filename.strip())
|
|
744
|
+
|
|
745
|
+
|
|
746
|
+
def _cleanup_temp_files():
|
|
747
|
+
"""Delete all registered temp files and reset the list."""
|
|
748
|
+
global temp_files
|
|
749
|
+
for f in temp_files:
|
|
750
|
+
try:
|
|
751
|
+
os.remove(f)
|
|
752
|
+
except FileNotFoundError:
|
|
753
|
+
pass
|
|
754
|
+
temp_files = []
|
|
755
|
+
|
|
756
|
+
|
|
757
|
+
def _pre_test_temp_cleanup():
|
|
758
|
+
"""Before a test: delete registered temp files and save the list for post-test cleanup."""
|
|
759
|
+
global temp_files, _active_temp_files
|
|
760
|
+
_active_temp_files = list(temp_files)
|
|
761
|
+
_cleanup_temp_files()
|
|
762
|
+
|
|
763
|
+
|
|
764
|
+
def _post_test_temp_cleanup():
|
|
765
|
+
"""After a test: delete the files that were registered when the test started."""
|
|
766
|
+
global _active_temp_files
|
|
767
|
+
for f in _active_temp_files:
|
|
768
|
+
try:
|
|
769
|
+
os.remove(f)
|
|
770
|
+
except FileNotFoundError:
|
|
771
|
+
pass
|
|
772
|
+
_active_temp_files = []
|
|
773
|
+
|
|
774
|
+
|
|
634
775
|
def timeout(timeout_sec):
|
|
635
776
|
"""Specifies the time limit in seconds for a test case to run. Defaults to 20 seconds.
|
|
636
777
|
|
|
@@ -743,6 +884,7 @@ tag_func_map = {
|
|
|
743
884
|
"OLEN": output_length,
|
|
744
885
|
"X": exit_code,
|
|
745
886
|
"SS": start_server,
|
|
887
|
+
"TEMP": register_temp_file,
|
|
746
888
|
}
|
|
747
889
|
|
|
748
890
|
|
|
@@ -840,6 +982,13 @@ def parse_tags(tags: list[str]):
|
|
|
840
982
|
print(f" {line_num}: {tag_line.rstrip()}")
|
|
841
983
|
print(f" Valid tags are: {', '.join(sorted(valid_tags))}")
|
|
842
984
|
sys.exit(1)
|
|
985
|
+
else:
|
|
986
|
+
# Check if this looks like a bare shell command missing a CMD prefix
|
|
987
|
+
first_word = tag_line.split()[0] if tag_line.split() else ""
|
|
988
|
+
if first_word.lower() in _BARE_SHELL_COMMANDS:
|
|
989
|
+
print(f"Warning on line {line_num}: bare shell command without CMD prefix, shell commands will be ignored")
|
|
990
|
+
print(f" {line_num}: {tag_line.rstrip()}")
|
|
991
|
+
print(f" Did you mean: CMD {tag_line.rstrip()}")
|
|
843
992
|
continue
|
|
844
993
|
|
|
845
994
|
tag = tag_match.group(1)
|
|
@@ -1138,12 +1287,14 @@ def check_test():
|
|
|
1138
1287
|
if len(lines) > 22:
|
|
1139
1288
|
print(f" ... ({len(lines) - 22} more lines)")
|
|
1140
1289
|
|
|
1290
|
+
_post_test_temp_cleanup()
|
|
1141
1291
|
cleanup()
|
|
1142
1292
|
|
|
1143
1293
|
# Exit program after failed test case
|
|
1144
1294
|
sys.exit(2)
|
|
1145
1295
|
|
|
1146
1296
|
# reinitialize test variables and files here
|
|
1297
|
+
_post_test_temp_cleanup()
|
|
1147
1298
|
setup()
|
|
1148
1299
|
|
|
1149
1300
|
|
|
@@ -1215,8 +1366,5 @@ def run_evaluation(codeval_file):
|
|
|
1215
1366
|
|
|
1216
1367
|
check_test()
|
|
1217
1368
|
|
|
1218
|
-
# cleanup
|
|
1219
|
-
cleanup()
|
|
1220
|
-
|
|
1221
1369
|
end_time_seconds = time.time()
|
|
1222
1370
|
print(f"took {end_time_seconds - start_time_seconds} seconds")
|
|
@@ -12,6 +12,7 @@ from assignment_codeval.canvas_utils import (
|
|
|
12
12
|
get_canvas_credentials, graphql_request, fetch_all_submissions,
|
|
13
13
|
)
|
|
14
14
|
from assignment_codeval.commons import despace
|
|
15
|
+
from assignment_codeval.submissions import find_codeval_file
|
|
15
16
|
|
|
16
17
|
COURSE_ASSIGNMENTS_QUERY = """
|
|
17
18
|
query CourseAssignmentsQuery($courseId: ID!) {
|
|
@@ -164,8 +165,7 @@ def recent_comments(course_name, active, time_period, codeval_prefix, verbose, s
|
|
|
164
165
|
# Check if this assignment has a codeval file
|
|
165
166
|
has_codeval = True
|
|
166
167
|
if codeval_dir:
|
|
167
|
-
|
|
168
|
-
has_codeval = os.path.exists(codeval_file)
|
|
168
|
+
has_codeval = find_codeval_file(codeval_dir, despace(assignment_name)) is not None
|
|
169
169
|
if not has_codeval:
|
|
170
170
|
if verbose:
|
|
171
171
|
click.echo(f" (no codeval file)", err=True)
|
|
@@ -17,13 +17,41 @@ from assignment_codeval.canvas_utils import connect_to_canvas, get_course, get_c
|
|
|
17
17
|
from assignment_codeval.commons import debug, error, info, warn, despace
|
|
18
18
|
|
|
19
19
|
|
|
20
|
+
def _extract_codeval_title(filepath):
|
|
21
|
+
"""Extract the assignment title from the CRT_HW START line of a codeval file.
|
|
22
|
+
Returns None if no CRT_HW START line is found."""
|
|
23
|
+
try:
|
|
24
|
+
with open(filepath, 'r') as f:
|
|
25
|
+
for line in f:
|
|
26
|
+
if 'CRT_HW START' in line:
|
|
27
|
+
idx = line.index('CRT_HW START') + len('CRT_HW START')
|
|
28
|
+
title = line[idx:].strip()
|
|
29
|
+
return title if title else None
|
|
30
|
+
except (OSError, UnicodeDecodeError):
|
|
31
|
+
pass
|
|
32
|
+
return None
|
|
33
|
+
|
|
34
|
+
|
|
20
35
|
def find_codeval_file(codeval_dir, assignment_name):
|
|
21
36
|
"""Find a codeval file by assignment name, case-insensitive.
|
|
37
|
+
First tries matching by filename, then falls back to checking
|
|
38
|
+
CRT_HW START titles inside each .codeval file.
|
|
22
39
|
Returns the full path if found, or None if not found."""
|
|
23
40
|
target = f"{assignment_name}.codeval".lower()
|
|
41
|
+
codeval_files = []
|
|
24
42
|
for f in os.listdir(codeval_dir):
|
|
25
43
|
if f.lower() == target:
|
|
26
44
|
return os.path.join(codeval_dir, f)
|
|
45
|
+
if f.endswith('.codeval'):
|
|
46
|
+
codeval_files.append(f)
|
|
47
|
+
|
|
48
|
+
# Fallback: check CRT_HW START titles
|
|
49
|
+
target_name = assignment_name.lower()
|
|
50
|
+
for f in codeval_files:
|
|
51
|
+
filepath = os.path.join(codeval_dir, f)
|
|
52
|
+
title = _extract_codeval_title(filepath)
|
|
53
|
+
if title and despace(title).lower() == target_name:
|
|
54
|
+
return filepath
|
|
27
55
|
return None
|
|
28
56
|
|
|
29
57
|
|
|
@@ -577,7 +605,8 @@ def get_submissions_by_id(assignment):
|
|
|
577
605
|
|
|
578
606
|
@click.command()
|
|
579
607
|
@click.argument("course_name", metavar="COURSE", required=False)
|
|
580
|
-
|
|
608
|
+
@click.option("--path", is_flag=True, help="Show the path to the codeval file")
|
|
609
|
+
def list_codeval_assignments(course_name, path):
|
|
581
610
|
"""
|
|
582
611
|
List all assignments that have corresponding codeval files.
|
|
583
612
|
|
|
@@ -596,7 +625,15 @@ def list_codeval_assignments(course_name):
|
|
|
596
625
|
raise click.UsageError(f"CODEVAL directory does not exist: {codeval_dir}")
|
|
597
626
|
|
|
598
627
|
# Get all codeval files in the directory (map lowercase name -> original name)
|
|
599
|
-
codeval_files = {
|
|
628
|
+
codeval_files = {}
|
|
629
|
+
codeval_titles = {} # map despaced-lowercase CRT_HW title -> original filename stem
|
|
630
|
+
for f in os.listdir(codeval_dir):
|
|
631
|
+
if f.endswith('.codeval'):
|
|
632
|
+
stem = f[:-8]
|
|
633
|
+
codeval_files[stem.lower()] = stem
|
|
634
|
+
title = _extract_codeval_title(os.path.join(codeval_dir, f))
|
|
635
|
+
if title:
|
|
636
|
+
codeval_titles[despace(title).lower()] = stem
|
|
600
637
|
matched_codeval_keys = set()
|
|
601
638
|
|
|
602
639
|
(canvas, user) = connect_to_canvas()
|
|
@@ -610,7 +647,18 @@ def list_codeval_assignments(course_name):
|
|
|
610
647
|
assignment_key = despace(assignment.name).lower()
|
|
611
648
|
if assignment_key in codeval_files:
|
|
612
649
|
matched_codeval_keys.add(assignment_key)
|
|
613
|
-
|
|
650
|
+
codeval_path = os.path.join(codeval_dir, f"{codeval_files[assignment_key]}.codeval")
|
|
651
|
+
if path:
|
|
652
|
+
info(f"{course.name}: {assignment.name} [{codeval_path}]")
|
|
653
|
+
else:
|
|
654
|
+
info(f"{course.name}: {assignment.name}")
|
|
655
|
+
elif assignment_key in codeval_titles:
|
|
656
|
+
matched_codeval_keys.add(codeval_titles[assignment_key].lower())
|
|
657
|
+
codeval_path = os.path.join(codeval_dir, f"{codeval_titles[assignment_key]}.codeval")
|
|
658
|
+
if path:
|
|
659
|
+
info(f"{course.name}: {assignment.name} [{codeval_path}]")
|
|
660
|
+
else:
|
|
661
|
+
info(f"{course.name}: {assignment.name}")
|
|
614
662
|
|
|
615
663
|
# Report codeval files that don't match any assignment
|
|
616
664
|
unmatched_keys = set(codeval_files.keys()) - matched_codeval_keys
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: assignment-codeval
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.30
|
|
4
4
|
Summary: CodEval for evaluating programming assignments
|
|
5
5
|
Requires-Python: >=3.12
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
@@ -57,6 +57,9 @@ Use a part of the course name that can uniquely identify the course on Canvas.
|
|
|
57
57
|
- Default: --no-copytmpdir
|
|
58
58
|
- Copy temporary directory content to current directory for debugging
|
|
59
59
|
|
|
60
|
+
### Codeval File Matching
|
|
61
|
+
When looking up the codeval file for an assignment, CodEval first tries to match by filename (case-insensitive). If no filename match is found, it falls back to checking the `CRT_HW START` title inside each `.codeval` file in the codeval directory. This allows the codeval filename to differ from the assignment name on Canvas as long as the `CRT_HW START` title matches.
|
|
62
|
+
|
|
60
63
|
### Specification Tags
|
|
61
64
|
Tags used in a spec file (\<course name>.codeval)
|
|
62
65
|
|
|
@@ -66,7 +69,8 @@ Tags used in a spec file (\<course name>.codeval)
|
|
|
66
69
|
| CTO | Compile Timeout | Timeout in seconds for the compile command to run |
|
|
67
70
|
| RUN | Run Script | Specifies the script to use to evaluate the specification file. Defaults to evaluate.sh. |
|
|
68
71
|
| Z | Download Zip | Will be followed by zip files to download from Canvas to use when running the test cases. |
|
|
69
|
-
| CF | Check Function |
|
|
72
|
+
| CF | Check Function | `CF <function_name> [filename]` — checks that the function is used. The filename is **optional** and omitting it (e.g. `CF strtol`) is the preferred usage; source files are inferred from the most recent `C` tag and the check uses compiled-artifact inspection only (objdump/javap/ast). Providing a filename (e.g. `CF strtol mycalc.c`) is supported for backwards compatibility and additionally enables a regex fallback when no compiled artifact is found. |
|
|
73
|
+
| NCF | Check Not Function | `NCF <function_name> [filename]` — checks that the function is **not** used. The filename is **optional** (same behaviour as CF). |
|
|
70
74
|
| CC | Check Container | Will be followed by a function name and a list of files to check to ensure that a container is used by one of those files. Primarily supports C++ containers such as std::vector |
|
|
71
75
|
| CO | Check Object | Will be followed by a function name and a list of files to check to ensure that an object is used by one of those files. Primarily support C++ stream operations |
|
|
72
76
|
| PRINT | Print Label | Prints a label/message to stdout. Cleaner alternative to `CMD echo "..."` for section labels. |
|
|
@@ -79,6 +83,7 @@ Tags used in a spec file (\<course name>.codeval)
|
|
|
79
83
|
| TO | Timeout | Specifies the time limit in seconds for a test case to run. Defaults to 20 seconds. |
|
|
80
84
|
| X | Exit Code | Specifies the expected exit code for a test case. Defaults to zero. |
|
|
81
85
|
| SS | Start Server | Command containing timeout (wait until server starts), kill timeout (wait to kill the server), and the command to start a server |
|
|
86
|
+
| TEMP | Temp File | Registers a file to be deleted before the next T, HT, or TCMD test runs (clean state) and again after it completes (cleanup). Only applies to the immediately following test — use a new TEMP tag for each test that needs it. |
|
|
82
87
|
|
|
83
88
|
Refer to a sample spec file [here](samples/assignment-name.codeval)
|
|
84
89
|
|
|
@@ -1,21 +1,21 @@
|
|
|
1
1
|
assignment_codeval/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
2
|
assignment_codeval/ai_benchmark.py,sha256=ZGWWrBeUcaArkuQqS9wY42gxP8g7J2nivJMedYq9RFs,18045
|
|
3
|
-
assignment_codeval/canvas_utils.py,sha256=
|
|
3
|
+
assignment_codeval/canvas_utils.py,sha256=NF2xw10A1okVtDnP98YRzEiSo7QlqYF-IVAih4-WeLQ,5998
|
|
4
4
|
assignment_codeval/check_grading.py,sha256=rnNdp8T_XWckDfF1zUi3r70BMe9CPNTD-RfjfarI-CM,5889
|
|
5
5
|
assignment_codeval/cli.py,sha256=Vl27zNI7kI0QEZoXTIE2aTrraEJWj9vgJiyVWou3vgU,1357
|
|
6
6
|
assignment_codeval/commons.py,sha256=rr09qHacHoLmmU8S9sBss4_utb0DH8-IyL9Dsj0ycE0,1412
|
|
7
7
|
assignment_codeval/convertMD2Html.py,sha256=cCfTWwxH-0SKZ7Br-Qn54OabPuEAhHpx3q4Aeqp_NAo,7682
|
|
8
8
|
assignment_codeval/create_assignment.py,sha256=duGI8wZRpeEnmYPAAth_QXcFHWd185zkjkSJsaeH-GY,11999
|
|
9
|
-
assignment_codeval/evaluate.py,sha256=
|
|
9
|
+
assignment_codeval/evaluate.py,sha256=PlpWGD-aHi_OrZzKmmmDOHN14D1p1BN-9AwWt0YZ4ts,42362
|
|
10
10
|
assignment_codeval/export_tests.py,sha256=IDAlGAp3zk-K5AFwLr85v4w9_WcYr6HXVF0joi4eYTg,5467
|
|
11
11
|
assignment_codeval/file_utils.py,sha256=RDgAgmyIOLKvRlxbAr-zILz2A24zSr-r3--p-dxl4G8,2132
|
|
12
12
|
assignment_codeval/github_connect.py,sha256=AP7mZPhzKKK8fxS-3JMfG6AQ-1wUutfsq-Bt-KG9WUs,4270
|
|
13
13
|
assignment_codeval/install_assignment.py,sha256=HsXzTex4G-U3t4tb-HyQ2AobBDBPcKeR6oAUDFqBP-A,4599
|
|
14
|
-
assignment_codeval/recent_comments.py,sha256=
|
|
15
|
-
assignment_codeval/submissions.py,sha256=
|
|
14
|
+
assignment_codeval/recent_comments.py,sha256=lmGjr5rcpdP6Q69AjG8kSsgOFQgdeyL8XcbNvwqGk-Q,9102
|
|
15
|
+
assignment_codeval/submissions.py,sha256=93lLooN3eI0hTc3ZdUGoMwIQkWragjkuyePuvcU52tk,30431
|
|
16
16
|
assignment_codeval/test_template.html,sha256=lgWsin5sYE_Pwjp3xqa7_UT-yZ4dWrVqAvNlSBBiNMI,2584
|
|
17
|
-
assignment_codeval-0.0.
|
|
18
|
-
assignment_codeval-0.0.
|
|
19
|
-
assignment_codeval-0.0.
|
|
20
|
-
assignment_codeval-0.0.
|
|
21
|
-
assignment_codeval-0.0.
|
|
17
|
+
assignment_codeval-0.0.30.dist-info/METADATA,sha256=GeGmxlA9BNfAo_s-4VnpfKKy90LKau0SRPAYIAjfzLY,16730
|
|
18
|
+
assignment_codeval-0.0.30.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
19
|
+
assignment_codeval-0.0.30.dist-info/entry_points.txt,sha256=Pk0eAhsXBpCsgkv6HYLSHTRf5x_zUf-Lg8KK9KsK2B8,66
|
|
20
|
+
assignment_codeval-0.0.30.dist-info/top_level.txt,sha256=a_P8W2IWjAELxKlTV2t4rLwWVoTxElxUMvWgWjR_1n4,19
|
|
21
|
+
assignment_codeval-0.0.30.dist-info/RECORD,,
|
{assignment_codeval-0.0.28.dist-info → assignment_codeval-0.0.30.dist-info}/entry_points.txt
RENAMED
|
File without changes
|
|
File without changes
|