rbx.cp 0.5.35__py3-none-any.whl → 0.5.37__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rbx/box/builder.py +5 -2
- rbx/box/contest/main.py +2 -1
- rbx/box/creation.py +7 -12
- rbx/box/generators.py +17 -318
- rbx/box/generators_test.py +1 -1
- rbx/box/main.py +10 -0
- rbx/box/package.py +2 -2
- rbx/box/schema.py +8 -1
- rbx/box/solutions.py +1 -1
- rbx/box/solutions_test.py +1 -1
- rbx/box/testcase_extractors.py +348 -0
- rbx/box/testcase_utils.py +17 -7
- rbx/box/testcases/main.py +8 -6
- rbx/box/validators.py +61 -33
- rbx/config.py +16 -2
- {rbx_cp-0.5.35.dist-info → rbx_cp-0.5.37.dist-info}/METADATA +1 -1
- {rbx_cp-0.5.35.dist-info → rbx_cp-0.5.37.dist-info}/RECORD +20 -19
- {rbx_cp-0.5.35.dist-info → rbx_cp-0.5.37.dist-info}/LICENSE +0 -0
- {rbx_cp-0.5.35.dist-info → rbx_cp-0.5.37.dist-info}/WHEEL +0 -0
- {rbx_cp-0.5.35.dist-info → rbx_cp-0.5.37.dist-info}/entry_points.txt +0 -0
rbx/box/builder.py
CHANGED
@@ -4,7 +4,6 @@ from rbx import console, utils
|
|
4
4
|
from rbx.box import environment, package
|
5
5
|
from rbx.box.environment import VerificationLevel
|
6
6
|
from rbx.box.generators import (
|
7
|
-
extract_generation_testcases_from_groups,
|
8
7
|
generate_outputs_for_testcases,
|
9
8
|
generate_testcases,
|
10
9
|
)
|
@@ -13,6 +12,7 @@ from rbx.box.solutions import (
|
|
13
12
|
print_run_report,
|
14
13
|
run_solutions,
|
15
14
|
)
|
15
|
+
from rbx.box.testcase_extractors import extract_generation_testcases_from_groups
|
16
16
|
from rbx.box.validators import (
|
17
17
|
has_validation_errors,
|
18
18
|
print_validation_report,
|
@@ -50,7 +50,10 @@ def build(
|
|
50
50
|
'Validated [item]{processed}[/item] testcases...',
|
51
51
|
keep=True,
|
52
52
|
) as s:
|
53
|
-
infos = validate_testcases(
|
53
|
+
infos = validate_testcases(
|
54
|
+
s,
|
55
|
+
groups=groups,
|
56
|
+
)
|
54
57
|
print_validation_report(infos)
|
55
58
|
|
56
59
|
if has_validation_errors(infos):
|
rbx/box/contest/main.py
CHANGED
@@ -128,7 +128,8 @@ def edit():
|
|
128
128
|
@app.command('add, a', help='Add new problem to contest.')
|
129
129
|
@within_contest
|
130
130
|
def add(path: str, short_name: str, preset: Optional[str] = None):
|
131
|
-
|
131
|
+
problem_path = pathlib.Path(path)
|
132
|
+
name = problem_path.stem
|
132
133
|
utils.validate_field(ContestProblem, 'short_name', short_name)
|
133
134
|
utils.validate_field(Package, 'name', name)
|
134
135
|
|
rbx/box/creation.py
CHANGED
@@ -4,9 +4,8 @@ from typing import Annotated, Optional
|
|
4
4
|
|
5
5
|
import typer
|
6
6
|
|
7
|
-
from rbx import console
|
8
|
-
from rbx.box import presets
|
9
|
-
from rbx.box.contest.contest_package import find_contest_yaml
|
7
|
+
from rbx import console, utils
|
8
|
+
from rbx.box import package, presets
|
10
9
|
from rbx.box.presets.fetch import get_preset_fetch_info
|
11
10
|
|
12
11
|
|
@@ -27,15 +26,6 @@ def create(
|
|
27
26
|
] = None,
|
28
27
|
path: Optional[pathlib.Path] = None,
|
29
28
|
):
|
30
|
-
if find_contest_yaml() is not None:
|
31
|
-
console.console.print(
|
32
|
-
'[error]Cannot [item]rbx create[/item] a problem inside a contest.[/error]'
|
33
|
-
)
|
34
|
-
console.console.print(
|
35
|
-
'[error]Instead, use [item]rbx contest add[/item] to add a problem to a contest.[/error]'
|
36
|
-
)
|
37
|
-
raise typer.Exit(1)
|
38
|
-
|
39
29
|
preset = preset or 'default'
|
40
30
|
console.console.print(f'Creating new problem [item]{name}[/item]...')
|
41
31
|
|
@@ -80,4 +70,9 @@ def create(
|
|
80
70
|
for lock in dest_path.rglob('.preset-lock.yml'):
|
81
71
|
lock.unlink(missing_ok=True)
|
82
72
|
|
73
|
+
# Change problem name.
|
74
|
+
ru, problem = package.get_ruyaml(dest_path)
|
75
|
+
problem['name'] = name
|
76
|
+
utils.save_ruyaml(dest_path / 'problem.rbx.yml', ru, problem)
|
77
|
+
|
83
78
|
presets.generate_lock(preset, root=dest_path)
|
rbx/box/generators.py
CHANGED
@@ -1,12 +1,8 @@
|
|
1
|
-
import abc
|
2
1
|
import pathlib
|
3
|
-
import shlex
|
4
2
|
import shutil
|
5
|
-
from
|
6
|
-
from typing import Dict, Iterable, List, Optional, Set, Tuple
|
3
|
+
from typing import Dict, List, Optional, Set
|
7
4
|
|
8
5
|
import typer
|
9
|
-
from pydantic import BaseModel
|
10
6
|
|
11
7
|
from rbx import console
|
12
8
|
from rbx.box import checkers, package, testcase_utils, validators
|
@@ -19,10 +15,20 @@ from rbx.box.schema import (
|
|
19
15
|
CodeItem,
|
20
16
|
GeneratorCall,
|
21
17
|
Testcase,
|
22
|
-
TestcaseSubgroup,
|
23
18
|
)
|
24
19
|
from rbx.box.stressing import generator_parser
|
25
|
-
from rbx.box.
|
20
|
+
from rbx.box.testcase_extractors import (
|
21
|
+
GenerationMetadata,
|
22
|
+
GenerationTestcaseEntry,
|
23
|
+
TestcaseGroupVisitor,
|
24
|
+
extract_generation_testcases,
|
25
|
+
run_testcase_visitor,
|
26
|
+
)
|
27
|
+
from rbx.box.testcase_utils import (
|
28
|
+
TestcaseEntry,
|
29
|
+
fill_output_for_defined_testcase,
|
30
|
+
find_built_testcases,
|
31
|
+
)
|
26
32
|
from rbx.grading.steps import (
|
27
33
|
DigestHolder,
|
28
34
|
DigestOrDest,
|
@@ -35,33 +41,11 @@ def _compile_generator(generator: CodeItem) -> str:
|
|
35
41
|
return compile_item(generator, sanitized=SanitizationLevel.PREFER)
|
36
42
|
|
37
43
|
|
38
|
-
def _get_group_input(
|
39
|
-
group_path: pathlib.Path, subgroup_prefix: str, i: int
|
40
|
-
) -> pathlib.Path:
|
41
|
-
return group_path / f'{subgroup_prefix}{i:03d}.in'
|
42
|
-
|
43
|
-
|
44
|
-
def _get_group_output(
|
45
|
-
group_path: pathlib.Path, subgroup_prefix: str, i: int
|
46
|
-
) -> pathlib.Path:
|
47
|
-
return group_path / f'{subgroup_prefix}{i:03d}.out'
|
48
|
-
|
49
|
-
|
50
|
-
def _fill_output_for_defined_testcase(testcase: Testcase) -> Testcase:
|
51
|
-
res = testcase.model_copy()
|
52
|
-
if res.outputPath is not None:
|
53
|
-
return res
|
54
|
-
output_path = res.inputPath.with_suffix('.ans')
|
55
|
-
if output_path.is_file():
|
56
|
-
res.outputPath = output_path
|
57
|
-
return res
|
58
|
-
|
59
|
-
|
60
44
|
def _copy_testcase_over(
|
61
45
|
testcase: Testcase,
|
62
46
|
dest: Testcase,
|
63
47
|
):
|
64
|
-
testcase =
|
48
|
+
testcase = fill_output_for_defined_testcase(testcase)
|
65
49
|
dest.inputPath.parent.mkdir(parents=True, exist_ok=True)
|
66
50
|
shutil.copy(
|
67
51
|
str(testcase.inputPath),
|
@@ -90,230 +74,6 @@ def get_call_from_string(call_str: str) -> GeneratorCall:
|
|
90
74
|
return GeneratorCall(name=name, args=args)
|
91
75
|
|
92
76
|
|
93
|
-
def _run_generator_script(testcase: TestcaseSubgroup) -> str:
|
94
|
-
assert testcase.generatorScript is not None
|
95
|
-
|
96
|
-
cacher = package.get_file_cacher()
|
97
|
-
|
98
|
-
if not testcase.generatorScript.path.is_file():
|
99
|
-
console.console.print(
|
100
|
-
f'[error]Generator script not found: [item]{testcase.generatorScript.path}[/item][/error]'
|
101
|
-
)
|
102
|
-
raise typer.Exit(1)
|
103
|
-
|
104
|
-
script_digest = DigestHolder()
|
105
|
-
if testcase.generatorScript.path.suffix == '.txt':
|
106
|
-
script_digest.value = cacher.put_file_from_path(testcase.generatorScript.path)
|
107
|
-
else:
|
108
|
-
try:
|
109
|
-
compiled_digest = compile_item(testcase.generatorScript)
|
110
|
-
except:
|
111
|
-
console.console.print(
|
112
|
-
f'[error]Failed compiling generator script for group [item]{testcase.name}[/item].[/error]'
|
113
|
-
)
|
114
|
-
raise
|
115
|
-
|
116
|
-
run_stderr = DigestHolder()
|
117
|
-
run_log = run_item(
|
118
|
-
testcase.generatorScript,
|
119
|
-
DigestOrSource.create(compiled_digest),
|
120
|
-
stdout=DigestOrDest.create(script_digest),
|
121
|
-
stderr=DigestOrDest.create(run_stderr),
|
122
|
-
)
|
123
|
-
|
124
|
-
if run_log is None or run_log.exitcode != 0:
|
125
|
-
console.console.print(
|
126
|
-
f'Could not run generator script for group {testcase.name}'
|
127
|
-
)
|
128
|
-
if run_log is not None:
|
129
|
-
console.console.print(
|
130
|
-
f'[error]Summary:[/error] {run_log.get_summary()}'
|
131
|
-
)
|
132
|
-
if run_stderr.value is not None:
|
133
|
-
console.console.print('[error]Stderr:[/error]')
|
134
|
-
console.console.print(
|
135
|
-
package.get_digest_as_string(run_stderr.value) or ''
|
136
|
-
)
|
137
|
-
raise typer.Exit(1)
|
138
|
-
|
139
|
-
assert script_digest.value
|
140
|
-
script = cacher.get_file_content(script_digest.value).decode()
|
141
|
-
return script
|
142
|
-
|
143
|
-
|
144
|
-
def _extract_script_lines(script: str) -> Iterable[Tuple[str, str, int]]:
|
145
|
-
lines = script.splitlines()
|
146
|
-
for i, line in enumerate(lines):
|
147
|
-
line = line.strip()
|
148
|
-
if not line:
|
149
|
-
continue
|
150
|
-
if line.startswith('#'):
|
151
|
-
continue
|
152
|
-
yield shlex.split(line)[0], shlex.join(shlex.split(line)[1:]), i + 1
|
153
|
-
|
154
|
-
|
155
|
-
class GeneratorScriptEntry(BaseModel):
|
156
|
-
path: pathlib.Path
|
157
|
-
line: int
|
158
|
-
|
159
|
-
|
160
|
-
class GenerationMetadata(BaseModel):
|
161
|
-
copied_to: Testcase
|
162
|
-
|
163
|
-
copied_from: Optional[Testcase] = None
|
164
|
-
generator_call: Optional[GeneratorCall] = None
|
165
|
-
generator_script: Optional[GeneratorScriptEntry] = None
|
166
|
-
|
167
|
-
|
168
|
-
class GenerationTestcaseEntry(BaseModel):
|
169
|
-
group_entry: TestcaseEntry
|
170
|
-
subgroup_entry: TestcaseEntry
|
171
|
-
|
172
|
-
metadata: GenerationMetadata
|
173
|
-
|
174
|
-
|
175
|
-
class TestcaseVisitor(abc.ABC):
|
176
|
-
@abc.abstractmethod
|
177
|
-
def visit(self, entry: GenerationTestcaseEntry):
|
178
|
-
pass
|
179
|
-
|
180
|
-
def should_visit_group(self, group_name: str) -> bool:
|
181
|
-
return True
|
182
|
-
|
183
|
-
def should_visit_subgroup(self, subgroup_path: str) -> bool:
|
184
|
-
return True
|
185
|
-
|
186
|
-
def should_visit_generator_scripts(
|
187
|
-
self, group_name: str, subgroup_path: str
|
188
|
-
) -> bool:
|
189
|
-
return True
|
190
|
-
|
191
|
-
|
192
|
-
class TestcaseGroupVisitor(TestcaseVisitor):
|
193
|
-
def __init__(self, groups: Optional[Set[str]] = None):
|
194
|
-
self.groups = groups
|
195
|
-
|
196
|
-
def should_visit_group(self, group_name: str) -> bool:
|
197
|
-
return self.groups is None or group_name in self.groups
|
198
|
-
|
199
|
-
|
200
|
-
def run_testcase_visitor(visitor: TestcaseVisitor):
|
201
|
-
pkg = package.find_problem_package_or_die()
|
202
|
-
|
203
|
-
def _explore_subgroup(
|
204
|
-
subgroup: TestcaseSubgroup, subgroup_index: Optional[int], prefix: List[str]
|
205
|
-
):
|
206
|
-
assert prefix and len(prefix) >= 1 and len(prefix) <= 2
|
207
|
-
group_path = prefix[0]
|
208
|
-
subgroup_path = '/'.join(prefix)
|
209
|
-
if not visitor.should_visit_subgroup(subgroup_path):
|
210
|
-
return
|
211
|
-
|
212
|
-
def _entry(i: int) -> TestcaseEntry:
|
213
|
-
return TestcaseEntry(group=group_path, index=i)
|
214
|
-
|
215
|
-
def _sub_entry(i: int) -> TestcaseEntry:
|
216
|
-
return TestcaseEntry(group=subgroup_path, index=i)
|
217
|
-
|
218
|
-
def _copied_to(i: int) -> Testcase:
|
219
|
-
group_fs_path = package.get_build_testgroup_path(group_path)
|
220
|
-
group_prefix = ''
|
221
|
-
if subgroup_index is not None:
|
222
|
-
group_prefix = f'{subgroup_index}-'
|
223
|
-
if len(prefix) == 2:
|
224
|
-
group_prefix += f'{prefix[1]}-'
|
225
|
-
return Testcase(
|
226
|
-
inputPath=_get_group_input(group_fs_path, group_prefix, i),
|
227
|
-
outputPath=_get_group_output(group_fs_path, group_prefix, i),
|
228
|
-
)
|
229
|
-
|
230
|
-
# Go through testcases.
|
231
|
-
i = 0
|
232
|
-
# Individual testcases.
|
233
|
-
for tc in subgroup.testcases or []:
|
234
|
-
visitor.visit(
|
235
|
-
GenerationTestcaseEntry(
|
236
|
-
group_entry=_entry(i),
|
237
|
-
subgroup_entry=_sub_entry(i),
|
238
|
-
metadata=GenerationMetadata(
|
239
|
-
copied_from=_fill_output_for_defined_testcase(tc),
|
240
|
-
copied_to=_copied_to(i),
|
241
|
-
),
|
242
|
-
)
|
243
|
-
)
|
244
|
-
i += 1
|
245
|
-
|
246
|
-
# Glob testcases.
|
247
|
-
if subgroup.testcaseGlob:
|
248
|
-
matched_inputs = sorted(PosixPath().glob(subgroup.testcaseGlob))
|
249
|
-
|
250
|
-
for input_path in matched_inputs:
|
251
|
-
if not input_path.is_file() or input_path.suffix != '.in':
|
252
|
-
continue
|
253
|
-
|
254
|
-
tc = Testcase(inputPath=input_path)
|
255
|
-
visitor.visit(
|
256
|
-
GenerationTestcaseEntry(
|
257
|
-
group_entry=_entry(i),
|
258
|
-
subgroup_entry=_sub_entry(i),
|
259
|
-
metadata=GenerationMetadata(
|
260
|
-
copied_from=_fill_output_for_defined_testcase(tc),
|
261
|
-
copied_to=_copied_to(i),
|
262
|
-
),
|
263
|
-
)
|
264
|
-
)
|
265
|
-
i += 1
|
266
|
-
|
267
|
-
# Single generators.
|
268
|
-
for generator_call in subgroup.generators:
|
269
|
-
visitor.visit(
|
270
|
-
GenerationTestcaseEntry(
|
271
|
-
group_entry=_entry(i),
|
272
|
-
subgroup_entry=_sub_entry(i),
|
273
|
-
metadata=GenerationMetadata(
|
274
|
-
generator_call=generator_call,
|
275
|
-
copied_to=_copied_to(i),
|
276
|
-
),
|
277
|
-
)
|
278
|
-
)
|
279
|
-
i += 1
|
280
|
-
|
281
|
-
if not visitor.should_visit_generator_scripts(group_path, subgroup_path):
|
282
|
-
return
|
283
|
-
|
284
|
-
# Run generator script.
|
285
|
-
if subgroup.generatorScript is not None:
|
286
|
-
script = _run_generator_script(subgroup)
|
287
|
-
|
288
|
-
# Run each line from generator script.
|
289
|
-
for generator_name, args, line_number in _extract_script_lines(script):
|
290
|
-
call = GeneratorCall(name=generator_name, args=args)
|
291
|
-
visitor.visit(
|
292
|
-
GenerationTestcaseEntry(
|
293
|
-
group_entry=_entry(i),
|
294
|
-
subgroup_entry=_sub_entry(i),
|
295
|
-
metadata=GenerationMetadata(
|
296
|
-
generator_call=call,
|
297
|
-
generator_script=GeneratorScriptEntry(
|
298
|
-
path=subgroup.generatorScript.path,
|
299
|
-
line=line_number,
|
300
|
-
),
|
301
|
-
copied_to=_copied_to(i),
|
302
|
-
),
|
303
|
-
)
|
304
|
-
)
|
305
|
-
i += 1
|
306
|
-
|
307
|
-
for group in pkg.testcases:
|
308
|
-
if not visitor.should_visit_group(group.name):
|
309
|
-
continue
|
310
|
-
|
311
|
-
_explore_subgroup(group, 0 if group.subgroups else None, [group.name])
|
312
|
-
|
313
|
-
for i, subgroup in enumerate(group.subgroups):
|
314
|
-
_explore_subgroup(subgroup, i + 1, [group.name, subgroup.name])
|
315
|
-
|
316
|
-
|
317
77
|
def _get_necessary_generators_for_groups(
|
318
78
|
groups: Optional[Set[str]] = None,
|
319
79
|
) -> Set[str]:
|
@@ -439,14 +199,14 @@ def generate_standalone(
|
|
439
199
|
_, validator_digest = validator_tp
|
440
200
|
if progress:
|
441
201
|
progress.update('Validating test...')
|
442
|
-
|
202
|
+
validation_info = validators.validate_one_off(
|
443
203
|
spec.copied_to.inputPath,
|
444
204
|
validator,
|
445
205
|
validator_digest,
|
446
206
|
)
|
447
|
-
if not ok:
|
207
|
+
if not validation_info.ok:
|
448
208
|
_print_error_header('failed validating testcase.')
|
449
|
-
console.console.print(f'[error]Message:[/error] {message}')
|
209
|
+
console.console.print(f'[error]Message:[/error] {validation_info.message}')
|
450
210
|
console.console.print(
|
451
211
|
f'Testcase written at [item]{spec.copied_to.inputPath}[/item]'
|
452
212
|
)
|
@@ -552,67 +312,6 @@ def generate_output_for_testcase(
|
|
552
312
|
raise typer.Exit(1)
|
553
313
|
|
554
314
|
|
555
|
-
def extract_generation_testcases(
|
556
|
-
entries: List[TestcaseEntry],
|
557
|
-
) -> List[GenerationTestcaseEntry]:
|
558
|
-
# TODO: support subgroups.
|
559
|
-
groups = set(entry.group for entry in entries)
|
560
|
-
entry_keys = set(entry.key() for entry in entries)
|
561
|
-
|
562
|
-
res: List[GenerationTestcaseEntry] = []
|
563
|
-
|
564
|
-
class ExtractGenerationTestcasesVisitor(TestcaseVisitor):
|
565
|
-
def should_visit_group(self, group_name: str) -> bool:
|
566
|
-
return group_name in groups
|
567
|
-
|
568
|
-
def visit(self, entry: GenerationTestcaseEntry):
|
569
|
-
# TODO: support subgroups.
|
570
|
-
if entry.group_entry.key() not in entry_keys:
|
571
|
-
return
|
572
|
-
res.append(entry)
|
573
|
-
|
574
|
-
run_testcase_visitor(ExtractGenerationTestcasesVisitor())
|
575
|
-
return res
|
576
|
-
|
577
|
-
|
578
|
-
def extract_generation_testcases_from_groups(
|
579
|
-
groups: Optional[Set[str]] = None,
|
580
|
-
) -> List[GenerationTestcaseEntry]:
|
581
|
-
res: List[GenerationTestcaseEntry] = []
|
582
|
-
|
583
|
-
class ExtractGenerationTestcasesVisitor(TestcaseGroupVisitor):
|
584
|
-
def visit(self, entry: GenerationTestcaseEntry):
|
585
|
-
res.append(entry)
|
586
|
-
|
587
|
-
run_testcase_visitor(ExtractGenerationTestcasesVisitor(groups))
|
588
|
-
return res
|
589
|
-
|
590
|
-
|
591
|
-
def extract_generation_testcases_from_patterns(
|
592
|
-
patterns: List[TestcasePattern],
|
593
|
-
) -> List[GenerationTestcaseEntry]:
|
594
|
-
res: List[GenerationTestcaseEntry] = []
|
595
|
-
|
596
|
-
class ExtractGenerationTestcasesVisitor(TestcaseVisitor):
|
597
|
-
def should_visit_group(self, group_name: str) -> bool:
|
598
|
-
return any(pattern.intersecting_group(group_name) for pattern in patterns)
|
599
|
-
|
600
|
-
def should_visit_subgroup(self, subgroup_path: str) -> bool:
|
601
|
-
return any(
|
602
|
-
pattern.intersecting_group(subgroup_path) for pattern in patterns
|
603
|
-
)
|
604
|
-
|
605
|
-
def visit(self, entry: GenerationTestcaseEntry):
|
606
|
-
if not any(
|
607
|
-
pattern.match(entry.group_entry) for pattern in patterns
|
608
|
-
) and not any(pattern.match(entry.subgroup_entry) for pattern in patterns):
|
609
|
-
return
|
610
|
-
res.append(entry)
|
611
|
-
|
612
|
-
run_testcase_visitor(ExtractGenerationTestcasesVisitor())
|
613
|
-
return res
|
614
|
-
|
615
|
-
|
616
315
|
def generate_outputs_for_testcases(
|
617
316
|
entries: List[TestcaseEntry],
|
618
317
|
progress: Optional[StatusProgress] = None,
|
rbx/box/generators_test.py
CHANGED
@@ -4,10 +4,10 @@ import pytest
|
|
4
4
|
|
5
5
|
from rbx.box import package
|
6
6
|
from rbx.box.generators import (
|
7
|
-
extract_generation_testcases_from_groups,
|
8
7
|
generate_outputs_for_testcases,
|
9
8
|
generate_testcases,
|
10
9
|
)
|
10
|
+
from rbx.box.testcase_extractors import extract_generation_testcases_from_groups
|
11
11
|
from rbx.testing_utils import print_directory_tree
|
12
12
|
|
13
13
|
|
rbx/box/main.py
CHANGED
@@ -37,6 +37,7 @@ from rbx.box import (
|
|
37
37
|
validators,
|
38
38
|
)
|
39
39
|
from rbx.box.contest import main as contest
|
40
|
+
from rbx.box.contest.contest_package import find_contest_yaml
|
40
41
|
from rbx.box.environment import VerificationLevel, get_environment_path
|
41
42
|
from rbx.box.packaging import main as packaging
|
42
43
|
from rbx.box.testcases import main as testcases
|
@@ -448,6 +449,15 @@ def create(
|
|
448
449
|
Optional[str], typer.Option(help='Preset to use when creating the problem.')
|
449
450
|
] = None,
|
450
451
|
):
|
452
|
+
if find_contest_yaml() is not None:
|
453
|
+
console.console.print(
|
454
|
+
'[error]Cannot [item]rbx create[/item] a problem inside a contest.[/error]'
|
455
|
+
)
|
456
|
+
console.console.print(
|
457
|
+
'[error]Instead, use [item]rbx contest add[/item] to add a problem to a contest.[/error]'
|
458
|
+
)
|
459
|
+
raise typer.Exit(1)
|
460
|
+
|
451
461
|
if preset is not None:
|
452
462
|
creation.create(name, preset=preset)
|
453
463
|
return
|
rbx/box/package.py
CHANGED
@@ -121,8 +121,8 @@ def save_package(
|
|
121
121
|
problem_yaml_path.write_text(utils.model_to_yaml(package))
|
122
122
|
|
123
123
|
|
124
|
-
def get_ruyaml() -> Tuple[ruyaml.YAML, ruyaml.Any]:
|
125
|
-
problem_yaml_path = find_problem_yaml()
|
124
|
+
def get_ruyaml(root: pathlib.Path = pathlib.Path()) -> Tuple[ruyaml.YAML, ruyaml.Any]:
|
125
|
+
problem_yaml_path = find_problem_yaml(root)
|
126
126
|
if problem_yaml_path is None:
|
127
127
|
console.console.print(
|
128
128
|
f'Problem not found in {pathlib.Path().absolute()}', style='error'
|
rbx/box/schema.py
CHANGED
@@ -232,6 +232,13 @@ A generator script to call to generate testcases for this group.
|
|
232
232
|
""",
|
233
233
|
)
|
234
234
|
|
235
|
+
extraValidators: List[CodeItem] = Field(
|
236
|
+
default=[],
|
237
|
+
description="""
|
238
|
+
A list of extra validators to use to validate the testcases of this subgroup.
|
239
|
+
""",
|
240
|
+
)
|
241
|
+
|
235
242
|
@model_validator(mode='after')
|
236
243
|
def check_oneof(self) -> 'TestcaseSubgroup':
|
237
244
|
_check_oneof(
|
@@ -260,7 +267,7 @@ A list of test subgroups to define for this group.
|
|
260
267
|
default=None,
|
261
268
|
description="""
|
262
269
|
A validator to use to validate the testcases of this group.
|
263
|
-
If
|
270
|
+
If specified, will use this validator instead of the package-level validator.
|
264
271
|
Useful in cases where the constraints vary across test groups.
|
265
272
|
""",
|
266
273
|
)
|
rbx/box/solutions.py
CHANGED
@@ -29,7 +29,6 @@ from rbx.box.formatting import get_formatted_memory, get_formatted_time
|
|
29
29
|
from rbx.box.generators import (
|
30
30
|
GenerationMetadata,
|
31
31
|
expand_generator_call,
|
32
|
-
extract_generation_testcases,
|
33
32
|
generate_output_for_testcase,
|
34
33
|
generate_standalone,
|
35
34
|
)
|
@@ -42,6 +41,7 @@ from rbx.box.schema import (
|
|
42
41
|
Testcase,
|
43
42
|
TestcaseGroup,
|
44
43
|
)
|
44
|
+
from rbx.box.testcase_extractors import extract_generation_testcases
|
45
45
|
from rbx.box.testcase_utils import TestcaseEntry, find_built_testcases
|
46
46
|
from rbx.grading.steps import (
|
47
47
|
DigestOrDest,
|
rbx/box/solutions_test.py
CHANGED
@@ -5,7 +5,6 @@ import pytest
|
|
5
5
|
|
6
6
|
from rbx.box.environment import VerificationLevel
|
7
7
|
from rbx.box.generators import (
|
8
|
-
extract_generation_testcases_from_groups,
|
9
8
|
generate_outputs_for_testcases,
|
10
9
|
generate_testcases,
|
11
10
|
)
|
@@ -13,6 +12,7 @@ from rbx.box.solutions import (
|
|
13
12
|
convert_list_of_solution_evaluations_to_dict,
|
14
13
|
run_solutions,
|
15
14
|
)
|
15
|
+
from rbx.box.testcase_extractors import extract_generation_testcases_from_groups
|
16
16
|
from rbx.grading.steps import Outcome
|
17
17
|
|
18
18
|
|
@@ -0,0 +1,348 @@
|
|
1
|
+
import abc
|
2
|
+
import pathlib
|
3
|
+
import shlex
|
4
|
+
from typing import Iterable, List, Optional, Set, Tuple
|
5
|
+
|
6
|
+
import typer
|
7
|
+
from pydantic import BaseModel
|
8
|
+
|
9
|
+
from rbx import console
|
10
|
+
from rbx.box import package
|
11
|
+
from rbx.box.code import compile_item, run_item
|
12
|
+
from rbx.box.schema import CodeItem, GeneratorCall, Testcase, TestcaseSubgroup
|
13
|
+
from rbx.box.testcase_utils import (
|
14
|
+
TestcaseEntry,
|
15
|
+
TestcasePattern,
|
16
|
+
fill_output_for_defined_testcase,
|
17
|
+
)
|
18
|
+
from rbx.grading.steps import DigestHolder, DigestOrDest, DigestOrSource
|
19
|
+
|
20
|
+
|
21
|
+
def _get_group_input(
|
22
|
+
group_path: pathlib.Path, subgroup_prefix: str, i: int
|
23
|
+
) -> pathlib.Path:
|
24
|
+
return group_path / f'{subgroup_prefix}{i:03d}.in'
|
25
|
+
|
26
|
+
|
27
|
+
def _get_group_output(
|
28
|
+
group_path: pathlib.Path, subgroup_prefix: str, i: int
|
29
|
+
) -> pathlib.Path:
|
30
|
+
return group_path / f'{subgroup_prefix}{i:03d}.out'
|
31
|
+
|
32
|
+
|
33
|
+
def _run_generator_script(testcase: TestcaseSubgroup) -> str:
|
34
|
+
assert testcase.generatorScript is not None
|
35
|
+
|
36
|
+
cacher = package.get_file_cacher()
|
37
|
+
|
38
|
+
if not testcase.generatorScript.path.is_file():
|
39
|
+
console.console.print(
|
40
|
+
f'[error]Generator script not found: [item]{testcase.generatorScript.path}[/item][/error]'
|
41
|
+
)
|
42
|
+
raise typer.Exit(1)
|
43
|
+
|
44
|
+
script_digest = DigestHolder()
|
45
|
+
if testcase.generatorScript.path.suffix == '.txt':
|
46
|
+
script_digest.value = cacher.put_file_from_path(testcase.generatorScript.path)
|
47
|
+
else:
|
48
|
+
try:
|
49
|
+
compiled_digest = compile_item(testcase.generatorScript)
|
50
|
+
except:
|
51
|
+
console.console.print(
|
52
|
+
f'[error]Failed compiling generator script for group [item]{testcase.name}[/item].[/error]'
|
53
|
+
)
|
54
|
+
raise
|
55
|
+
|
56
|
+
run_stderr = DigestHolder()
|
57
|
+
run_log = run_item(
|
58
|
+
testcase.generatorScript,
|
59
|
+
DigestOrSource.create(compiled_digest),
|
60
|
+
stdout=DigestOrDest.create(script_digest),
|
61
|
+
stderr=DigestOrDest.create(run_stderr),
|
62
|
+
)
|
63
|
+
|
64
|
+
if run_log is None or run_log.exitcode != 0:
|
65
|
+
console.console.print(
|
66
|
+
f'Could not run generator script for group {testcase.name}'
|
67
|
+
)
|
68
|
+
if run_log is not None:
|
69
|
+
console.console.print(
|
70
|
+
f'[error]Summary:[/error] {run_log.get_summary()}'
|
71
|
+
)
|
72
|
+
if run_stderr.value is not None:
|
73
|
+
console.console.print('[error]Stderr:[/error]')
|
74
|
+
console.console.print(
|
75
|
+
package.get_digest_as_string(run_stderr.value) or ''
|
76
|
+
)
|
77
|
+
raise typer.Exit(1)
|
78
|
+
|
79
|
+
assert script_digest.value
|
80
|
+
script = cacher.get_file_content(script_digest.value).decode()
|
81
|
+
return script
|
82
|
+
|
83
|
+
|
84
|
+
def _extract_script_lines(script: str) -> Iterable[Tuple[str, str, int]]:
|
85
|
+
lines = script.splitlines()
|
86
|
+
for i, line in enumerate(lines):
|
87
|
+
line = line.strip()
|
88
|
+
if not line:
|
89
|
+
continue
|
90
|
+
if line.startswith('#'):
|
91
|
+
continue
|
92
|
+
yield shlex.split(line)[0], shlex.join(shlex.split(line)[1:]), i + 1
|
93
|
+
|
94
|
+
|
95
|
+
class GeneratorScriptEntry(BaseModel):
|
96
|
+
path: pathlib.Path
|
97
|
+
line: int
|
98
|
+
|
99
|
+
|
100
|
+
class GenerationMetadata(BaseModel):
|
101
|
+
copied_to: Testcase
|
102
|
+
|
103
|
+
copied_from: Optional[Testcase] = None
|
104
|
+
generator_call: Optional[GeneratorCall] = None
|
105
|
+
generator_script: Optional[GeneratorScriptEntry] = None
|
106
|
+
|
107
|
+
|
108
|
+
class GenerationTestcaseEntry(BaseModel):
|
109
|
+
group_entry: TestcaseEntry
|
110
|
+
subgroup_entry: TestcaseEntry
|
111
|
+
|
112
|
+
metadata: GenerationMetadata
|
113
|
+
validator: Optional[CodeItem] = None
|
114
|
+
extra_validators: List[CodeItem] = []
|
115
|
+
|
116
|
+
|
117
|
+
class TestcaseVisitor(abc.ABC):
|
118
|
+
@abc.abstractmethod
|
119
|
+
def visit(self, entry: GenerationTestcaseEntry):
|
120
|
+
pass
|
121
|
+
|
122
|
+
def should_visit_group(self, group_name: str) -> bool:
|
123
|
+
return True
|
124
|
+
|
125
|
+
def should_visit_subgroup(self, subgroup_path: str) -> bool:
|
126
|
+
return True
|
127
|
+
|
128
|
+
def should_visit_generator_scripts(
|
129
|
+
self, group_name: str, subgroup_path: str
|
130
|
+
) -> bool:
|
131
|
+
return True
|
132
|
+
|
133
|
+
|
134
|
+
class TestcaseGroupVisitor(TestcaseVisitor):
|
135
|
+
def __init__(self, groups: Optional[Set[str]] = None):
|
136
|
+
self.groups = groups
|
137
|
+
|
138
|
+
def should_visit_group(self, group_name: str) -> bool:
|
139
|
+
return self.groups is None or group_name in self.groups
|
140
|
+
|
141
|
+
|
142
|
+
def run_testcase_visitor(visitor: TestcaseVisitor):
|
143
|
+
pkg = package.find_problem_package_or_die()
|
144
|
+
|
145
|
+
def _explore_subgroup(
|
146
|
+
subgroup: TestcaseSubgroup,
|
147
|
+
subgroup_index: Optional[int],
|
148
|
+
prefix: List[str],
|
149
|
+
validator: Optional[CodeItem] = None,
|
150
|
+
extra_validators: Optional[List[CodeItem]] = None,
|
151
|
+
):
|
152
|
+
extra_validators = extra_validators or []
|
153
|
+
|
154
|
+
assert prefix and len(prefix) >= 1 and len(prefix) <= 2
|
155
|
+
group_path = prefix[0]
|
156
|
+
subgroup_path = '/'.join(prefix)
|
157
|
+
if not visitor.should_visit_subgroup(subgroup_path):
|
158
|
+
return
|
159
|
+
|
160
|
+
def _entry(i: int) -> TestcaseEntry:
|
161
|
+
return TestcaseEntry(group=group_path, index=i)
|
162
|
+
|
163
|
+
def _sub_entry(i: int) -> TestcaseEntry:
|
164
|
+
return TestcaseEntry(group=subgroup_path, index=i)
|
165
|
+
|
166
|
+
def _copied_to(i: int) -> Testcase:
|
167
|
+
group_fs_path = package.get_build_testgroup_path(group_path)
|
168
|
+
group_prefix = ''
|
169
|
+
if subgroup_index is not None:
|
170
|
+
group_prefix = f'{subgroup_index}-'
|
171
|
+
if len(prefix) == 2:
|
172
|
+
group_prefix += f'{prefix[1]}-'
|
173
|
+
return Testcase(
|
174
|
+
inputPath=_get_group_input(group_fs_path, group_prefix, i),
|
175
|
+
outputPath=_get_group_output(group_fs_path, group_prefix, i),
|
176
|
+
)
|
177
|
+
|
178
|
+
# Go through testcases.
|
179
|
+
i = 0
|
180
|
+
# Individual testcases.
|
181
|
+
for tc in subgroup.testcases or []:
|
182
|
+
visitor.visit(
|
183
|
+
GenerationTestcaseEntry(
|
184
|
+
group_entry=_entry(i),
|
185
|
+
subgroup_entry=_sub_entry(i),
|
186
|
+
metadata=GenerationMetadata(
|
187
|
+
copied_from=fill_output_for_defined_testcase(tc),
|
188
|
+
copied_to=_copied_to(i),
|
189
|
+
),
|
190
|
+
validator=validator,
|
191
|
+
extra_validators=extra_validators,
|
192
|
+
)
|
193
|
+
)
|
194
|
+
i += 1
|
195
|
+
|
196
|
+
# Glob testcases.
|
197
|
+
if subgroup.testcaseGlob:
|
198
|
+
matched_inputs = sorted(pathlib.PosixPath().glob(subgroup.testcaseGlob))
|
199
|
+
|
200
|
+
for input_path in matched_inputs:
|
201
|
+
if not input_path.is_file() or input_path.suffix != '.in':
|
202
|
+
continue
|
203
|
+
|
204
|
+
tc = Testcase(inputPath=input_path)
|
205
|
+
visitor.visit(
|
206
|
+
GenerationTestcaseEntry(
|
207
|
+
group_entry=_entry(i),
|
208
|
+
subgroup_entry=_sub_entry(i),
|
209
|
+
metadata=GenerationMetadata(
|
210
|
+
copied_from=fill_output_for_defined_testcase(tc),
|
211
|
+
copied_to=_copied_to(i),
|
212
|
+
),
|
213
|
+
validator=validator,
|
214
|
+
extra_validators=extra_validators,
|
215
|
+
)
|
216
|
+
)
|
217
|
+
i += 1
|
218
|
+
|
219
|
+
# Single generators.
|
220
|
+
for generator_call in subgroup.generators:
|
221
|
+
visitor.visit(
|
222
|
+
GenerationTestcaseEntry(
|
223
|
+
group_entry=_entry(i),
|
224
|
+
subgroup_entry=_sub_entry(i),
|
225
|
+
metadata=GenerationMetadata(
|
226
|
+
generator_call=generator_call,
|
227
|
+
copied_to=_copied_to(i),
|
228
|
+
),
|
229
|
+
validator=validator,
|
230
|
+
extra_validators=extra_validators,
|
231
|
+
)
|
232
|
+
)
|
233
|
+
i += 1
|
234
|
+
|
235
|
+
if not visitor.should_visit_generator_scripts(group_path, subgroup_path):
|
236
|
+
return
|
237
|
+
|
238
|
+
# Run generator script.
|
239
|
+
if subgroup.generatorScript is not None:
|
240
|
+
script = _run_generator_script(subgroup)
|
241
|
+
|
242
|
+
# Run each line from generator script.
|
243
|
+
for generator_name, args, line_number in _extract_script_lines(script):
|
244
|
+
call = GeneratorCall(name=generator_name, args=args)
|
245
|
+
visitor.visit(
|
246
|
+
GenerationTestcaseEntry(
|
247
|
+
group_entry=_entry(i),
|
248
|
+
subgroup_entry=_sub_entry(i),
|
249
|
+
metadata=GenerationMetadata(
|
250
|
+
generator_call=call,
|
251
|
+
generator_script=GeneratorScriptEntry(
|
252
|
+
path=subgroup.generatorScript.path,
|
253
|
+
line=line_number,
|
254
|
+
),
|
255
|
+
copied_to=_copied_to(i),
|
256
|
+
),
|
257
|
+
validator=validator,
|
258
|
+
extra_validators=extra_validators,
|
259
|
+
)
|
260
|
+
)
|
261
|
+
i += 1
|
262
|
+
|
263
|
+
for group in pkg.testcases:
|
264
|
+
if not visitor.should_visit_group(group.name):
|
265
|
+
continue
|
266
|
+
|
267
|
+
group_validator = pkg.validator
|
268
|
+
if group.validator is not None:
|
269
|
+
group_validator = group.validator
|
270
|
+
|
271
|
+
extra_validators = group.extraValidators
|
272
|
+
_explore_subgroup(
|
273
|
+
group,
|
274
|
+
0 if group.subgroups else None,
|
275
|
+
[group.name],
|
276
|
+
validator=group_validator,
|
277
|
+
extra_validators=extra_validators,
|
278
|
+
)
|
279
|
+
|
280
|
+
for i, subgroup in enumerate(group.subgroups):
|
281
|
+
_explore_subgroup(
|
282
|
+
subgroup,
|
283
|
+
i + 1,
|
284
|
+
[group.name, subgroup.name],
|
285
|
+
validator=group_validator,
|
286
|
+
extra_validators=extra_validators + subgroup.extraValidators,
|
287
|
+
)
|
288
|
+
|
289
|
+
|
290
|
+
def extract_generation_testcases(
|
291
|
+
entries: List[TestcaseEntry],
|
292
|
+
) -> List[GenerationTestcaseEntry]:
|
293
|
+
# TODO: support subgroups.
|
294
|
+
groups = set(entry.group for entry in entries)
|
295
|
+
entry_keys = set(entry.key() for entry in entries)
|
296
|
+
|
297
|
+
res: List[GenerationTestcaseEntry] = []
|
298
|
+
|
299
|
+
class ExtractGenerationTestcasesVisitor(TestcaseVisitor):
|
300
|
+
def should_visit_group(self, group_name: str) -> bool:
|
301
|
+
return group_name in groups
|
302
|
+
|
303
|
+
def visit(self, entry: GenerationTestcaseEntry):
|
304
|
+
# TODO: support subgroups.
|
305
|
+
if entry.group_entry.key() not in entry_keys:
|
306
|
+
return
|
307
|
+
res.append(entry)
|
308
|
+
|
309
|
+
run_testcase_visitor(ExtractGenerationTestcasesVisitor())
|
310
|
+
return res
|
311
|
+
|
312
|
+
|
313
|
+
def extract_generation_testcases_from_groups(
|
314
|
+
groups: Optional[Set[str]] = None,
|
315
|
+
) -> List[GenerationTestcaseEntry]:
|
316
|
+
res: List[GenerationTestcaseEntry] = []
|
317
|
+
|
318
|
+
class ExtractGenerationTestcasesVisitor(TestcaseGroupVisitor):
|
319
|
+
def visit(self, entry: GenerationTestcaseEntry):
|
320
|
+
res.append(entry)
|
321
|
+
|
322
|
+
run_testcase_visitor(ExtractGenerationTestcasesVisitor(groups))
|
323
|
+
return res
|
324
|
+
|
325
|
+
|
326
|
+
def extract_generation_testcases_from_patterns(
|
327
|
+
patterns: List[TestcasePattern],
|
328
|
+
) -> List[GenerationTestcaseEntry]:
|
329
|
+
res: List[GenerationTestcaseEntry] = []
|
330
|
+
|
331
|
+
class ExtractGenerationTestcasesVisitor(TestcaseVisitor):
|
332
|
+
def should_visit_group(self, group_name: str) -> bool:
|
333
|
+
return any(pattern.intersecting_group(group_name) for pattern in patterns)
|
334
|
+
|
335
|
+
def should_visit_subgroup(self, subgroup_path: str) -> bool:
|
336
|
+
return any(
|
337
|
+
pattern.intersecting_group(subgroup_path) for pattern in patterns
|
338
|
+
)
|
339
|
+
|
340
|
+
def visit(self, entry: GenerationTestcaseEntry):
|
341
|
+
if not any(
|
342
|
+
pattern.match(entry.group_entry) for pattern in patterns
|
343
|
+
) and not any(pattern.match(entry.subgroup_entry) for pattern in patterns):
|
344
|
+
return
|
345
|
+
res.append(entry)
|
346
|
+
|
347
|
+
run_testcase_visitor(ExtractGenerationTestcasesVisitor())
|
348
|
+
return res
|
rbx/box/testcase_utils.py
CHANGED
@@ -70,20 +70,20 @@ class TestcasePattern(BaseModel):
|
|
70
70
|
|
71
71
|
def __str__(self) -> str:
|
72
72
|
prefix = '/'.join(self.group_prefix)
|
73
|
+
if not prefix:
|
74
|
+
return '*'
|
73
75
|
if self.index is None:
|
74
76
|
return f'{prefix}/'
|
75
77
|
return f'{prefix}/{self.index}'
|
76
78
|
|
77
79
|
@classmethod
|
78
80
|
def parse(cls, spec: str) -> 'TestcasePattern':
|
79
|
-
|
80
|
-
if
|
81
|
-
|
82
|
-
f'[error]Invalid testcase pattern [item]{spec}[/item].[/error]',
|
83
|
-
)
|
84
|
-
raise typer.Exit(1)
|
81
|
+
spec = spec.strip()
|
82
|
+
if spec == '*':
|
83
|
+
return cls(group_prefix=[], index=None)
|
85
84
|
|
86
|
-
|
85
|
+
parts = spec.split('/')
|
86
|
+
if len(parts) <= 1:
|
87
87
|
return cls(group_prefix=parts, index=None)
|
88
88
|
|
89
89
|
if parts[-1].isdigit():
|
@@ -133,3 +133,13 @@ def get_samples() -> List[Testcase]:
|
|
133
133
|
)
|
134
134
|
for tc in tcs
|
135
135
|
]
|
136
|
+
|
137
|
+
|
138
|
+
def fill_output_for_defined_testcase(testcase: Testcase) -> Testcase:
|
139
|
+
res = testcase.model_copy()
|
140
|
+
if res.outputPath is not None:
|
141
|
+
return res
|
142
|
+
output_path = res.inputPath.with_suffix('.ans')
|
143
|
+
if output_path.is_file():
|
144
|
+
res.outputPath = output_path
|
145
|
+
return res
|
rbx/box/testcases/main.py
CHANGED
@@ -7,11 +7,13 @@ from rbx import annotations, config, utils
|
|
7
7
|
from rbx.box import package
|
8
8
|
from rbx.box.generators import (
|
9
9
|
GenerationTestcaseEntry,
|
10
|
-
extract_generation_testcases,
|
11
|
-
extract_generation_testcases_from_patterns,
|
12
10
|
generate_outputs_for_testcases,
|
13
11
|
generate_standalone,
|
14
12
|
)
|
13
|
+
from rbx.box.testcase_extractors import (
|
14
|
+
extract_generation_testcases,
|
15
|
+
extract_generation_testcases_from_patterns,
|
16
|
+
)
|
15
17
|
from rbx.box.testcase_utils import TestcaseEntry, TestcasePattern
|
16
18
|
from rbx.console import console
|
17
19
|
|
@@ -116,19 +118,19 @@ def view(
|
|
116
118
|
items = _generate_for_editing(
|
117
119
|
testcase, input=not output_only, output=not input_only, progress=s
|
118
120
|
)
|
119
|
-
config.edit_multiple(items)
|
121
|
+
config.edit_multiple(items, readonly=True)
|
120
122
|
|
121
123
|
|
122
124
|
@app.command('info, i', help='Show information about testcases.')
|
123
125
|
def info(
|
124
126
|
pattern: Annotated[
|
125
|
-
str,
|
127
|
+
Optional[str],
|
126
128
|
typer.Argument(
|
127
129
|
help='Testcases to detail, as a pattern. Might be a group, or a specific test in the format [group]/[index].'
|
128
130
|
),
|
129
|
-
],
|
131
|
+
] = None,
|
130
132
|
):
|
131
|
-
tc_pattern = TestcasePattern.parse(pattern)
|
133
|
+
tc_pattern = TestcasePattern.parse(pattern or '*')
|
132
134
|
testcases = extract_generation_testcases_from_patterns([tc_pattern])
|
133
135
|
if not testcases:
|
134
136
|
console.print(
|
rbx/box/validators.py
CHANGED
@@ -9,7 +9,10 @@ from rbx import console
|
|
9
9
|
from rbx.box import package
|
10
10
|
from rbx.box.code import SanitizationLevel, compile_item, run_item
|
11
11
|
from rbx.box.schema import CodeItem, Primitive
|
12
|
-
from rbx.box.
|
12
|
+
from rbx.box.testcase_extractors import (
|
13
|
+
GenerationTestcaseEntry,
|
14
|
+
extract_generation_testcases_from_groups,
|
15
|
+
)
|
13
16
|
from rbx.grading.judge.sandbox import SandboxBase
|
14
17
|
from rbx.grading.steps import (
|
15
18
|
DigestHolder,
|
@@ -23,6 +26,7 @@ HitBounds = Dict[str, Tuple[bool, bool]]
|
|
23
26
|
|
24
27
|
|
25
28
|
class TestcaseValidationInfo(BaseModel):
|
29
|
+
validator: CodeItem
|
26
30
|
group: str
|
27
31
|
path: pathlib.Path
|
28
32
|
ok: bool
|
@@ -136,7 +140,7 @@ def _validate_testcase(
|
|
136
140
|
)
|
137
141
|
|
138
142
|
|
139
|
-
def
|
143
|
+
def _validate_test(
|
140
144
|
testcase: pathlib.Path,
|
141
145
|
validator: CodeItem,
|
142
146
|
validator_digest: str,
|
@@ -160,8 +164,9 @@ def validate_one_off(
|
|
160
164
|
validator: CodeItem,
|
161
165
|
validator_digest: str,
|
162
166
|
) -> TestcaseValidationInfo:
|
163
|
-
ok, message, _ =
|
167
|
+
ok, message, _ = _validate_test(testcase, validator, validator_digest)
|
164
168
|
info = TestcaseValidationInfo(
|
169
|
+
validator=validator,
|
165
170
|
group='interactive',
|
166
171
|
path=testcase,
|
167
172
|
ok=ok,
|
@@ -172,23 +177,29 @@ def validate_one_off(
|
|
172
177
|
|
173
178
|
|
174
179
|
def compile_validators(
|
180
|
+
validation_entries: List[GenerationTestcaseEntry],
|
175
181
|
progress: Optional[StatusProgress] = None,
|
176
182
|
) -> Dict[str, str]:
|
177
|
-
|
183
|
+
validators = []
|
184
|
+
|
185
|
+
for entry in validation_entries:
|
186
|
+
if entry.validator is not None:
|
187
|
+
validators.append(entry.validator)
|
188
|
+
validators.extend(entry.extra_validators)
|
178
189
|
|
179
|
-
|
190
|
+
validator_to_compiled_digest = {}
|
180
191
|
|
181
|
-
for
|
182
|
-
validator
|
183
|
-
if validator is None:
|
192
|
+
for validator in validators:
|
193
|
+
if str(validator.path) in validator_to_compiled_digest:
|
184
194
|
continue
|
195
|
+
|
185
196
|
if progress:
|
186
|
-
progress.update(
|
187
|
-
|
188
|
-
|
189
|
-
|
197
|
+
progress.update(f'Compiling validator [item]{validator.path}[/item]...')
|
198
|
+
validator_to_compiled_digest[str(validator.path)] = _compile_validator(
|
199
|
+
validator
|
200
|
+
)
|
190
201
|
|
191
|
-
return
|
202
|
+
return validator_to_compiled_digest
|
192
203
|
|
193
204
|
|
194
205
|
def validate_testcases(
|
@@ -199,38 +210,52 @@ def validate_testcases(
|
|
199
210
|
if progress is not None:
|
200
211
|
progress.step()
|
201
212
|
|
202
|
-
|
203
|
-
|
204
|
-
|
213
|
+
validation_entries = extract_generation_testcases_from_groups(groups)
|
214
|
+
validator_to_compiled_digest = compile_validators(
|
215
|
+
validation_entries, progress=progress
|
216
|
+
)
|
205
217
|
|
206
218
|
validation_info = []
|
207
219
|
|
208
|
-
for
|
209
|
-
|
210
|
-
if
|
211
|
-
continue
|
212
|
-
if group.name not in group_to_compiled_digest:
|
213
|
-
continue
|
214
|
-
if groups is not None and group.name not in groups:
|
220
|
+
for entry in validation_entries:
|
221
|
+
input_path = entry.metadata.copied_to.inputPath
|
222
|
+
if not input_path.is_file():
|
215
223
|
continue
|
216
|
-
compiled_digest = group_to_compiled_digest[group.name]
|
217
224
|
|
218
|
-
|
225
|
+
# Main validation.
|
226
|
+
if entry.validator is not None:
|
227
|
+
compiled_digest = validator_to_compiled_digest[str(entry.validator.path)]
|
228
|
+
ok, message, hit_bounds = _validate_test(
|
229
|
+
input_path, entry.validator, compiled_digest
|
230
|
+
)
|
231
|
+
validation_info.append(
|
232
|
+
TestcaseValidationInfo(
|
233
|
+
validator=entry.validator,
|
234
|
+
group=entry.group_entry.group,
|
235
|
+
path=input_path,
|
236
|
+
ok=ok,
|
237
|
+
hit_bounds=hit_bounds,
|
238
|
+
message=message,
|
239
|
+
)
|
240
|
+
)
|
219
241
|
|
220
|
-
for
|
221
|
-
|
222
|
-
|
242
|
+
for extra_validator in entry.extra_validators:
|
243
|
+
compiled_digest = validator_to_compiled_digest[str(extra_validator.path)]
|
244
|
+
ok, message, hit_bounds = _validate_test(
|
245
|
+
input_path, extra_validator, compiled_digest
|
223
246
|
)
|
224
247
|
validation_info.append(
|
225
248
|
TestcaseValidationInfo(
|
226
|
-
|
227
|
-
|
249
|
+
validator=extra_validator,
|
250
|
+
group=entry.group_entry.group,
|
251
|
+
path=input_path,
|
228
252
|
ok=ok,
|
229
253
|
hit_bounds=hit_bounds,
|
230
254
|
message=message,
|
231
255
|
)
|
232
256
|
)
|
233
|
-
|
257
|
+
|
258
|
+
step()
|
234
259
|
|
235
260
|
return validation_info
|
236
261
|
|
@@ -242,11 +267,14 @@ def has_validation_errors(infos: List[TestcaseValidationInfo]) -> bool:
|
|
242
267
|
def print_validation_report(infos: List[TestcaseValidationInfo]):
|
243
268
|
console.console.rule('Validation report', style='status')
|
244
269
|
hit_bounds_per_group: Dict[Optional[str], HitBounds] = {}
|
270
|
+
any_failure = False
|
245
271
|
for info in infos:
|
246
272
|
if not info.ok:
|
247
273
|
console.console.print(
|
248
|
-
f'[error]Testcase [item]{info.path}[/item] failed verification
|
274
|
+
f'[error]Testcase [item]{info.path}[/item] failed verification on validator [item]{info.validator.path}[/item]:[/error]'
|
249
275
|
)
|
276
|
+
console.console.print(info.message)
|
277
|
+
any_failure = True
|
250
278
|
continue
|
251
279
|
|
252
280
|
if info.group not in hit_bounds_per_group:
|
@@ -278,7 +306,7 @@ def print_validation_report(infos: List[TestcaseValidationInfo]):
|
|
278
306
|
# If there's only the samples group, do not check for hit bounds.
|
279
307
|
hit_bounds_per_group = {}
|
280
308
|
|
281
|
-
if not hit_bounds_per_group:
|
309
|
+
if not hit_bounds_per_group and not any_failure:
|
282
310
|
console.console.print('[info]No validation issues found.[/info]')
|
283
311
|
return
|
284
312
|
|
rbx/config.py
CHANGED
@@ -5,6 +5,7 @@ import os
|
|
5
5
|
import pathlib
|
6
6
|
import shutil
|
7
7
|
import subprocess
|
8
|
+
import tempfile
|
8
9
|
from typing import Any, Dict, List, Optional
|
9
10
|
|
10
11
|
import requests
|
@@ -227,10 +228,23 @@ def open_editor(path: Any, *args):
|
|
227
228
|
subprocess.run([editor, str(path), *[str(arg) for arg in args]])
|
228
229
|
|
229
230
|
|
230
|
-
def
|
231
|
+
def _readonly_copy(path: pathlib.Path) -> pathlib.Path:
|
232
|
+
temp_file = tempfile.NamedTemporaryFile(mode='w', delete=False)
|
233
|
+
shutil.copy(str(path), temp_file.name)
|
234
|
+
temp_file.close()
|
235
|
+
return pathlib.Path(temp_file.name)
|
236
|
+
|
237
|
+
|
238
|
+
def edit_multiple(paths: List[pathlib.Path], readonly: bool = False):
|
231
239
|
if is_vim_editor():
|
232
|
-
|
240
|
+
if readonly:
|
241
|
+
open_editor('-R', '-O', *paths)
|
242
|
+
else:
|
243
|
+
open_editor('-O', *paths)
|
233
244
|
return
|
245
|
+
|
246
|
+
if readonly:
|
247
|
+
paths = [_readonly_copy(path) for path in paths]
|
234
248
|
open_editor(*paths)
|
235
249
|
|
236
250
|
|
@@ -2,7 +2,7 @@ rbx/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
2
|
rbx/annotations.py,sha256=Z3jBUyZoXkrz34jko3Rft0bnMME6nWb0vsV5I3HlgR0,3064
|
3
3
|
rbx/autoenum.py,sha256=cusv8ClXRlDVvhZ8eDrtYcL_2peXlHugAey_ht8roXk,12025
|
4
4
|
rbx/box/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
5
|
-
rbx/box/builder.py,sha256=
|
5
|
+
rbx/box/builder.py,sha256=qIXgV-div21Tw8knwCrTtHyDCgYwBrJc0I5b9KhZuKM,3577
|
6
6
|
rbx/box/cd.py,sha256=9a_SOnzoJBXxxffp4Wbf3UKXIwKuN3Hvj7K6SocALwE,1194
|
7
7
|
rbx/box/checkers.py,sha256=VpgDzevOK7hrffG2zJGxquNiu-a9Fl3wquLn7xadcK0,6285
|
8
8
|
rbx/box/code.py,sha256=UFy7jOeTvxtIu9pdVUDv2-D6IW-beJGPC3uCanIKZh0,13412
|
@@ -12,19 +12,19 @@ rbx/box/contest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
12
12
|
rbx/box/contest/build_contest_statements.py,sha256=H2MwmkiPO_cHUEenzfPxHuJ3XcwjHakGZwKojNJQt74,11380
|
13
13
|
rbx/box/contest/contest_package.py,sha256=OaUbpBtkhkgOPzJ1ccI_Vq4FMSaJvZm3gMOKfVY8oy4,3032
|
14
14
|
rbx/box/contest/contest_utils.py,sha256=TDE7I6YQJlu4dQd68wzOp019bNgqiT0RlM-LMQMjL9w,301
|
15
|
-
rbx/box/contest/main.py,sha256=
|
15
|
+
rbx/box/contest/main.py,sha256=oL-GbyLKdpMjIWiSuWTQgRhQ9hcb7DuNn0axkunx0io,7436
|
16
16
|
rbx/box/contest/schema.py,sha256=JMAig5WpaOahNgAHxA9vX4zYeVYDxpjKP_PFGvmmkE0,4954
|
17
17
|
rbx/box/contest/statements.py,sha256=Pe4uo1hxvEON8O11VAzsOP3DxUel0vmwiAmolh4ltEs,2910
|
18
|
-
rbx/box/creation.py,sha256=
|
18
|
+
rbx/box/creation.py,sha256=Evz7K6JoarD-4JJQsZsgoxU9FgCF9Z7-LfuroG4Cqls,2444
|
19
19
|
rbx/box/deferred.py,sha256=II3X9e87JCOZtmspnHh-n4PFqh-FsH_oc0XJHZ9ZYVQ,691
|
20
20
|
rbx/box/download.py,sha256=MFP-R26JiYGAP89I0TK-0fYc69Fsd20tsBqgtRCy5AE,2234
|
21
21
|
rbx/box/environment.py,sha256=47NtyuVC6zSQKAtQaXPEXvqcD-KJiuWRpWF8pYvcG4c,11158
|
22
22
|
rbx/box/extensions.py,sha256=Von8kIeXvNFTkGlMRMTvL2HIHPwlkuiMswr-ydbGV1w,519
|
23
23
|
rbx/box/formatting.py,sha256=3phFRHzqVXj4Ok1yDhCq6Clbw6KlqwJNpMhs--oTWFI,405
|
24
|
-
rbx/box/generators.py,sha256=
|
25
|
-
rbx/box/generators_test.py,sha256=
|
26
|
-
rbx/box/main.py,sha256=
|
27
|
-
rbx/box/package.py,sha256=
|
24
|
+
rbx/box/generators.py,sha256=nJx4U0Cd5DgHIf1n-06c03hyLi5AVdH9tvi0Hnh8iQ8,12158
|
25
|
+
rbx/box/generators_test.py,sha256=ZRqdolU7YE8HXjxr0met5oGn4DCJ5erdsMt5cSOoXIw,1945
|
26
|
+
rbx/box/main.py,sha256=0lrPLJTGvatcGZCYgLRKKaRkWKEFmNKyA48Shm-uof8,24308
|
27
|
+
rbx/box/package.py,sha256=80SDHvSzfraCUYutMn_kwsFsmmrSZiaeRHhhrWGmIY4,12081
|
28
28
|
rbx/box/packaging/boca/extension.py,sha256=hQhcbocNfW2ESv5RalS1wf6uvOoOfOnR_gHvbXUbSzY,852
|
29
29
|
rbx/box/packaging/boca/packager.py,sha256=FOhSRg5K5Y4qNB0WyTR3DKgrpObf9I0JbyGpJHOtxpo,10673
|
30
30
|
rbx/box/packaging/contest_main.py,sha256=Hbxh7geNqrePs5tWhPgdg5W2qhaW5yoreK_VP0Sm19k,2727
|
@@ -39,10 +39,10 @@ rbx/box/presets/lock_schema.py,sha256=6sRPnyePOC8yy-5WcD5JRZdDJHf8loqbvpQ1IPiOU9
|
|
39
39
|
rbx/box/presets/schema.py,sha256=mZmSPkQsw7eQM0lQN6er1MO_LiW1ObwwAZFDK0F5fxE,1962
|
40
40
|
rbx/box/retries.py,sha256=z7cIh1QmLVUsTr3Attt_28dbwNg6KWTwpulcWCFwMPo,4667
|
41
41
|
rbx/box/sanitizers/warning_stack.py,sha256=RI97_GJgdjTKIXY_r0EKp5h0qQQSDSdNDh5K7zINrqs,2861
|
42
|
-
rbx/box/schema.py,sha256=
|
42
|
+
rbx/box/schema.py,sha256=I7Uh_KXBqAX8fHZr4s9LGPEFHxyBttoLSq_hYJefwto,14581
|
43
43
|
rbx/box/setter_config.py,sha256=ZM7_G2tbaixaFr0NvRaXkowwfxSWF2Gb4XHBsr2Prpc,4279
|
44
|
-
rbx/box/solutions.py,sha256=
|
45
|
-
rbx/box/solutions_test.py,sha256=
|
44
|
+
rbx/box/solutions.py,sha256=mXW1o3uGNFYmQhfZJzIxfs7nCR7WfWtJtLypLFGriuE,44014
|
45
|
+
rbx/box/solutions_test.py,sha256=txjAg-n_pkHHolw4WF4foBrpJAL-llAXw6fUIrGURMc,1716
|
46
46
|
rbx/box/state.py,sha256=yTpjfASpnSXkRB3JiDNvCg5b9JNnNxuYT4uMcbdr59s,109
|
47
47
|
rbx/box/statements/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
48
48
|
rbx/box/statements/build_statements.py,sha256=qR6WxUNvSQTBs241qH-qDRbD8IoN_hKeZ2TwY7NXJBQ,12024
|
@@ -55,19 +55,20 @@ rbx/box/stresses.py,sha256=ceFpkZVKBfKKVrKFjeARdub5VGKmU9JPZwj-FxcqYjQ,11771
|
|
55
55
|
rbx/box/stressing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
56
56
|
rbx/box/stressing/finder_parser.py,sha256=jXpYNa4FyugzmHi3r96Uv4rU1krRQJc5Ihr9jf1cvNo,11918
|
57
57
|
rbx/box/stressing/generator_parser.py,sha256=oHZryjR3YohgaSO9WEirQ7b2e-98WgZStF0N99W4Thw,7380
|
58
|
-
rbx/box/
|
58
|
+
rbx/box/testcase_extractors.py,sha256=jh75iTaJ_8TVVGldcdNhQe999GFOwSDUDOccvagDqLw,11745
|
59
|
+
rbx/box/testcase_utils.py,sha256=qtv7-bJbbblMgINvcf_3YTdD85MTtWpD23KUSZUL1as,4327
|
59
60
|
rbx/box/testcases/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
60
|
-
rbx/box/testcases/main.py,sha256=
|
61
|
+
rbx/box/testcases/main.py,sha256=vDj7ErK0Y5jUGrwGvDIisqSm5uObKoeknbP5gPxhgzU,5256
|
61
62
|
rbx/box/ui/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
62
63
|
rbx/box/ui/captured_log.py,sha256=ptICDPViVnz-_2NfrcB0SSBXNW5L74zI-vAZNN7kSok,11319
|
63
64
|
rbx/box/ui/css/app.tcss,sha256=apd5PkPEvl5jK3kE2qrxPyVED1VnvSsj08QQwzUPwEA,786
|
64
65
|
rbx/box/ui/main.py,sha256=b0rHcBF42W4AOCv7WhtiGf_rUnY0yxpqO5oj3wfR4R4,984
|
65
66
|
rbx/box/ui/run.py,sha256=wMEXrEFdQvMHz2hRKAFIithTnTtaL0kNQZu0jKmb8jI,7060
|
66
|
-
rbx/box/validators.py,sha256=
|
67
|
+
rbx/box/validators.py,sha256=WX6PR-eVXm9ghv0cJYYhoe9eyQJDZrkXoK6p_Ya_BY0,10106
|
67
68
|
rbx/box/validators_test.py,sha256=hriR6rD32Ouu64eKYYTPLZVvqMxXj7Q2h1l_JAefL7U,344
|
68
69
|
rbx/checker.py,sha256=pj1jO3my48ru-qugbER5onccANCjoR0-PaFe3H3VGEY,4118
|
69
70
|
rbx/clone.py,sha256=wpHyED0_7ST7LD3vj7HjXhzqEzlwh6dRQvKQVDYhGeU,6744
|
70
|
-
rbx/config.py,sha256=
|
71
|
+
rbx/config.py,sha256=78xKH0zddEF32uIbIs10snqvACx20DmzjQTCex7w95Y,8136
|
71
72
|
rbx/conftest.py,sha256=ouilbOIpvX8jTEdCAiWT85CbdBQKUUf41BjmDI82u-Y,967
|
72
73
|
rbx/console.py,sha256=X8EJy68OROgh6ao3ZcUjZm5Y56VFMzen58ywAuQ7pAU,990
|
73
74
|
rbx/create.py,sha256=ezUq9KiSA-88ASd8CtjWXw8UB4LCaQ3Gib3OgvsLK-Q,986
|
@@ -167,8 +168,8 @@ rbx/testdata/caching/executable.py,sha256=WKRHNf_fprFJd1Fq1ubmQtR3mZzTYVNwKPLWuZ
|
|
167
168
|
rbx/testdata/compatible,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
168
169
|
rbx/testing_utils.py,sha256=ZZLKMUHlZ4HwsuNY50jqSBJ9HhpnFdba7opjDsvXE1U,2084
|
169
170
|
rbx/utils.py,sha256=AITbkWpWtSp-x3Xept_aObfj_jPL7XL0JJoz5-F9Fp8,4671
|
170
|
-
rbx_cp-0.5.
|
171
|
-
rbx_cp-0.5.
|
172
|
-
rbx_cp-0.5.
|
173
|
-
rbx_cp-0.5.
|
174
|
-
rbx_cp-0.5.
|
171
|
+
rbx_cp-0.5.37.dist-info/LICENSE,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
|
172
|
+
rbx_cp-0.5.37.dist-info/METADATA,sha256=-52cZu24PhrGmSfudYYYgw2R-Pfxlmk5YAa1BB96Vw0,3263
|
173
|
+
rbx_cp-0.5.37.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
|
174
|
+
rbx_cp-0.5.37.dist-info/entry_points.txt,sha256=qBTLBOeifT1F00LWaEewRRE_jQPgvH7BUdJfZ-dYsFU,57
|
175
|
+
rbx_cp-0.5.37.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|