rbx.cp 0.5.40__py3-none-any.whl → 0.5.45__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rbx/box/builder.py +6 -6
- rbx/box/checkers.py +100 -25
- rbx/box/cli.py +868 -0
- rbx/box/code.py +272 -84
- rbx/box/contest/statements.py +4 -2
- rbx/box/generators.py +55 -49
- rbx/box/generators_test.py +7 -7
- rbx/box/main.py +1 -868
- rbx/box/package.py +57 -2
- rbx/box/packaging/boca/packager.py +2 -1
- rbx/box/packaging/main.py +17 -9
- rbx/box/packaging/moj/packager.py +49 -10
- rbx/box/retries.py +5 -5
- rbx/box/schema.py +20 -4
- rbx/box/solutions.py +46 -108
- rbx/box/solutions_test.py +5 -6
- rbx/box/state.py +1 -0
- rbx/box/statements/build_statements.py +4 -2
- rbx/box/stresses.py +23 -12
- rbx/box/tasks.py +277 -0
- rbx/box/testcase_extractors.py +21 -21
- rbx/box/testcases/main.py +19 -14
- rbx/box/unit.py +10 -7
- rbx/box/validators.py +10 -10
- rbx/box/validators_test.py +3 -3
- rbx/grading/judge/cacher.py +0 -4
- rbx/grading/judge/digester.py +0 -3
- rbx/grading/judge/sandbox.py +15 -0
- rbx/grading/judge/sandboxes/stupid_sandbox.py +20 -6
- rbx/grading/judge/sandboxes/timeit.py +117 -7
- rbx/grading/judge/storage.py +0 -4
- rbx/grading/steps.py +76 -2
- rbx/grading/steps_with_caching.py +45 -3
- rbx/grading/steps_with_caching_run_test.py +51 -49
- rbx/main.py +0 -4
- rbx/resources/packagers/moj/scripts/compare.sh +25 -6
- rbx/test.py +6 -4
- {rbx_cp-0.5.40.dist-info → rbx_cp-0.5.45.dist-info}/METADATA +2 -2
- {rbx_cp-0.5.40.dist-info → rbx_cp-0.5.45.dist-info}/RECORD +42 -55
- {rbx_cp-0.5.40.dist-info → rbx_cp-0.5.45.dist-info}/WHEEL +1 -1
- rbx/testdata/box1/gen1.cpp +0 -7
- rbx/testdata/box1/gen2.cpp +0 -9
- rbx/testdata/box1/genScript.py +0 -2
- rbx/testdata/box1/hard-tle.sol.cpp +0 -26
- rbx/testdata/box1/ole.cpp +0 -17
- rbx/testdata/box1/problem.rbx.yml +0 -39
- rbx/testdata/box1/re.sol.cpp +0 -23
- rbx/testdata/box1/sol.cpp +0 -22
- rbx/testdata/box1/tests/1.in +0 -1
- rbx/testdata/box1/tle-and-incorrect.sol.cpp +0 -33
- rbx/testdata/box1/tle.sol.cpp +0 -35
- rbx/testdata/box1/validator.cpp +0 -11
- rbx/testdata/box1/wa.sol.cpp +0 -22
- rbx/testdata/caching/executable.py +0 -1
- rbx/testdata/compatible +0 -0
- {rbx_cp-0.5.40.dist-info → rbx_cp-0.5.45.dist-info}/LICENSE +0 -0
- {rbx_cp-0.5.40.dist-info → rbx_cp-0.5.45.dist-info}/entry_points.txt +0 -0
rbx/box/package.py
CHANGED
@@ -1,5 +1,8 @@
|
|
1
|
+
import atexit
|
1
2
|
import functools
|
3
|
+
import os
|
2
4
|
import pathlib
|
5
|
+
import shutil
|
3
6
|
import sys
|
4
7
|
from typing import Dict, List, Optional, Tuple
|
5
8
|
|
@@ -201,7 +204,9 @@ def get_digest_as_string(
|
|
201
204
|
|
202
205
|
|
203
206
|
def get_new_sandbox(root: pathlib.Path = pathlib.Path()) -> SandboxBase:
|
204
|
-
|
207
|
+
sandbox = get_sandbox_type()(file_cacher=get_file_cacher(root), temp_dir=TEMP_DIR)
|
208
|
+
atexit.register(lambda: sandbox.cleanup(delete=True))
|
209
|
+
return sandbox
|
205
210
|
|
206
211
|
|
207
212
|
@functools.cache
|
@@ -209,6 +214,13 @@ def get_singleton_sandbox(root: pathlib.Path = pathlib.Path()) -> SandboxBase:
|
|
209
214
|
return get_new_sandbox(root)
|
210
215
|
|
211
216
|
|
217
|
+
@functools.cache
|
218
|
+
def get_singleton_interactor_sandbox(
|
219
|
+
root: pathlib.Path = pathlib.Path(),
|
220
|
+
) -> SandboxBase:
|
221
|
+
return get_new_sandbox(root)
|
222
|
+
|
223
|
+
|
212
224
|
@functools.cache
|
213
225
|
def get_build_path(root: pathlib.Path = pathlib.Path()) -> pathlib.Path:
|
214
226
|
return find_problem(root) / 'build'
|
@@ -266,6 +278,23 @@ def get_checker(root: pathlib.Path = pathlib.Path()) -> CodeItem:
|
|
266
278
|
)
|
267
279
|
|
268
280
|
|
281
|
+
@functools.cache
|
282
|
+
def get_interactor_or_nil(root: pathlib.Path = pathlib.Path()) -> Optional[CodeItem]:
|
283
|
+
package = find_problem_package_or_die(root)
|
284
|
+
return package.interactor
|
285
|
+
|
286
|
+
|
287
|
+
@functools.cache
|
288
|
+
def get_interactor(root: pathlib.Path = pathlib.Path()) -> CodeItem:
|
289
|
+
interactor = get_interactor_or_nil(root)
|
290
|
+
if interactor is None:
|
291
|
+
console.console.print(
|
292
|
+
'[error]Problem does not have an interactor configured.[/error]'
|
293
|
+
)
|
294
|
+
raise typer.Exit(1)
|
295
|
+
return interactor
|
296
|
+
|
297
|
+
|
269
298
|
@functools.cache
|
270
299
|
def get_solutions(root: pathlib.Path = pathlib.Path()) -> List[Solution]:
|
271
300
|
package = find_problem_package_or_die(root)
|
@@ -364,9 +393,35 @@ def get_compilation_files(code: CodeItem) -> List[Tuple[pathlib.Path, pathlib.Pa
|
|
364
393
|
return res
|
365
394
|
|
366
395
|
|
396
|
+
@functools.cache
|
397
|
+
def get_shared_dir(root: pathlib.Path = pathlib.Path()) -> pathlib.Path:
|
398
|
+
shared_dir = get_problem_cache_dir(root) / '.shared'
|
399
|
+
shared_dir.mkdir(parents=True, exist_ok=True)
|
400
|
+
return shared_dir
|
401
|
+
|
402
|
+
|
367
403
|
@functools.cache
|
368
404
|
def get_empty_sentinel_path(root: pathlib.Path = pathlib.Path()) -> pathlib.Path:
|
369
|
-
path =
|
405
|
+
path = get_shared_dir(root) / '.empty'
|
406
|
+
path.write_text('')
|
407
|
+
return path
|
408
|
+
|
409
|
+
|
410
|
+
@functools.cache
|
411
|
+
def get_fifos(root: pathlib.Path = pathlib.Path()) -> Tuple[pathlib.Path, pathlib.Path]:
|
412
|
+
path = get_shared_dir(root) / '.fifos'
|
413
|
+
shutil.rmtree(path, ignore_errors=True)
|
414
|
+
path.mkdir(parents=True, exist_ok=True)
|
415
|
+
fifo_in = path / 'fifo.in'
|
416
|
+
fifo_out = path / 'fifo.out'
|
417
|
+
os.mkfifo(fifo_in)
|
418
|
+
os.mkfifo(fifo_out)
|
419
|
+
return fifo_in, fifo_out
|
420
|
+
|
421
|
+
|
422
|
+
@functools.cache
|
423
|
+
def get_merged_capture_path(root: pathlib.Path = pathlib.Path()) -> pathlib.Path:
|
424
|
+
path = get_shared_dir(root) / '.merged_capture'
|
370
425
|
path.write_text('')
|
371
426
|
return path
|
372
427
|
|
@@ -169,7 +169,8 @@ class BocaPackager(BasePackager):
|
|
169
169
|
compile_text = compile_text.replace('{{rbxFlags}}', flags[language])
|
170
170
|
return compile_text
|
171
171
|
|
172
|
-
def _copy_solutions(self, into_path: pathlib.Path):
|
172
|
+
def _copy_solutions(self, into_path: pathlib.Path, fix_java: bool = True):
|
173
|
+
into_path = into_path / 'solutions'
|
173
174
|
for solution in package.get_solutions():
|
174
175
|
dest_path = (
|
175
176
|
into_path
|
rbx/box/packaging/main.py
CHANGED
@@ -2,6 +2,7 @@ import pathlib
|
|
2
2
|
import tempfile
|
3
3
|
from typing import Type
|
4
4
|
|
5
|
+
import syncer
|
5
6
|
import typer
|
6
7
|
|
7
8
|
from rbx import annotations, console
|
@@ -13,20 +14,21 @@ from rbx.box.statements.build_statements import build_statement
|
|
13
14
|
app = typer.Typer(no_args_is_help=True, cls=annotations.AliasGroup)
|
14
15
|
|
15
16
|
|
16
|
-
def run_packager(
|
17
|
+
async def run_packager(
|
17
18
|
packager_cls: Type[BasePackager],
|
18
19
|
verification: environment.VerificationParam,
|
20
|
+
**kwargs,
|
19
21
|
) -> pathlib.Path:
|
20
22
|
from rbx.box import builder
|
21
23
|
|
22
|
-
if not builder.verify(verification=verification):
|
24
|
+
if not await builder.verify(verification=verification):
|
23
25
|
console.console.print(
|
24
26
|
'[error]Build or verification failed, check the report.[/error]'
|
25
27
|
)
|
26
28
|
raise typer.Exit(1)
|
27
29
|
|
28
30
|
pkg = package.find_problem_package_or_die()
|
29
|
-
packager = packager_cls()
|
31
|
+
packager = packager_cls(**kwargs)
|
30
32
|
|
31
33
|
statement_types = packager.statement_types()
|
32
34
|
built_statements = []
|
@@ -55,27 +57,33 @@ def run_packager(
|
|
55
57
|
|
56
58
|
|
57
59
|
@app.command('polygon', help='Build a package for Polygon.')
|
58
|
-
|
60
|
+
@syncer.sync
|
61
|
+
async def polygon(
|
59
62
|
verification: environment.VerificationParam,
|
60
63
|
):
|
61
64
|
from rbx.box.packaging.polygon.packager import PolygonPackager
|
62
65
|
|
63
|
-
run_packager(PolygonPackager, verification=verification)
|
66
|
+
await run_packager(PolygonPackager, verification=verification)
|
64
67
|
|
65
68
|
|
66
69
|
@app.command('boca', help='Build a package for BOCA.')
|
67
|
-
|
70
|
+
@syncer.sync
|
71
|
+
async def boca(
|
68
72
|
verification: environment.VerificationParam,
|
69
73
|
):
|
70
74
|
from rbx.box.packaging.boca.packager import BocaPackager
|
71
75
|
|
72
|
-
run_packager(BocaPackager, verification=verification)
|
76
|
+
await run_packager(BocaPackager, verification=verification)
|
73
77
|
|
74
78
|
|
75
79
|
@app.command('moj', help='Build a package for MOJ.')
|
76
|
-
|
80
|
+
@syncer.sync
|
81
|
+
async def moj(
|
77
82
|
verification: environment.VerificationParam,
|
83
|
+
for_boca: bool = typer.Option(
|
84
|
+
False, help='Build a package for BOCA instead of MOJ.'
|
85
|
+
),
|
78
86
|
):
|
79
87
|
from rbx.box.packaging.moj.packager import MojPackager
|
80
88
|
|
81
|
-
run_packager(MojPackager, verification=verification)
|
89
|
+
await run_packager(MojPackager, verification=verification, for_boca=for_boca)
|
@@ -10,11 +10,16 @@ from rbx.box.environment import get_extension_or_default
|
|
10
10
|
from rbx.box.packaging.boca.extension import BocaExtension
|
11
11
|
from rbx.box.packaging.boca.packager import BocaPackager
|
12
12
|
from rbx.box.packaging.packager import BuiltStatement
|
13
|
-
from rbx.
|
13
|
+
from rbx.box.schema import ExpectedOutcome
|
14
|
+
from rbx.config import get_default_app_path, get_testlib
|
14
15
|
from rbx.grading.judge.digester import digest_cooperatively
|
15
16
|
|
16
17
|
|
17
18
|
class MojPackager(BocaPackager):
|
19
|
+
def __init__(self, for_boca: bool = False):
|
20
|
+
super().__init__()
|
21
|
+
self.for_boca = for_boca
|
22
|
+
|
18
23
|
def _get_problem_info(self) -> str:
|
19
24
|
statement = self._get_main_statement()
|
20
25
|
return (
|
@@ -23,17 +28,20 @@ class MojPackager(BocaPackager):
|
|
23
28
|
f'descfile={self._get_problem_name()}.pdf\n'
|
24
29
|
)
|
25
30
|
|
26
|
-
def
|
31
|
+
def _get_tl(self) -> str:
|
27
32
|
extension = get_extension_or_default('boca', BocaExtension)
|
28
33
|
|
29
34
|
pkg = package.find_problem_package_or_die()
|
30
|
-
|
35
|
+
res = f'TL[default]={pkg.timeLimit / 1000}\n'
|
36
|
+
for language in extension.languages:
|
37
|
+
res += f'TL[{language}]={self._get_pkg_timelimit(language) / 1000}\n'
|
38
|
+
return res
|
39
|
+
|
40
|
+
def _get_limits(self) -> str:
|
41
|
+
pkg = package.find_problem_package_or_die()
|
31
42
|
ml = pkg.memoryLimit
|
32
43
|
ol = pkg.outputLimit
|
33
|
-
|
34
|
-
for language in extension.languages:
|
35
|
-
conf += f'TL[{language}]={self._get_pkg_timelimit(language) / 1000}\n'
|
36
|
-
return conf
|
44
|
+
return f'ULIMITS[-f]={ol}\n' f'ULIMITS[-v]={ml * 1024}\n'
|
37
45
|
|
38
46
|
def _get_compare(self) -> str:
|
39
47
|
extension = get_extension_or_default('boca', BocaExtension)
|
@@ -57,6 +65,24 @@ class MojPackager(BocaPackager):
|
|
57
65
|
def _get_checker(self) -> str:
|
58
66
|
return package.get_checker().path.read_text()
|
59
67
|
|
68
|
+
def _copy_solutions_moj(self, into_path: pathlib.Path):
|
69
|
+
into_path = into_path / 'sols'
|
70
|
+
has_good = False
|
71
|
+
for solution in package.get_solutions():
|
72
|
+
tag = 'wrong'
|
73
|
+
if solution.outcome == ExpectedOutcome.ACCEPTED:
|
74
|
+
tag = 'good'
|
75
|
+
has_good = True
|
76
|
+
elif solution.outcome.is_slow():
|
77
|
+
tag = 'slow'
|
78
|
+
dest_path = into_path / tag / solution.path.name
|
79
|
+
dest_path.parent.mkdir(parents=True, exist_ok=True)
|
80
|
+
shutil.copy(str(solution.path), dest_path)
|
81
|
+
|
82
|
+
if not has_good:
|
83
|
+
console.console.print('[error]No good solution found.[/error]')
|
84
|
+
raise typer.Exit(1)
|
85
|
+
|
60
86
|
def name(self) -> str:
|
61
87
|
return 'moj'
|
62
88
|
|
@@ -80,10 +106,22 @@ class MojPackager(BocaPackager):
|
|
80
106
|
limits_path.parent.mkdir(parents=True, exist_ok=True)
|
81
107
|
limits_path.write_text(self._get_limits())
|
82
108
|
|
109
|
+
# Prepare TL
|
110
|
+
if self.for_boca:
|
111
|
+
tl_path = into_path / 'tl'
|
112
|
+
tl_path.parent.mkdir(parents=True, exist_ok=True)
|
113
|
+
tl_path.write_text(self._get_tl())
|
114
|
+
|
83
115
|
# Prepare compare
|
84
116
|
compare_path = into_path / 'scripts' / 'compare.sh'
|
85
117
|
compare_path.parent.mkdir(parents=True, exist_ok=True)
|
86
118
|
compare_path.write_text(self._get_compare())
|
119
|
+
compare_path.chmod(0o755)
|
120
|
+
|
121
|
+
# Prepare testlib
|
122
|
+
testlib_path = into_path / 'scripts' / 'testlib.h'
|
123
|
+
testlib_path.parent.mkdir(parents=True, exist_ok=True)
|
124
|
+
testlib_path.write_text(get_testlib().read_text())
|
87
125
|
|
88
126
|
# Prepare checker
|
89
127
|
checker_path = into_path / 'scripts' / 'checker.cpp'
|
@@ -99,9 +137,10 @@ class MojPackager(BocaPackager):
|
|
99
137
|
)
|
100
138
|
|
101
139
|
# Copy solutions
|
102
|
-
|
103
|
-
|
104
|
-
|
140
|
+
if self.for_boca:
|
141
|
+
self._copy_solutions(into_path, fix_java=False)
|
142
|
+
else:
|
143
|
+
self._copy_solutions_moj(into_path)
|
105
144
|
|
106
145
|
# Prepare IO
|
107
146
|
inputs_path = into_path / 'tests' / 'input'
|
rbx/box/retries.py
CHANGED
@@ -3,7 +3,7 @@ import pathlib
|
|
3
3
|
import shutil
|
4
4
|
import tempfile
|
5
5
|
from contextlib import contextmanager
|
6
|
-
from typing import Callable, List, Optional
|
6
|
+
from typing import Awaitable, Callable, List, Optional
|
7
7
|
|
8
8
|
from rbx.box import package
|
9
9
|
from rbx.box.setter_config import RepeatsConfig, get_setter_config
|
@@ -104,18 +104,18 @@ class Retrier:
|
|
104
104
|
self.retries_for_stress = self.config.retries_for_stress
|
105
105
|
self.retry_index = 0
|
106
106
|
|
107
|
-
def repeat(
|
107
|
+
async def repeat(
|
108
108
|
self,
|
109
|
-
func: Callable[[int], Evaluation],
|
109
|
+
func: Callable[[int], Awaitable[Evaluation]],
|
110
110
|
) -> Evaluation:
|
111
111
|
self.retry_index += 1
|
112
|
-
eval = func(self.retry_index)
|
112
|
+
eval = await func(self.retry_index)
|
113
113
|
if self.should_repeat(eval):
|
114
114
|
with _temp_retry_dir() as temp_dir:
|
115
115
|
# Move files to temp dir to open run for repeat.
|
116
116
|
recover = _move_logs_to_temp_dir(eval, temp_dir)
|
117
117
|
# Actually repeat and choose the best evaluation.
|
118
|
-
next_eval = self.repeat(func)
|
118
|
+
next_eval = await self.repeat(func)
|
119
119
|
chosen_eval = _merge_evaluations(eval, next_eval)
|
120
120
|
|
121
121
|
if id(chosen_eval) == id(eval):
|
rbx/box/schema.py
CHANGED
@@ -82,16 +82,16 @@ class ExpectedOutcome(AutoEnum):
|
|
82
82
|
RUNTIME_ERROR = alias('runtime error', 'rte', 're') # type: ignore
|
83
83
|
"""Expected outcome solutions that finish with non-zero code (RTE)."""
|
84
84
|
|
85
|
-
TIME_LIMIT_EXCEEDED = alias('time limit exceeded', 'timeout', 'tle') # type: ignore
|
85
|
+
TIME_LIMIT_EXCEEDED = alias('time limit exceeded', 'timeout', 'tle', 'tl') # type: ignore
|
86
86
|
"""Expected outcome for solutions that do not finish in time."""
|
87
87
|
|
88
|
-
MEMORY_LIMIT_EXCEEDED = alias('memory limit exceeded', 'mle') # type: ignore
|
88
|
+
MEMORY_LIMIT_EXCEEDED = alias('memory limit exceeded', 'mle', 'ml') # type: ignore
|
89
89
|
"""Expected outcome for solutions that use more memory than allowed."""
|
90
90
|
|
91
|
-
OUTPUT_LIMIT_EXCEEDED = alias('output limit exceeded', 'ole') # type: ignore
|
91
|
+
OUTPUT_LIMIT_EXCEEDED = alias('output limit exceeded', 'ole', 'ol') # type: ignore
|
92
92
|
"""Expected outcome for solutions that use more output than allowed."""
|
93
93
|
|
94
|
-
TLE_OR_RTE = alias('tle or rte', 'tle/rte', 'tle+rte') # type: ignore
|
94
|
+
TLE_OR_RTE = alias('tle or rte', 'tle/rte', 'tle+rte', 'tle or re', 'tle+re') # type: ignore
|
95
95
|
"""Expected outcome for solutions that finish with either TLE or RTE.
|
96
96
|
|
97
97
|
Especially useful for environments where TLE and RTE are indistinguishable."""
|
@@ -156,6 +156,14 @@ class ValidatorOutcome(AutoEnum):
|
|
156
156
|
"""Expected outcome for invalid tests."""
|
157
157
|
|
158
158
|
|
159
|
+
class TaskType(AutoEnum):
|
160
|
+
BATCH = alias('batch') # type: ignore
|
161
|
+
"""Batch task."""
|
162
|
+
|
163
|
+
COMMUNICATION = alias('communication') # type: ignore
|
164
|
+
"""Communication task."""
|
165
|
+
|
166
|
+
|
159
167
|
class CodeItem(BaseModel):
|
160
168
|
model_config = ConfigDict(extra='forbid')
|
161
169
|
|
@@ -404,6 +412,10 @@ class Package(BaseModel):
|
|
404
412
|
# Name of the problem.
|
405
413
|
name: str = NameField(description='The name of the problem.')
|
406
414
|
|
415
|
+
type: TaskType = Field(
|
416
|
+
default=TaskType.BATCH, description='The type of the problem.'
|
417
|
+
)
|
418
|
+
|
407
419
|
timeLimit: int = Field(description='Time limit of the problem, in milliseconds.')
|
408
420
|
|
409
421
|
memoryLimit: int = Field(description='Memory limit of the problem, in MB.')
|
@@ -423,6 +435,10 @@ class Package(BaseModel):
|
|
423
435
|
default=None, description='The checker for this problem.'
|
424
436
|
)
|
425
437
|
|
438
|
+
interactor: Optional[CodeItem] = Field(
|
439
|
+
default=None, description='The interactor for this problem.'
|
440
|
+
)
|
441
|
+
|
426
442
|
validator: Optional[CodeItem] = Field(
|
427
443
|
default=None, description='The validator for this problem.'
|
428
444
|
)
|
rbx/box/solutions.py
CHANGED
@@ -17,11 +17,13 @@ from pydantic import BaseModel
|
|
17
17
|
|
18
18
|
from rbx import console, utils
|
19
19
|
from rbx.box import checkers, package
|
20
|
-
from rbx.box.code import
|
20
|
+
from rbx.box.code import (
|
21
|
+
SanitizationLevel,
|
22
|
+
compile_item,
|
23
|
+
find_language_name,
|
24
|
+
)
|
21
25
|
from rbx.box.deferred import Deferred
|
22
26
|
from rbx.box.environment import (
|
23
|
-
EnvironmentSandbox,
|
24
|
-
ExecutionConfig,
|
25
27
|
VerificationLevel,
|
26
28
|
)
|
27
29
|
from rbx.box.formatting import get_formatted_memory, get_formatted_time
|
@@ -31,26 +33,26 @@ from rbx.box.generators import (
|
|
31
33
|
generate_output_for_testcase,
|
32
34
|
generate_standalone,
|
33
35
|
)
|
34
|
-
from rbx.box.retries import Retrier
|
35
36
|
from rbx.box.schema import (
|
36
37
|
ExpectedOutcome,
|
37
38
|
GeneratorCall,
|
38
39
|
Limits,
|
39
40
|
Solution,
|
41
|
+
TaskType,
|
40
42
|
Testcase,
|
41
43
|
TestcaseGroup,
|
42
44
|
)
|
45
|
+
from rbx.box.tasks import (
|
46
|
+
get_limits_for_language,
|
47
|
+
run_solution_on_testcase,
|
48
|
+
)
|
43
49
|
from rbx.box.testcase_extractors import extract_generation_testcases
|
44
50
|
from rbx.box.testcase_utils import TestcaseEntry, find_built_testcases
|
45
51
|
from rbx.grading.steps import (
|
46
|
-
DigestOrDest,
|
47
|
-
DigestOrSource,
|
48
52
|
Evaluation,
|
49
53
|
Outcome,
|
50
|
-
TestcaseIO,
|
51
|
-
TestcaseLog,
|
52
54
|
)
|
53
|
-
from rbx.utils import StatusProgress
|
55
|
+
from rbx.utils import StatusProgress
|
54
56
|
|
55
57
|
StructuredEvaluation = Dict[str, Dict[str, List[Optional[Deferred[Evaluation]]]]]
|
56
58
|
|
@@ -152,102 +154,13 @@ def compile_solutions(
|
|
152
154
|
return compiled_solutions
|
153
155
|
|
154
156
|
|
155
|
-
def get_limits_for_language(
|
156
|
-
lang: Optional[str],
|
157
|
-
verification: VerificationLevel,
|
158
|
-
timelimit_override: Optional[int],
|
159
|
-
) -> Limits:
|
160
|
-
pkg = package.find_problem_package_or_die()
|
161
|
-
time = timelimit_override or pkg.timelimit_for_language(lang)
|
162
|
-
isDoubleTL = verification.value >= VerificationLevel.FULL.value
|
163
|
-
memory = pkg.memorylimit_for_language(lang)
|
164
|
-
return Limits(
|
165
|
-
time=time, memory=memory, output=pkg.outputLimit, isDoubleTL=isDoubleTL
|
166
|
-
)
|
167
|
-
|
168
|
-
|
169
|
-
def _run_solution_on_testcase(
|
170
|
-
solution: Solution,
|
171
|
-
compiled_digest: str,
|
172
|
-
checker_digest: Optional[str],
|
173
|
-
testcase: Testcase,
|
174
|
-
output_dir: pathlib.Path,
|
175
|
-
testcase_index: int = 0,
|
176
|
-
verification: VerificationLevel = VerificationLevel.NONE,
|
177
|
-
timelimit_override: Optional[int] = None,
|
178
|
-
) -> Evaluation:
|
179
|
-
def run_fn(retry_index: int) -> Evaluation:
|
180
|
-
actual_sandbox = package.get_singleton_sandbox()
|
181
|
-
|
182
|
-
limits = get_limits_for_language(
|
183
|
-
solution.language, verification, timelimit_override
|
184
|
-
)
|
185
|
-
|
186
|
-
sandbox = EnvironmentSandbox()
|
187
|
-
sandbox.timeLimit = limits.time
|
188
|
-
if limits.isDoubleTL and sandbox.timeLimit is not None:
|
189
|
-
# Double TL.
|
190
|
-
sandbox.timeLimit = sandbox.timeLimit * 2
|
191
|
-
sandbox.wallTimeLimit = sandbox.timeLimit
|
192
|
-
if sandbox.timeLimit is not None and actual_sandbox.use_soft_timeout():
|
193
|
-
sandbox.wallTimeLimit = sandbox.timeLimit * 2
|
194
|
-
sandbox.memoryLimit = limits.memory
|
195
|
-
sandbox.fileSizeLimit = limits.output
|
196
|
-
extra_config = ExecutionConfig(sandbox=sandbox)
|
197
|
-
|
198
|
-
output_path = output_dir / testcase.inputPath.with_suffix('.out').name
|
199
|
-
error_path = output_path.with_suffix('.err')
|
200
|
-
log_path = output_path.with_suffix('.log')
|
201
|
-
output_path.parent.mkdir(parents=True, exist_ok=True)
|
202
|
-
|
203
|
-
run_log = run_item(
|
204
|
-
solution,
|
205
|
-
DigestOrSource.create(compiled_digest),
|
206
|
-
stdin=DigestOrSource.create(testcase.inputPath),
|
207
|
-
stdout=DigestOrDest.create(output_path),
|
208
|
-
stderr=DigestOrDest.create(error_path),
|
209
|
-
extra_config=extra_config,
|
210
|
-
retry_index=retry_index,
|
211
|
-
)
|
212
|
-
|
213
|
-
if checker_digest is not None:
|
214
|
-
checker_result = checkers.check(
|
215
|
-
checker_digest,
|
216
|
-
run_log,
|
217
|
-
testcase,
|
218
|
-
program_output=output_path,
|
219
|
-
)
|
220
|
-
else:
|
221
|
-
checker_result = checkers.check_with_no_output(run_log)
|
222
|
-
|
223
|
-
eval = Evaluation(
|
224
|
-
result=checker_result,
|
225
|
-
testcase=TestcaseIO(
|
226
|
-
index=testcase_index,
|
227
|
-
input=testcase.inputPath,
|
228
|
-
output=testcase.outputPath,
|
229
|
-
),
|
230
|
-
log=TestcaseLog(
|
231
|
-
**(run_log.model_dump() if run_log is not None else {}),
|
232
|
-
stdout_absolute_path=output_path.absolute(),
|
233
|
-
stderr_absolute_path=error_path.absolute(),
|
234
|
-
log_absolute_path=log_path.absolute(),
|
235
|
-
),
|
236
|
-
)
|
237
|
-
|
238
|
-
log_path.write_text(model_to_yaml(eval))
|
239
|
-
return eval
|
240
|
-
|
241
|
-
retrier = Retrier()
|
242
|
-
return retrier.repeat(run_fn)
|
243
|
-
|
244
|
-
|
245
157
|
def _run_solution(
|
246
158
|
solution: Solution,
|
247
159
|
compiled_digest: str,
|
248
160
|
checker_digest: Optional[str],
|
249
161
|
solution_index: int,
|
250
162
|
group_name: str,
|
163
|
+
interactor_digest: Optional[str] = None,
|
251
164
|
progress: Optional[StatusProgress] = None,
|
252
165
|
verification: VerificationLevel = VerificationLevel.NONE,
|
253
166
|
timelimit_override: Optional[int] = None,
|
@@ -267,12 +180,13 @@ def _run_solution(
|
|
267
180
|
)
|
268
181
|
|
269
182
|
async def run_fn(i=i, testcase=testcase, output_path=output_path):
|
270
|
-
return
|
183
|
+
return await run_solution_on_testcase(
|
271
184
|
solution,
|
272
185
|
compiled_digest,
|
273
186
|
checker_digest,
|
274
187
|
testcase,
|
275
188
|
output_path,
|
189
|
+
interactor_digest=interactor_digest,
|
276
190
|
testcase_index=i,
|
277
191
|
verification=verification,
|
278
192
|
timelimit_override=timelimit_override,
|
@@ -343,7 +257,15 @@ def _produce_solution_items(
|
|
343
257
|
) -> List[EvaluationItem]:
|
344
258
|
pkg = package.find_problem_package_or_die()
|
345
259
|
|
346
|
-
|
260
|
+
if pkg.type == TaskType.COMMUNICATION:
|
261
|
+
checker_digest = (
|
262
|
+
checkers.compile_checker() if check and pkg.checker is not None else None
|
263
|
+
)
|
264
|
+
interactor_digest = checkers.compile_interactor()
|
265
|
+
else:
|
266
|
+
checker_digest = checkers.compile_checker() if check else None
|
267
|
+
interactor_digest = None
|
268
|
+
|
347
269
|
compiled_solutions = compile_solutions(
|
348
270
|
progress=progress, tracked_solutions=tracked_solutions, sanitized=sanitized
|
349
271
|
)
|
@@ -374,6 +296,7 @@ def _produce_solution_items(
|
|
374
296
|
checker_digest,
|
375
297
|
solution_index,
|
376
298
|
group_name,
|
299
|
+
interactor_digest=interactor_digest,
|
377
300
|
progress=progress,
|
378
301
|
verification=verification,
|
379
302
|
timelimit_override=timelimit_override,
|
@@ -451,7 +374,7 @@ async def _generate_testcase_interactively(
|
|
451
374
|
copied_to=testcase,
|
452
375
|
)
|
453
376
|
elif testcase_entry is not None:
|
454
|
-
extracted = extract_generation_testcases([testcase_entry])
|
377
|
+
extracted = await extract_generation_testcases([testcase_entry])
|
455
378
|
if not extracted:
|
456
379
|
console.console.print(
|
457
380
|
f'[error]Failed searching for testcase [item]{testcase_entry}[/item].[/error]'
|
@@ -483,7 +406,7 @@ async def _generate_testcase_interactively(
|
|
483
406
|
|
484
407
|
# 1. Generate testcase.
|
485
408
|
if generation_metadata is not None:
|
486
|
-
generate_standalone(
|
409
|
+
await generate_standalone(
|
487
410
|
generation_metadata,
|
488
411
|
progress=progress,
|
489
412
|
validate=True,
|
@@ -531,10 +454,20 @@ async def _generate_testcase_interactively(
|
|
531
454
|
raise
|
532
455
|
|
533
456
|
if main_solution_digest is not None:
|
457
|
+
pkg = package.find_problem_package_or_die()
|
458
|
+
if pkg.type == TaskType.COMMUNICATION:
|
459
|
+
interactor_digest = checkers.compile_interactor(progress)
|
460
|
+
else:
|
461
|
+
interactor_digest = None
|
462
|
+
|
534
463
|
if progress:
|
535
464
|
progress.update('Generating output for test...')
|
536
465
|
# TODO: Add stderr path
|
537
|
-
generate_output_for_testcase(
|
466
|
+
await generate_output_for_testcase(
|
467
|
+
main_solution_digest,
|
468
|
+
testcase,
|
469
|
+
interactor_digest=interactor_digest,
|
470
|
+
)
|
538
471
|
|
539
472
|
if check and testcase.outputPath is not None and not testcase.outputPath.is_file():
|
540
473
|
# Output was not created, throw an error.
|
@@ -559,9 +492,13 @@ def _run_interactive_solutions(
|
|
559
492
|
) -> Iterator[EvaluationItem]:
|
560
493
|
pkg = package.find_problem_package_or_die()
|
561
494
|
|
562
|
-
if
|
563
|
-
|
564
|
-
|
495
|
+
if pkg.type == TaskType.COMMUNICATION:
|
496
|
+
checker_digest = checkers.compile_checker() if check else None
|
497
|
+
interactor_digest = checkers.compile_interactor()
|
498
|
+
else:
|
499
|
+
checker_digest = checkers.compile_checker() if check else None
|
500
|
+
interactor_digest = None
|
501
|
+
|
565
502
|
compiled_solutions = compile_solutions(
|
566
503
|
progress=progress, tracked_solutions=tracked_solutions, sanitized=sanitized
|
567
504
|
)
|
@@ -581,12 +518,13 @@ def _run_interactive_solutions(
|
|
581
518
|
output_dir = irun_dir / f'{i}'
|
582
519
|
|
583
520
|
async def run_fn(solution=solution, output_dir=output_dir):
|
584
|
-
return
|
521
|
+
return await run_solution_on_testcase(
|
585
522
|
solution,
|
586
523
|
compiled_solutions[solution.path],
|
587
524
|
checker_digest,
|
588
525
|
testcase,
|
589
526
|
output_dir,
|
527
|
+
interactor_digest=interactor_digest,
|
590
528
|
verification=verification,
|
591
529
|
)
|
592
530
|
|
rbx/box/solutions_test.py
CHANGED
@@ -1,4 +1,3 @@
|
|
1
|
-
import asyncio
|
2
1
|
import pathlib
|
3
2
|
|
4
3
|
import pytest
|
@@ -17,15 +16,15 @@ from rbx.grading.steps import Outcome
|
|
17
16
|
|
18
17
|
|
19
18
|
@pytest.mark.test_pkg('box1')
|
20
|
-
def test_solutions(pkg_from_testdata: pathlib.Path):
|
21
|
-
generate_testcases()
|
19
|
+
async def test_solutions(pkg_from_testdata: pathlib.Path):
|
20
|
+
await generate_testcases()
|
22
21
|
entries = [
|
23
|
-
entry.group_entry for entry in extract_generation_testcases_from_groups()
|
22
|
+
entry.group_entry for entry in await extract_generation_testcases_from_groups()
|
24
23
|
]
|
25
|
-
generate_outputs_for_testcases(entries)
|
24
|
+
await generate_outputs_for_testcases(entries)
|
26
25
|
|
27
26
|
result = run_solutions(verification=VerificationLevel.FULL)
|
28
|
-
res =
|
27
|
+
res = await convert_list_of_solution_evaluations_to_dict(result.items)
|
29
28
|
|
30
29
|
# First solution should pass all tests.
|
31
30
|
assert all(chk.result.outcome == Outcome.ACCEPTED for chk in res[0]['gen1'])
|