rbx.cp 0.5.66__py3-none-any.whl → 0.5.67__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rbx/box/checkers.py +4 -2
- rbx/box/cli.py +6 -7
- rbx/box/compile.py +3 -1
- rbx/box/formatting.py +5 -1
- rbx/box/generators.py +5 -5
- rbx/box/package.py +11 -3
- rbx/box/packaging/boca/upload.py +1 -0
- rbx/box/presets/__init__.py +2 -1
- rbx/box/remote.py +151 -0
- rbx/box/retries.py +3 -9
- rbx/box/schema.py +25 -10
- rbx/box/solutions.py +41 -64
- rbx/box/stresses.py +17 -6
- rbx/box/ui/css/app.tcss +27 -7
- rbx/box/ui/main.py +3 -0
- rbx/box/ui/screens/rich_log_modal.py +29 -0
- rbx/box/ui/screens/run_explorer.py +20 -3
- rbx/box/ui/screens/run_test_explorer.py +20 -0
- rbx/box/ui/screens/selector.py +6 -4
- rbx/box/ui/screens/test_explorer.py +2 -13
- rbx/box/ui/utils/run_ui.py +13 -0
- rbx/box/unit.py +34 -0
- rbx/grading/steps.py +7 -0
- rbx/resources/checkers/noop.cpp +13 -0
- rbx/resources/packagers/boca/compile/java +10 -5
- rbx/resources/packagers/boca/compile/py2 +52 -44
- rbx/resources/packagers/boca/compile/py3 +52 -44
- rbx/resources/packagers/boca/interactive/java +2 -3
- rbx/resources/packagers/boca/run/java +2 -3
- rbx/test.py +1 -1
- {rbx_cp-0.5.66.dist-info → rbx_cp-0.5.67.dist-info}/METADATA +1 -1
- {rbx_cp-0.5.66.dist-info → rbx_cp-0.5.67.dist-info}/RECORD +35 -32
- {rbx_cp-0.5.66.dist-info → rbx_cp-0.5.67.dist-info}/LICENSE +0 -0
- {rbx_cp-0.5.66.dist-info → rbx_cp-0.5.67.dist-info}/WHEEL +0 -0
- {rbx_cp-0.5.66.dist-info → rbx_cp-0.5.67.dist-info}/entry_points.txt +0 -0
rbx/box/checkers.py
CHANGED
@@ -91,8 +91,10 @@ def _check_pre_output(run_log: Optional[RunLog]) -> CheckerResult:
|
|
91
91
|
|
92
92
|
if run_log.exitstatus in [SandboxBase.EXIT_SIGNAL, SandboxBase.EXIT_NONZERO_RETURN]:
|
93
93
|
return CheckerResult(outcome=Outcome.RUNTIME_ERROR)
|
94
|
-
if run_log.exitstatus
|
94
|
+
if run_log.exitstatus == SandboxBase.EXIT_TIMEOUT:
|
95
95
|
return CheckerResult(outcome=Outcome.TIME_LIMIT_EXCEEDED)
|
96
|
+
if run_log.exitstatus == SandboxBase.EXIT_TIMEOUT_WALL:
|
97
|
+
return CheckerResult(outcome=Outcome.IDLENESS_LIMIT_EXCEEDED)
|
96
98
|
if run_log.exitstatus == SandboxBase.EXIT_MEMORY_LIMIT_EXCEEDED:
|
97
99
|
return CheckerResult(outcome=Outcome.MEMORY_LIMIT_EXCEEDED)
|
98
100
|
if run_log.exitstatus == SandboxBase.EXIT_SANDBOX_ERROR:
|
@@ -103,7 +105,7 @@ def _check_pre_output(run_log: Optional[RunLog]) -> CheckerResult:
|
|
103
105
|
|
104
106
|
|
105
107
|
def _convert_tle(result: CheckerResult, run_log: Optional[RunLog]) -> CheckerResult:
|
106
|
-
if result.outcome
|
108
|
+
if result.outcome.is_slow():
|
107
109
|
# This already is a TLE outcome.
|
108
110
|
return result
|
109
111
|
is_sanitized = (
|
rbx/box/cli.py
CHANGED
@@ -552,18 +552,17 @@ def create(
|
|
552
552
|
@syncer.sync
|
553
553
|
async def stress(
|
554
554
|
name: Annotated[
|
555
|
-
str,
|
555
|
+
Optional[str],
|
556
556
|
typer.Argument(
|
557
|
-
help='Name of the stress test to run (specified in problem.rbx.yml)
|
558
|
-
'or the generator to run, in case -g is specified.'
|
557
|
+
help='Name of the stress test to run (specified in problem.rbx.yml).'
|
559
558
|
),
|
560
|
-
],
|
559
|
+
] = None,
|
561
560
|
generator_args: Annotated[
|
562
561
|
Optional[str],
|
563
562
|
typer.Option(
|
564
563
|
'--generator',
|
565
564
|
'-g',
|
566
|
-
help='
|
565
|
+
help='Generator call to use to generate a single test for execution.',
|
567
566
|
),
|
568
567
|
] = None,
|
569
568
|
finder: Annotated[
|
@@ -610,9 +609,9 @@ async def stress(
|
|
610
609
|
|
611
610
|
with utils.StatusProgress('Running stress...') as s:
|
612
611
|
report = await stresses.run_stress(
|
613
|
-
name,
|
614
612
|
timeout,
|
615
|
-
|
613
|
+
name=name,
|
614
|
+
generator_call=generator_args,
|
616
615
|
finder=finder,
|
617
616
|
findingsLimit=findings,
|
618
617
|
progress=s,
|
rbx/box/compile.py
CHANGED
@@ -3,7 +3,7 @@ import pathlib
|
|
3
3
|
import typer
|
4
4
|
|
5
5
|
from rbx import annotations, console
|
6
|
-
from rbx.box import code, package
|
6
|
+
from rbx.box import code, package, remote
|
7
7
|
from rbx.box.code import SanitizationLevel
|
8
8
|
from rbx.box.sanitizers import warning_stack
|
9
9
|
from rbx.box.schema import CodeItem
|
@@ -35,6 +35,8 @@ def _compile(item: CodeItem, sanitized: SanitizationLevel, warnings: bool):
|
|
35
35
|
def any(path: str, sanitized: bool = False, warnings: bool = False):
|
36
36
|
pkg = package.find_problem_package_or_die()
|
37
37
|
|
38
|
+
path = str(remote.expand_file(path))
|
39
|
+
|
38
40
|
solution = package.get_solution_or_nil(path)
|
39
41
|
if solution is not None:
|
40
42
|
_compile(
|
rbx/box/formatting.py
CHANGED
@@ -1,10 +1,14 @@
|
|
1
1
|
import os
|
2
2
|
import pathlib
|
3
|
-
from typing import Optional
|
3
|
+
from typing import Any, Optional
|
4
4
|
|
5
5
|
from rbx.box import setter_config
|
6
6
|
|
7
7
|
|
8
|
+
def ref(text: Any) -> str:
|
9
|
+
return f'[item]{text}[/item]'
|
10
|
+
|
11
|
+
|
8
12
|
def href(url: os.PathLike[str], text: Optional[str] = None, style: str = 'item') -> str:
|
9
13
|
custom_text = False
|
10
14
|
if text is None:
|
rbx/box/generators.py
CHANGED
@@ -110,7 +110,10 @@ def get_all_built_testcases() -> Dict[str, List[Testcase]]:
|
|
110
110
|
|
111
111
|
|
112
112
|
def get_call_from_string(call_str: str) -> GeneratorCall:
|
113
|
-
|
113
|
+
try:
|
114
|
+
name, args = call_str.split(None, 1)
|
115
|
+
except ValueError:
|
116
|
+
return GeneratorCall(name=call_str, args='')
|
114
117
|
return GeneratorCall(name=name, args=args)
|
115
118
|
|
116
119
|
|
@@ -317,10 +320,7 @@ async def generate_output_for_testcase(
|
|
317
320
|
capture_pipes=True,
|
318
321
|
)
|
319
322
|
|
320
|
-
if (
|
321
|
-
eval.result.outcome == Outcome.TIME_LIMIT_EXCEEDED
|
322
|
-
and eval.result.no_tle_outcome == Outcome.ACCEPTED
|
323
|
-
):
|
323
|
+
if eval.result.outcome.is_slow() and eval.result.no_tle_outcome == Outcome.ACCEPTED:
|
324
324
|
console.console.print(
|
325
325
|
f'[warning]Testcase [item]{testcase.inputPath}[/item] finished in TLE, but test was generated successfully.[/warning]'
|
326
326
|
)
|
rbx/box/package.py
CHANGED
@@ -21,6 +21,7 @@ from rbx.box.schema import (
|
|
21
21
|
Package,
|
22
22
|
Solution,
|
23
23
|
Stress,
|
24
|
+
TaskType,
|
24
25
|
TestcaseGroup,
|
25
26
|
TestcaseSubgroup,
|
26
27
|
)
|
@@ -32,6 +33,7 @@ from rbx.grading.judge.storage import FilesystemStorage, Storage
|
|
32
33
|
|
33
34
|
YAML_NAME = 'problem.rbx.yml'
|
34
35
|
_DEFAULT_CHECKER = 'wcmp.cpp'
|
36
|
+
_NOOP_CHECKER = 'noop.cpp'
|
35
37
|
TEMP_DIR = None
|
36
38
|
CACHE_STEP_VERSION = 1
|
37
39
|
|
@@ -290,13 +292,19 @@ def get_validator_or_nil(root: pathlib.Path = pathlib.Path()) -> Optional[CodeIt
|
|
290
292
|
return package.validator
|
291
293
|
|
292
294
|
|
295
|
+
@functools.cache
|
296
|
+
def get_default_checker(root: pathlib.Path = pathlib.Path()) -> CodeItem:
|
297
|
+
package = find_problem_package_or_die(root)
|
298
|
+
if package.type == TaskType.COMMUNICATION:
|
299
|
+
return CodeItem(path=get_builtin_checker(_NOOP_CHECKER).absolute())
|
300
|
+
return CodeItem(path=get_builtin_checker(_DEFAULT_CHECKER).absolute())
|
301
|
+
|
302
|
+
|
293
303
|
@functools.cache
|
294
304
|
def get_checker(root: pathlib.Path = pathlib.Path()) -> CodeItem:
|
295
305
|
package = find_problem_package_or_die(root)
|
296
306
|
|
297
|
-
return package.checker or
|
298
|
-
path=get_builtin_checker(_DEFAULT_CHECKER).absolute()
|
299
|
-
)
|
307
|
+
return package.checker or get_default_checker(root)
|
300
308
|
|
301
309
|
|
302
310
|
@functools.cache
|
rbx/box/packaging/boca/upload.py
CHANGED
@@ -290,6 +290,7 @@ class BocaUploader:
|
|
290
290
|
if tmp_file is None:
|
291
291
|
self.raw_error('Error while downloading run:\nDownloaded file is None.')
|
292
292
|
final_path = into_dir / filename.with_stem(f'{run_number}-{site_number}')
|
293
|
+
final_path.parent.mkdir(parents=True, exist_ok=True)
|
293
294
|
shutil.move(tmp_file, final_path)
|
294
295
|
return final_path
|
295
296
|
|
rbx/box/presets/__init__.py
CHANGED
@@ -3,7 +3,6 @@ import shutil
|
|
3
3
|
import tempfile
|
4
4
|
from typing import Annotated, Iterable, List, Optional, Sequence, Union
|
5
5
|
|
6
|
-
import questionary
|
7
6
|
import rich
|
8
7
|
import rich.prompt
|
9
8
|
import typer
|
@@ -498,6 +497,8 @@ def copy_local_preset(
|
|
498
497
|
if preset_remote_uri is None:
|
499
498
|
return
|
500
499
|
|
500
|
+
import questionary
|
501
|
+
|
501
502
|
add_submodule = questionary.confirm(
|
502
503
|
'The preset is installed from a remote Git repository. Do you want to add it as a submodule of your project?',
|
503
504
|
default=False,
|
rbx/box/remote.py
ADDED
@@ -0,0 +1,151 @@
|
|
1
|
+
import pathlib
|
2
|
+
import re
|
3
|
+
from abc import ABC, abstractmethod
|
4
|
+
from typing import List, Optional, Tuple, Union
|
5
|
+
|
6
|
+
import typer
|
7
|
+
|
8
|
+
from rbx import console
|
9
|
+
from rbx.box import cd, package
|
10
|
+
from rbx.box.formatting import href, ref
|
11
|
+
|
12
|
+
PathLike = Union[str, pathlib.Path]
|
13
|
+
|
14
|
+
|
15
|
+
class Expander(ABC):
|
16
|
+
def get_remote_path(self, path: pathlib.Path) -> pathlib.Path:
|
17
|
+
return package.get_problem_remote_dir() / path
|
18
|
+
|
19
|
+
def cacheable_paths(self, path: pathlib.Path) -> List[pathlib.Path]:
|
20
|
+
return []
|
21
|
+
|
22
|
+
def cacheable_globs(self, path: pathlib.Path) -> List[str]:
|
23
|
+
return []
|
24
|
+
|
25
|
+
@abstractmethod
|
26
|
+
def expand(self, path: pathlib.Path) -> Optional[pathlib.Path]:
|
27
|
+
pass
|
28
|
+
|
29
|
+
|
30
|
+
class BocaExpander(Expander):
|
31
|
+
BOCA_REGEX = re.compile(r'\@boca\/(\d+)(?:\-(\d+))?')
|
32
|
+
|
33
|
+
def get_match(self, path_str: str) -> Optional[Tuple[int, int]]:
|
34
|
+
match = self.BOCA_REGEX.match(path_str)
|
35
|
+
if match is None:
|
36
|
+
return None
|
37
|
+
run_number = int(match.group(1))
|
38
|
+
site_number = int(match.group(2)) if match.group(2) is not None else 1
|
39
|
+
return run_number, site_number
|
40
|
+
|
41
|
+
def get_boca_folder(self) -> pathlib.Path:
|
42
|
+
return self.get_remote_path(pathlib.Path('boca'))
|
43
|
+
|
44
|
+
def get_boca_path(self, run_number: int, site_number: int) -> pathlib.Path:
|
45
|
+
return self.get_boca_folder() / f'{run_number}-{site_number}'
|
46
|
+
|
47
|
+
def cacheable_globs(self, path: pathlib.Path) -> List[str]:
|
48
|
+
match = self.get_match(str(path))
|
49
|
+
if match is None:
|
50
|
+
return []
|
51
|
+
run_number, site_number = match
|
52
|
+
return [str(self.get_boca_path(run_number, site_number)) + '.*']
|
53
|
+
|
54
|
+
def expand(self, path: pathlib.Path) -> Optional[pathlib.Path]:
|
55
|
+
from rbx.box.packaging.boca import upload as boca_upload
|
56
|
+
|
57
|
+
match = self.get_match(str(path))
|
58
|
+
if match is None:
|
59
|
+
return None
|
60
|
+
run_number, site_number = match
|
61
|
+
|
62
|
+
boca_uploader = boca_upload.get_boca_uploader()
|
63
|
+
boca_uploader.login()
|
64
|
+
sol_path = boca_uploader.download_run(
|
65
|
+
run_number, site_number, self.get_boca_folder()
|
66
|
+
)
|
67
|
+
console.console.print(f'Downloaded {href(sol_path)} from BOCA...')
|
68
|
+
return sol_path
|
69
|
+
|
70
|
+
|
71
|
+
REGISTERED_EXPANDERS: List['Expander'] = [
|
72
|
+
BocaExpander(),
|
73
|
+
]
|
74
|
+
|
75
|
+
|
76
|
+
def _relative_to_pkg(path: pathlib.Path) -> pathlib.Path:
|
77
|
+
return path.resolve().relative_to(pathlib.Path.cwd())
|
78
|
+
|
79
|
+
|
80
|
+
def _try_cacheable_paths(
|
81
|
+
path: pathlib.Path, expander: Expander
|
82
|
+
) -> Optional[pathlib.Path]:
|
83
|
+
cached_paths = expander.cacheable_paths(path)
|
84
|
+
for cached_path in cached_paths:
|
85
|
+
if cached_path.exists():
|
86
|
+
return _relative_to_pkg(cached_path)
|
87
|
+
return None
|
88
|
+
|
89
|
+
|
90
|
+
def _try_cacheable_globs(
|
91
|
+
path: pathlib.Path, expander: Expander
|
92
|
+
) -> Optional[pathlib.Path]:
|
93
|
+
cached_globs = expander.cacheable_globs(path)
|
94
|
+
for cached_glob in cached_globs:
|
95
|
+
rel_glob = _relative_to_pkg(pathlib.Path(cached_glob))
|
96
|
+
globbed = list(pathlib.Path.cwd().glob(str(rel_glob)))
|
97
|
+
if not globbed:
|
98
|
+
continue
|
99
|
+
return _relative_to_pkg(globbed[0])
|
100
|
+
return None
|
101
|
+
|
102
|
+
|
103
|
+
def _try_cache(path: pathlib.Path, expander: Expander) -> Optional[pathlib.Path]:
|
104
|
+
cached = _try_cacheable_paths(path, expander)
|
105
|
+
if cached is not None:
|
106
|
+
return cached
|
107
|
+
return _try_cacheable_globs(path, expander)
|
108
|
+
|
109
|
+
|
110
|
+
def _expand_path(path: pathlib.Path) -> Optional[pathlib.Path]:
|
111
|
+
if not cd.is_problem_package():
|
112
|
+
console.console.print(
|
113
|
+
f'Skipping expansion of {ref(path)} because we are not in a problem package.'
|
114
|
+
)
|
115
|
+
raise typer.Exit(1)
|
116
|
+
|
117
|
+
for expander in REGISTERED_EXPANDERS:
|
118
|
+
cached = _try_cache(path, expander)
|
119
|
+
if cached is not None:
|
120
|
+
return cached
|
121
|
+
expanded = expander.expand(path)
|
122
|
+
if expanded is not None:
|
123
|
+
return _relative_to_pkg(expanded)
|
124
|
+
return None
|
125
|
+
|
126
|
+
|
127
|
+
def _expand_paths(paths: List[pathlib.Path]) -> List[pathlib.Path]:
|
128
|
+
res = []
|
129
|
+
for path in paths:
|
130
|
+
if not str(path).startswith('@'):
|
131
|
+
res.append(path)
|
132
|
+
continue
|
133
|
+
expanded = _expand_path(path)
|
134
|
+
if expanded is None:
|
135
|
+
continue
|
136
|
+
res.append(expanded)
|
137
|
+
return res
|
138
|
+
|
139
|
+
|
140
|
+
def expand_files(files: List[str]) -> List[pathlib.Path]:
|
141
|
+
return _expand_paths([pathlib.Path(file) for file in files])
|
142
|
+
|
143
|
+
|
144
|
+
def expand_file(file: str) -> pathlib.Path:
|
145
|
+
res = expand_files([file])
|
146
|
+
if len(res) != 1:
|
147
|
+
console.console.print(
|
148
|
+
f'Could not expand {ref(file)} because it is not a valid expansion.'
|
149
|
+
)
|
150
|
+
raise typer.Exit(1)
|
151
|
+
return res[0]
|
rbx/box/retries.py
CHANGED
@@ -18,10 +18,7 @@ def _both_accepted(eval_a: Evaluation, eval_b: Evaluation) -> bool:
|
|
18
18
|
|
19
19
|
|
20
20
|
def _any_tle(eval_a: Evaluation, eval_b: Evaluation) -> bool:
|
21
|
-
return (
|
22
|
-
eval_a.result.outcome == Outcome.TIME_LIMIT_EXCEEDED
|
23
|
-
or eval_b.result.outcome == Outcome.TIME_LIMIT_EXCEEDED
|
24
|
-
)
|
21
|
+
return eval_a.result.outcome.is_slow() or eval_b.result.outcome.is_slow()
|
25
22
|
|
26
23
|
|
27
24
|
def _get_faster(eval_a: Evaluation, eval_b: Evaluation) -> Evaluation:
|
@@ -130,13 +127,10 @@ class Retrier:
|
|
130
127
|
|
131
128
|
def should_repeat(self, eval: Evaluation) -> bool:
|
132
129
|
if self.is_stress:
|
133
|
-
if (
|
134
|
-
eval.result.outcome == Outcome.TIME_LIMIT_EXCEEDED
|
135
|
-
and self.retries_for_stress > 0
|
136
|
-
):
|
130
|
+
if eval.result.outcome.is_slow() and self.retries_for_stress > 0:
|
137
131
|
self.retries_for_stress -= 1
|
138
132
|
return True
|
139
|
-
if eval.result.outcome
|
133
|
+
if eval.result.outcome.is_slow() and self.retries > 0:
|
140
134
|
self.retries -= 1
|
141
135
|
return True
|
142
136
|
if self.reps > 0:
|
rbx/box/schema.py
CHANGED
@@ -57,6 +57,9 @@ def expand_var(value: Primitive) -> Primitive:
|
|
57
57
|
|
58
58
|
|
59
59
|
class ExpectedOutcome(AutoEnum):
|
60
|
+
ANY = alias('any') # type: ignore
|
61
|
+
"""Expected outcome for any outcome."""
|
62
|
+
|
60
63
|
ACCEPTED = alias('accepted', 'ac', 'correct') # type: ignore
|
61
64
|
"""Expected outcome for correct solutions (AC)."""
|
62
65
|
|
@@ -97,6 +100,8 @@ class ExpectedOutcome(AutoEnum):
|
|
97
100
|
Especially useful for environments where TLE and RTE are indistinguishable."""
|
98
101
|
|
99
102
|
def style(self) -> str:
|
103
|
+
if self == ExpectedOutcome.ANY:
|
104
|
+
return 'orange'
|
100
105
|
if self == ExpectedOutcome.ACCEPTED:
|
101
106
|
return 'green'
|
102
107
|
if self == ExpectedOutcome.WRONG_ANSWER:
|
@@ -114,29 +119,39 @@ class ExpectedOutcome(AutoEnum):
|
|
114
119
|
def is_slow(self) -> bool:
|
115
120
|
return self in [ExpectedOutcome.TIME_LIMIT_EXCEEDED, ExpectedOutcome.TLE_OR_RTE]
|
116
121
|
|
122
|
+
def matches_tle_and_is_incorrect(self) -> bool:
|
123
|
+
return self.match(Outcome.TIME_LIMIT_EXCEEDED) and not self.match(
|
124
|
+
Outcome.ACCEPTED
|
125
|
+
)
|
126
|
+
|
117
127
|
def match(self, outcome: Outcome) -> bool:
|
128
|
+
if self == ExpectedOutcome.ANY:
|
129
|
+
return True
|
118
130
|
if self == ExpectedOutcome.ACCEPTED:
|
119
131
|
return outcome == Outcome.ACCEPTED
|
120
132
|
if self == ExpectedOutcome.ACCEPTED_OR_TLE:
|
121
|
-
return outcome in {Outcome.ACCEPTED
|
133
|
+
return outcome in {Outcome.ACCEPTED} or outcome.is_slow()
|
122
134
|
if self == ExpectedOutcome.WRONG_ANSWER:
|
123
135
|
return outcome == Outcome.WRONG_ANSWER
|
124
136
|
if self == ExpectedOutcome.INCORRECT:
|
125
|
-
return
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
137
|
+
return (
|
138
|
+
outcome
|
139
|
+
in {
|
140
|
+
Outcome.WRONG_ANSWER,
|
141
|
+
Outcome.RUNTIME_ERROR,
|
142
|
+
Outcome.MEMORY_LIMIT_EXCEEDED,
|
143
|
+
Outcome.OUTPUT_LIMIT_EXCEEDED,
|
144
|
+
}
|
145
|
+
or outcome.is_slow()
|
146
|
+
)
|
132
147
|
if self == ExpectedOutcome.RUNTIME_ERROR:
|
133
148
|
return outcome == Outcome.RUNTIME_ERROR
|
134
149
|
if self == ExpectedOutcome.TIME_LIMIT_EXCEEDED:
|
135
|
-
return outcome
|
150
|
+
return outcome.is_slow()
|
136
151
|
if self == ExpectedOutcome.MEMORY_LIMIT_EXCEEDED:
|
137
152
|
return outcome == Outcome.MEMORY_LIMIT_EXCEEDED
|
138
153
|
if self == ExpectedOutcome.TLE_OR_RTE:
|
139
|
-
return outcome in {Outcome.
|
154
|
+
return outcome in {Outcome.RUNTIME_ERROR} or outcome.is_slow()
|
140
155
|
if self == ExpectedOutcome.OUTPUT_LIMIT_EXCEEDED:
|
141
156
|
return outcome == Outcome.OUTPUT_LIMIT_EXCEEDED
|
142
157
|
return False
|
rbx/box/solutions.py
CHANGED
@@ -16,7 +16,7 @@ import typer
|
|
16
16
|
from pydantic import BaseModel
|
17
17
|
|
18
18
|
from rbx import console, utils
|
19
|
-
from rbx.box import checkers, environment, package, state
|
19
|
+
from rbx.box import checkers, environment, package, remote, state
|
20
20
|
from rbx.box.code import (
|
21
21
|
SanitizationLevel,
|
22
22
|
compile_item,
|
@@ -130,7 +130,7 @@ class RunSolutionResult:
|
|
130
130
|
|
131
131
|
def is_fast(solution: Solution) -> bool:
|
132
132
|
# If solution has TLE tag, it is considered slow.
|
133
|
-
return not solution.outcome.
|
133
|
+
return not solution.outcome.is_slow()
|
134
134
|
|
135
135
|
|
136
136
|
def get_matching_solutions(expected_outcome: ExpectedOutcome) -> List[Solution]:
|
@@ -720,74 +720,38 @@ def _get_solution_repr(sol: Solution) -> List[Tuple[str, str]]:
|
|
720
720
|
]
|
721
721
|
|
722
722
|
|
723
|
-
|
724
|
-
def _expand_remote_sol(sol: str) -> Optional[str]:
|
725
|
-
from rbx.box.packaging.boca import upload as boca_upload
|
726
|
-
|
727
|
-
# Only BOCA supported for now.
|
728
|
-
assert sol.startswith('@')
|
729
|
-
sol = sol[1:]
|
730
|
-
if '-' in sol:
|
731
|
-
run_number, site_number = sol.split('-', 1)
|
732
|
-
run_number = int(run_number)
|
733
|
-
site_number = int(site_number)
|
734
|
-
else:
|
735
|
-
run_number = int(sol)
|
736
|
-
site_number = 1
|
737
|
-
|
738
|
-
expected_glob = (
|
739
|
-
package.get_problem_remote_dir('boca') / f'{run_number}-{site_number}.*'
|
740
|
-
)
|
741
|
-
expected_glob = expected_glob.relative_to(pathlib.Path.cwd())
|
742
|
-
for path in pathlib.Path.cwd().glob(str(expected_glob)):
|
743
|
-
if path.is_file():
|
744
|
-
relpath = path.resolve().relative_to(pathlib.Path.cwd())
|
745
|
-
console.console.print(f'Retrieving {href(relpath)} from cache...')
|
746
|
-
return str(relpath)
|
747
|
-
|
748
|
-
boca_uploader = boca_upload.get_boca_uploader()
|
749
|
-
boca_uploader.login()
|
750
|
-
sol_path = boca_uploader.download_run(
|
751
|
-
run_number, site_number, package.get_problem_remote_dir('boca')
|
752
|
-
)
|
753
|
-
console.console.print(f'Downloaded {href(sol_path)} from BOCA...')
|
754
|
-
return str(sol_path.resolve().relative_to(pathlib.Path.cwd()))
|
755
|
-
|
756
|
-
|
757
|
-
def _expand_remote_sols(sols: List[str]) -> List[str]:
|
758
|
-
res = []
|
759
|
-
for sol in sols:
|
760
|
-
if not sol.startswith('@'):
|
761
|
-
res.append(sol)
|
762
|
-
continue
|
763
|
-
expanded = _expand_remote_sol(sol)
|
764
|
-
if expanded is None:
|
765
|
-
continue
|
766
|
-
res.append(expanded)
|
767
|
-
return res
|
768
|
-
|
769
|
-
|
770
|
-
def expand_solutions(sols: List[str]) -> List[Solution]:
|
723
|
+
def expand_solutions_with_source(sols: List[str]) -> List[Tuple[Solution, bool]]:
|
771
724
|
pkg = package.find_problem_package_or_die()
|
772
|
-
|
725
|
+
pkg_sols = {str(sol.path): sol for sol in pkg.solutions}
|
773
726
|
|
774
727
|
# Download remote sols.
|
775
|
-
sols =
|
728
|
+
sols = remote.expand_files(sols)
|
776
729
|
|
777
730
|
# Ensure sols exist.
|
778
731
|
sols = [sol for sol in sols if pathlib.Path(sol).is_file()]
|
779
732
|
|
780
|
-
|
733
|
+
seen_sols = set()
|
734
|
+
res: List[Tuple[Solution, bool]] = []
|
781
735
|
for sol in sols:
|
782
736
|
if sol in seen_sols:
|
783
|
-
|
737
|
+
# This solution was already added.
|
738
|
+
continue
|
739
|
+
if sol in pkg_sols:
|
740
|
+
# This solution is in the package.
|
741
|
+
res.append((pkg_sols[sol], False))
|
784
742
|
else:
|
743
|
+
# This solution is fetched from some source.
|
785
744
|
res.append(
|
786
|
-
Solution(path=pathlib.Path(sol), outcome=ExpectedOutcome.
|
745
|
+
(Solution(path=pathlib.Path(sol), outcome=ExpectedOutcome.ANY), True)
|
787
746
|
)
|
747
|
+
seen_sols.add(sol)
|
788
748
|
return res
|
789
749
|
|
790
750
|
|
751
|
+
def expand_solutions(sols: List[str]) -> List[Solution]:
|
752
|
+
return [sol for sol, _ in expand_solutions_with_source(sols)]
|
753
|
+
|
754
|
+
|
791
755
|
async def pick_solutions(
|
792
756
|
tracked_solutions: Optional[Set[str]],
|
793
757
|
extra_solutions: Optional[List[str]] = None,
|
@@ -830,7 +794,7 @@ def get_outcome_style_verdict(outcome: Outcome) -> str:
|
|
830
794
|
return 'green'
|
831
795
|
if outcome == Outcome.WRONG_ANSWER:
|
832
796
|
return 'red'
|
833
|
-
if outcome
|
797
|
+
if outcome.is_slow():
|
834
798
|
return 'yellow'
|
835
799
|
if outcome == Outcome.RUNTIME_ERROR:
|
836
800
|
return 'blue'
|
@@ -843,7 +807,7 @@ def get_outcome_markup_verdict(outcome: Outcome) -> str:
|
|
843
807
|
res = '✓'
|
844
808
|
if outcome != Outcome.ACCEPTED:
|
845
809
|
res = '✗'
|
846
|
-
if outcome
|
810
|
+
if outcome.is_slow():
|
847
811
|
res = '⧖'
|
848
812
|
if outcome == Outcome.RUNTIME_ERROR:
|
849
813
|
res = '✗'
|
@@ -873,6 +837,19 @@ def get_full_testcase_markup_verdict(eval: Evaluation) -> str:
|
|
873
837
|
def _get_evals_time_in_ms(evals: List[Evaluation]) -> int:
|
874
838
|
if not evals:
|
875
839
|
return 0
|
840
|
+
evals_with_ile = [
|
841
|
+
eval for eval in evals if eval.result.outcome == Outcome.IDLENESS_LIMIT_EXCEEDED
|
842
|
+
]
|
843
|
+
for eval in evals_with_ile:
|
844
|
+
# Try every way of estimating a ILE max timelimit.
|
845
|
+
if eval.log.metadata is None:
|
846
|
+
continue
|
847
|
+
if eval.log.metadata.limits is not None:
|
848
|
+
expanded_tl = eval.log.metadata.limits.get_expanded_tl()
|
849
|
+
if expanded_tl is not None:
|
850
|
+
return expanded_tl
|
851
|
+
if eval.log.metadata.timeLimit is not None:
|
852
|
+
return eval.log.metadata.timeLimit
|
876
853
|
return max(int((eval.log.time or 0.0) * 1000) for eval in evals)
|
877
854
|
|
878
855
|
|
@@ -893,7 +870,10 @@ def get_capped_evals_formatted_time(
|
|
893
870
|
pkg = package.find_problem_package_or_die()
|
894
871
|
|
895
872
|
max_time = _get_evals_time_in_ms(evals)
|
896
|
-
has_tle = any(eval.result.outcome
|
873
|
+
has_tle = any(eval.result.outcome.is_slow() for eval in evals)
|
874
|
+
has_ile = any(
|
875
|
+
eval.result.outcome == Outcome.IDLENESS_LIMIT_EXCEEDED for eval in evals
|
876
|
+
)
|
897
877
|
timelimits = [
|
898
878
|
eval.log.metadata.limits.get_expanded_tl()
|
899
879
|
for eval in evals
|
@@ -911,7 +891,7 @@ def get_capped_evals_formatted_time(
|
|
911
891
|
# Using double TL for verification.
|
912
892
|
tl = tl * 2
|
913
893
|
|
914
|
-
if has_tle and max_time >= tl:
|
894
|
+
if has_tle and max_time >= tl or has_ile:
|
915
895
|
return f'>{tl} ms'
|
916
896
|
return f'{max_time} ms'
|
917
897
|
|
@@ -996,8 +976,7 @@ def get_solution_outcome_report(
|
|
996
976
|
):
|
997
977
|
no_tle_bad_verdicts.add(eval.result.no_tle_outcome)
|
998
978
|
has_plain_tle = has_plain_tle or (
|
999
|
-
eval.result.outcome
|
1000
|
-
and eval.result.no_tle_outcome is None
|
979
|
+
eval.result.outcome.is_slow() and eval.result.no_tle_outcome is None
|
1001
980
|
)
|
1002
981
|
has_sanitizer_warnings = (
|
1003
982
|
has_sanitizer_warnings or eval.result.sanitizer_warnings
|
@@ -1030,9 +1009,7 @@ def get_solution_outcome_report(
|
|
1030
1009
|
report_got_verdicts = {Outcome.ACCEPTED}
|
1031
1010
|
|
1032
1011
|
evals_time = _get_evals_time_in_ms(evals)
|
1033
|
-
expected_outcome_is_tle = solution.outcome.
|
1034
|
-
Outcome.TIME_LIMIT_EXCEEDED
|
1035
|
-
) and not solution.outcome.match(Outcome.ACCEPTED)
|
1012
|
+
expected_outcome_is_tle = solution.outcome.matches_tle_and_is_incorrect()
|
1036
1013
|
if (
|
1037
1014
|
# Running verification with double TL.
|
1038
1015
|
verification.value >= VerificationLevel.FULL.value
|
rbx/box/stresses.py
CHANGED
@@ -8,7 +8,7 @@ import typer
|
|
8
8
|
from pydantic import BaseModel
|
9
9
|
|
10
10
|
from rbx import console
|
11
|
-
from rbx.box import checkers, package, validators
|
11
|
+
from rbx.box import checkers, generators, package, validators
|
12
12
|
from rbx.box.code import SanitizationLevel, compile_item, run_item
|
13
13
|
from rbx.box.generators import (
|
14
14
|
GenerationMetadata,
|
@@ -51,10 +51,10 @@ def _compile_finder(finder: CodeItem) -> str:
|
|
51
51
|
|
52
52
|
|
53
53
|
async def run_stress(
|
54
|
-
name: str,
|
55
54
|
timeoutInSeconds: int,
|
55
|
+
name: Optional[str] = None,
|
56
56
|
finder: Optional[str] = None,
|
57
|
-
|
57
|
+
generator_call: Optional[str] = None,
|
58
58
|
findingsLimit: int = 1,
|
59
59
|
verbose: bool = False,
|
60
60
|
progress: Optional[StatusProgress] = None,
|
@@ -68,12 +68,23 @@ async def run_stress(
|
|
68
68
|
raise typer.Exit(1)
|
69
69
|
|
70
70
|
if finder:
|
71
|
+
if generator_call is None:
|
72
|
+
console.console.print(
|
73
|
+
'[error]Generator arguments are required for stress testing. Specify them through the [item]-g[/item] flag.[/error]'
|
74
|
+
)
|
75
|
+
raise typer.Exit(1)
|
76
|
+
generator = generators.get_call_from_string(generator_call)
|
71
77
|
stress = Stress(
|
72
|
-
name=f'{name}',
|
73
|
-
generator=
|
78
|
+
name=f'{generator.name}',
|
79
|
+
generator=generator,
|
74
80
|
finder=finder,
|
75
81
|
)
|
76
82
|
else:
|
83
|
+
if name is None:
|
84
|
+
console.console.print(
|
85
|
+
'[error]Invalid stress test paramaters. Either provide a stress test name, or provide a finder expression (-f) and generator arguments (-g).[/error]'
|
86
|
+
)
|
87
|
+
raise typer.Exit(1)
|
77
88
|
stress = package.get_stress(name)
|
78
89
|
|
79
90
|
call = stress.generator
|
@@ -258,7 +269,7 @@ async def run_stress(
|
|
258
269
|
|
259
270
|
if internal_error_results:
|
260
271
|
console.console.print(
|
261
|
-
f'[error]Checkers failed during stress test [item]{name}[/item] with args [info]{expanded_generator_call.name} {expanded_generator_call.args}[/info][/error]'
|
272
|
+
f'[error]Checkers failed during stress test [item]{stress.name}[/item] with args [info]{expanded_generator_call.name} {expanded_generator_call.args}[/info][/error]'
|
262
273
|
)
|
263
274
|
for internal_error_result in internal_error_results:
|
264
275
|
assert internal_error_result.checker is not None
|