rbx.cp 0.5.66__py3-none-any.whl → 0.5.67__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
rbx/box/checkers.py CHANGED
@@ -91,8 +91,10 @@ def _check_pre_output(run_log: Optional[RunLog]) -> CheckerResult:
91
91
 
92
92
  if run_log.exitstatus in [SandboxBase.EXIT_SIGNAL, SandboxBase.EXIT_NONZERO_RETURN]:
93
93
  return CheckerResult(outcome=Outcome.RUNTIME_ERROR)
94
- if run_log.exitstatus in [SandboxBase.EXIT_TIMEOUT, SandboxBase.EXIT_TIMEOUT_WALL]:
94
+ if run_log.exitstatus == SandboxBase.EXIT_TIMEOUT:
95
95
  return CheckerResult(outcome=Outcome.TIME_LIMIT_EXCEEDED)
96
+ if run_log.exitstatus == SandboxBase.EXIT_TIMEOUT_WALL:
97
+ return CheckerResult(outcome=Outcome.IDLENESS_LIMIT_EXCEEDED)
96
98
  if run_log.exitstatus == SandboxBase.EXIT_MEMORY_LIMIT_EXCEEDED:
97
99
  return CheckerResult(outcome=Outcome.MEMORY_LIMIT_EXCEEDED)
98
100
  if run_log.exitstatus == SandboxBase.EXIT_SANDBOX_ERROR:
@@ -103,7 +105,7 @@ def _check_pre_output(run_log: Optional[RunLog]) -> CheckerResult:
103
105
 
104
106
 
105
107
  def _convert_tle(result: CheckerResult, run_log: Optional[RunLog]) -> CheckerResult:
106
- if result.outcome == Outcome.TIME_LIMIT_EXCEEDED:
108
+ if result.outcome.is_slow():
107
109
  # This already is a TLE outcome.
108
110
  return result
109
111
  is_sanitized = (
rbx/box/cli.py CHANGED
@@ -552,18 +552,17 @@ def create(
552
552
  @syncer.sync
553
553
  async def stress(
554
554
  name: Annotated[
555
- str,
555
+ Optional[str],
556
556
  typer.Argument(
557
- help='Name of the stress test to run (specified in problem.rbx.yml), '
558
- 'or the generator to run, in case -g is specified.'
557
+ help='Name of the stress test to run (specified in problem.rbx.yml).'
559
558
  ),
560
- ],
559
+ ] = None,
561
560
  generator_args: Annotated[
562
561
  Optional[str],
563
562
  typer.Option(
564
563
  '--generator',
565
564
  '-g',
566
- help='Run generator [name] with these args.',
565
+ help='Generator call to use to generate a single test for execution.',
567
566
  ),
568
567
  ] = None,
569
568
  finder: Annotated[
@@ -610,9 +609,9 @@ async def stress(
610
609
 
611
610
  with utils.StatusProgress('Running stress...') as s:
612
611
  report = await stresses.run_stress(
613
- name,
614
612
  timeout,
615
- args=generator_args,
613
+ name=name,
614
+ generator_call=generator_args,
616
615
  finder=finder,
617
616
  findingsLimit=findings,
618
617
  progress=s,
rbx/box/compile.py CHANGED
@@ -3,7 +3,7 @@ import pathlib
3
3
  import typer
4
4
 
5
5
  from rbx import annotations, console
6
- from rbx.box import code, package
6
+ from rbx.box import code, package, remote
7
7
  from rbx.box.code import SanitizationLevel
8
8
  from rbx.box.sanitizers import warning_stack
9
9
  from rbx.box.schema import CodeItem
@@ -35,6 +35,8 @@ def _compile(item: CodeItem, sanitized: SanitizationLevel, warnings: bool):
35
35
  def any(path: str, sanitized: bool = False, warnings: bool = False):
36
36
  pkg = package.find_problem_package_or_die()
37
37
 
38
+ path = str(remote.expand_file(path))
39
+
38
40
  solution = package.get_solution_or_nil(path)
39
41
  if solution is not None:
40
42
  _compile(
rbx/box/formatting.py CHANGED
@@ -1,10 +1,14 @@
1
1
  import os
2
2
  import pathlib
3
- from typing import Optional
3
+ from typing import Any, Optional
4
4
 
5
5
  from rbx.box import setter_config
6
6
 
7
7
 
8
+ def ref(text: Any) -> str:
9
+ return f'[item]{text}[/item]'
10
+
11
+
8
12
  def href(url: os.PathLike[str], text: Optional[str] = None, style: str = 'item') -> str:
9
13
  custom_text = False
10
14
  if text is None:
rbx/box/generators.py CHANGED
@@ -110,7 +110,10 @@ def get_all_built_testcases() -> Dict[str, List[Testcase]]:
110
110
 
111
111
 
112
112
  def get_call_from_string(call_str: str) -> GeneratorCall:
113
- name, args = call_str.split(None, 1)
113
+ try:
114
+ name, args = call_str.split(None, 1)
115
+ except ValueError:
116
+ return GeneratorCall(name=call_str, args='')
114
117
  return GeneratorCall(name=name, args=args)
115
118
 
116
119
 
@@ -317,10 +320,7 @@ async def generate_output_for_testcase(
317
320
  capture_pipes=True,
318
321
  )
319
322
 
320
- if (
321
- eval.result.outcome == Outcome.TIME_LIMIT_EXCEEDED
322
- and eval.result.no_tle_outcome == Outcome.ACCEPTED
323
- ):
323
+ if eval.result.outcome.is_slow() and eval.result.no_tle_outcome == Outcome.ACCEPTED:
324
324
  console.console.print(
325
325
  f'[warning]Testcase [item]{testcase.inputPath}[/item] finished in TLE, but test was generated successfully.[/warning]'
326
326
  )
rbx/box/package.py CHANGED
@@ -21,6 +21,7 @@ from rbx.box.schema import (
21
21
  Package,
22
22
  Solution,
23
23
  Stress,
24
+ TaskType,
24
25
  TestcaseGroup,
25
26
  TestcaseSubgroup,
26
27
  )
@@ -32,6 +33,7 @@ from rbx.grading.judge.storage import FilesystemStorage, Storage
32
33
 
33
34
  YAML_NAME = 'problem.rbx.yml'
34
35
  _DEFAULT_CHECKER = 'wcmp.cpp'
36
+ _NOOP_CHECKER = 'noop.cpp'
35
37
  TEMP_DIR = None
36
38
  CACHE_STEP_VERSION = 1
37
39
 
@@ -290,13 +292,19 @@ def get_validator_or_nil(root: pathlib.Path = pathlib.Path()) -> Optional[CodeIt
290
292
  return package.validator
291
293
 
292
294
 
295
+ @functools.cache
296
+ def get_default_checker(root: pathlib.Path = pathlib.Path()) -> CodeItem:
297
+ package = find_problem_package_or_die(root)
298
+ if package.type == TaskType.COMMUNICATION:
299
+ return CodeItem(path=get_builtin_checker(_NOOP_CHECKER).absolute())
300
+ return CodeItem(path=get_builtin_checker(_DEFAULT_CHECKER).absolute())
301
+
302
+
293
303
  @functools.cache
294
304
  def get_checker(root: pathlib.Path = pathlib.Path()) -> CodeItem:
295
305
  package = find_problem_package_or_die(root)
296
306
 
297
- return package.checker or CodeItem(
298
- path=get_builtin_checker(_DEFAULT_CHECKER).absolute()
299
- )
307
+ return package.checker or get_default_checker(root)
300
308
 
301
309
 
302
310
  @functools.cache
@@ -290,6 +290,7 @@ class BocaUploader:
290
290
  if tmp_file is None:
291
291
  self.raw_error('Error while downloading run:\nDownloaded file is None.')
292
292
  final_path = into_dir / filename.with_stem(f'{run_number}-{site_number}')
293
+ final_path.parent.mkdir(parents=True, exist_ok=True)
293
294
  shutil.move(tmp_file, final_path)
294
295
  return final_path
295
296
 
@@ -3,7 +3,6 @@ import shutil
3
3
  import tempfile
4
4
  from typing import Annotated, Iterable, List, Optional, Sequence, Union
5
5
 
6
- import questionary
7
6
  import rich
8
7
  import rich.prompt
9
8
  import typer
@@ -498,6 +497,8 @@ def copy_local_preset(
498
497
  if preset_remote_uri is None:
499
498
  return
500
499
 
500
+ import questionary
501
+
501
502
  add_submodule = questionary.confirm(
502
503
  'The preset is installed from a remote Git repository. Do you want to add it as a submodule of your project?',
503
504
  default=False,
rbx/box/remote.py ADDED
@@ -0,0 +1,151 @@
1
+ import pathlib
2
+ import re
3
+ from abc import ABC, abstractmethod
4
+ from typing import List, Optional, Tuple, Union
5
+
6
+ import typer
7
+
8
+ from rbx import console
9
+ from rbx.box import cd, package
10
+ from rbx.box.formatting import href, ref
11
+
12
+ PathLike = Union[str, pathlib.Path]
13
+
14
+
15
+ class Expander(ABC):
16
+ def get_remote_path(self, path: pathlib.Path) -> pathlib.Path:
17
+ return package.get_problem_remote_dir() / path
18
+
19
+ def cacheable_paths(self, path: pathlib.Path) -> List[pathlib.Path]:
20
+ return []
21
+
22
+ def cacheable_globs(self, path: pathlib.Path) -> List[str]:
23
+ return []
24
+
25
+ @abstractmethod
26
+ def expand(self, path: pathlib.Path) -> Optional[pathlib.Path]:
27
+ pass
28
+
29
+
30
+ class BocaExpander(Expander):
31
+ BOCA_REGEX = re.compile(r'\@boca\/(\d+)(?:\-(\d+))?')
32
+
33
+ def get_match(self, path_str: str) -> Optional[Tuple[int, int]]:
34
+ match = self.BOCA_REGEX.match(path_str)
35
+ if match is None:
36
+ return None
37
+ run_number = int(match.group(1))
38
+ site_number = int(match.group(2)) if match.group(2) is not None else 1
39
+ return run_number, site_number
40
+
41
+ def get_boca_folder(self) -> pathlib.Path:
42
+ return self.get_remote_path(pathlib.Path('boca'))
43
+
44
+ def get_boca_path(self, run_number: int, site_number: int) -> pathlib.Path:
45
+ return self.get_boca_folder() / f'{run_number}-{site_number}'
46
+
47
+ def cacheable_globs(self, path: pathlib.Path) -> List[str]:
48
+ match = self.get_match(str(path))
49
+ if match is None:
50
+ return []
51
+ run_number, site_number = match
52
+ return [str(self.get_boca_path(run_number, site_number)) + '.*']
53
+
54
+ def expand(self, path: pathlib.Path) -> Optional[pathlib.Path]:
55
+ from rbx.box.packaging.boca import upload as boca_upload
56
+
57
+ match = self.get_match(str(path))
58
+ if match is None:
59
+ return None
60
+ run_number, site_number = match
61
+
62
+ boca_uploader = boca_upload.get_boca_uploader()
63
+ boca_uploader.login()
64
+ sol_path = boca_uploader.download_run(
65
+ run_number, site_number, self.get_boca_folder()
66
+ )
67
+ console.console.print(f'Downloaded {href(sol_path)} from BOCA...')
68
+ return sol_path
69
+
70
+
71
+ REGISTERED_EXPANDERS: List['Expander'] = [
72
+ BocaExpander(),
73
+ ]
74
+
75
+
76
+ def _relative_to_pkg(path: pathlib.Path) -> pathlib.Path:
77
+ return path.resolve().relative_to(pathlib.Path.cwd())
78
+
79
+
80
+ def _try_cacheable_paths(
81
+ path: pathlib.Path, expander: Expander
82
+ ) -> Optional[pathlib.Path]:
83
+ cached_paths = expander.cacheable_paths(path)
84
+ for cached_path in cached_paths:
85
+ if cached_path.exists():
86
+ return _relative_to_pkg(cached_path)
87
+ return None
88
+
89
+
90
+ def _try_cacheable_globs(
91
+ path: pathlib.Path, expander: Expander
92
+ ) -> Optional[pathlib.Path]:
93
+ cached_globs = expander.cacheable_globs(path)
94
+ for cached_glob in cached_globs:
95
+ rel_glob = _relative_to_pkg(pathlib.Path(cached_glob))
96
+ globbed = list(pathlib.Path.cwd().glob(str(rel_glob)))
97
+ if not globbed:
98
+ continue
99
+ return _relative_to_pkg(globbed[0])
100
+ return None
101
+
102
+
103
+ def _try_cache(path: pathlib.Path, expander: Expander) -> Optional[pathlib.Path]:
104
+ cached = _try_cacheable_paths(path, expander)
105
+ if cached is not None:
106
+ return cached
107
+ return _try_cacheable_globs(path, expander)
108
+
109
+
110
+ def _expand_path(path: pathlib.Path) -> Optional[pathlib.Path]:
111
+ if not cd.is_problem_package():
112
+ console.console.print(
113
+ f'Skipping expansion of {ref(path)} because we are not in a problem package.'
114
+ )
115
+ raise typer.Exit(1)
116
+
117
+ for expander in REGISTERED_EXPANDERS:
118
+ cached = _try_cache(path, expander)
119
+ if cached is not None:
120
+ return cached
121
+ expanded = expander.expand(path)
122
+ if expanded is not None:
123
+ return _relative_to_pkg(expanded)
124
+ return None
125
+
126
+
127
+ def _expand_paths(paths: List[pathlib.Path]) -> List[pathlib.Path]:
128
+ res = []
129
+ for path in paths:
130
+ if not str(path).startswith('@'):
131
+ res.append(path)
132
+ continue
133
+ expanded = _expand_path(path)
134
+ if expanded is None:
135
+ continue
136
+ res.append(expanded)
137
+ return res
138
+
139
+
140
+ def expand_files(files: List[str]) -> List[pathlib.Path]:
141
+ return _expand_paths([pathlib.Path(file) for file in files])
142
+
143
+
144
+ def expand_file(file: str) -> pathlib.Path:
145
+ res = expand_files([file])
146
+ if len(res) != 1:
147
+ console.console.print(
148
+ f'Could not expand {ref(file)} because it is not a valid expansion.'
149
+ )
150
+ raise typer.Exit(1)
151
+ return res[0]
rbx/box/retries.py CHANGED
@@ -18,10 +18,7 @@ def _both_accepted(eval_a: Evaluation, eval_b: Evaluation) -> bool:
18
18
 
19
19
 
20
20
  def _any_tle(eval_a: Evaluation, eval_b: Evaluation) -> bool:
21
- return (
22
- eval_a.result.outcome == Outcome.TIME_LIMIT_EXCEEDED
23
- or eval_b.result.outcome == Outcome.TIME_LIMIT_EXCEEDED
24
- )
21
+ return eval_a.result.outcome.is_slow() or eval_b.result.outcome.is_slow()
25
22
 
26
23
 
27
24
  def _get_faster(eval_a: Evaluation, eval_b: Evaluation) -> Evaluation:
@@ -130,13 +127,10 @@ class Retrier:
130
127
 
131
128
  def should_repeat(self, eval: Evaluation) -> bool:
132
129
  if self.is_stress:
133
- if (
134
- eval.result.outcome == Outcome.TIME_LIMIT_EXCEEDED
135
- and self.retries_for_stress > 0
136
- ):
130
+ if eval.result.outcome.is_slow() and self.retries_for_stress > 0:
137
131
  self.retries_for_stress -= 1
138
132
  return True
139
- if eval.result.outcome == Outcome.TIME_LIMIT_EXCEEDED and self.retries > 0:
133
+ if eval.result.outcome.is_slow() and self.retries > 0:
140
134
  self.retries -= 1
141
135
  return True
142
136
  if self.reps > 0:
rbx/box/schema.py CHANGED
@@ -57,6 +57,9 @@ def expand_var(value: Primitive) -> Primitive:
57
57
 
58
58
 
59
59
  class ExpectedOutcome(AutoEnum):
60
+ ANY = alias('any') # type: ignore
61
+ """Expected outcome for any outcome."""
62
+
60
63
  ACCEPTED = alias('accepted', 'ac', 'correct') # type: ignore
61
64
  """Expected outcome for correct solutions (AC)."""
62
65
 
@@ -97,6 +100,8 @@ class ExpectedOutcome(AutoEnum):
97
100
  Especially useful for environments where TLE and RTE are indistinguishable."""
98
101
 
99
102
  def style(self) -> str:
103
+ if self == ExpectedOutcome.ANY:
104
+ return 'orange'
100
105
  if self == ExpectedOutcome.ACCEPTED:
101
106
  return 'green'
102
107
  if self == ExpectedOutcome.WRONG_ANSWER:
@@ -114,29 +119,39 @@ class ExpectedOutcome(AutoEnum):
114
119
  def is_slow(self) -> bool:
115
120
  return self in [ExpectedOutcome.TIME_LIMIT_EXCEEDED, ExpectedOutcome.TLE_OR_RTE]
116
121
 
122
+ def matches_tle_and_is_incorrect(self) -> bool:
123
+ return self.match(Outcome.TIME_LIMIT_EXCEEDED) and not self.match(
124
+ Outcome.ACCEPTED
125
+ )
126
+
117
127
  def match(self, outcome: Outcome) -> bool:
128
+ if self == ExpectedOutcome.ANY:
129
+ return True
118
130
  if self == ExpectedOutcome.ACCEPTED:
119
131
  return outcome == Outcome.ACCEPTED
120
132
  if self == ExpectedOutcome.ACCEPTED_OR_TLE:
121
- return outcome in {Outcome.ACCEPTED, Outcome.TIME_LIMIT_EXCEEDED}
133
+ return outcome in {Outcome.ACCEPTED} or outcome.is_slow()
122
134
  if self == ExpectedOutcome.WRONG_ANSWER:
123
135
  return outcome == Outcome.WRONG_ANSWER
124
136
  if self == ExpectedOutcome.INCORRECT:
125
- return outcome in {
126
- Outcome.WRONG_ANSWER,
127
- Outcome.RUNTIME_ERROR,
128
- Outcome.MEMORY_LIMIT_EXCEEDED,
129
- Outcome.TIME_LIMIT_EXCEEDED,
130
- Outcome.OUTPUT_LIMIT_EXCEEDED,
131
- }
137
+ return (
138
+ outcome
139
+ in {
140
+ Outcome.WRONG_ANSWER,
141
+ Outcome.RUNTIME_ERROR,
142
+ Outcome.MEMORY_LIMIT_EXCEEDED,
143
+ Outcome.OUTPUT_LIMIT_EXCEEDED,
144
+ }
145
+ or outcome.is_slow()
146
+ )
132
147
  if self == ExpectedOutcome.RUNTIME_ERROR:
133
148
  return outcome == Outcome.RUNTIME_ERROR
134
149
  if self == ExpectedOutcome.TIME_LIMIT_EXCEEDED:
135
- return outcome == Outcome.TIME_LIMIT_EXCEEDED
150
+ return outcome.is_slow()
136
151
  if self == ExpectedOutcome.MEMORY_LIMIT_EXCEEDED:
137
152
  return outcome == Outcome.MEMORY_LIMIT_EXCEEDED
138
153
  if self == ExpectedOutcome.TLE_OR_RTE:
139
- return outcome in {Outcome.TIME_LIMIT_EXCEEDED, Outcome.RUNTIME_ERROR}
154
+ return outcome in {Outcome.RUNTIME_ERROR} or outcome.is_slow()
140
155
  if self == ExpectedOutcome.OUTPUT_LIMIT_EXCEEDED:
141
156
  return outcome == Outcome.OUTPUT_LIMIT_EXCEEDED
142
157
  return False
rbx/box/solutions.py CHANGED
@@ -16,7 +16,7 @@ import typer
16
16
  from pydantic import BaseModel
17
17
 
18
18
  from rbx import console, utils
19
- from rbx.box import checkers, environment, package, state
19
+ from rbx.box import checkers, environment, package, remote, state
20
20
  from rbx.box.code import (
21
21
  SanitizationLevel,
22
22
  compile_item,
@@ -130,7 +130,7 @@ class RunSolutionResult:
130
130
 
131
131
  def is_fast(solution: Solution) -> bool:
132
132
  # If solution has TLE tag, it is considered slow.
133
- return not solution.outcome.match(Outcome.TIME_LIMIT_EXCEEDED)
133
+ return not solution.outcome.is_slow()
134
134
 
135
135
 
136
136
  def get_matching_solutions(expected_outcome: ExpectedOutcome) -> List[Solution]:
@@ -720,74 +720,38 @@ def _get_solution_repr(sol: Solution) -> List[Tuple[str, str]]:
720
720
  ]
721
721
 
722
722
 
723
- # TODO: refactor this function
724
- def _expand_remote_sol(sol: str) -> Optional[str]:
725
- from rbx.box.packaging.boca import upload as boca_upload
726
-
727
- # Only BOCA supported for now.
728
- assert sol.startswith('@')
729
- sol = sol[1:]
730
- if '-' in sol:
731
- run_number, site_number = sol.split('-', 1)
732
- run_number = int(run_number)
733
- site_number = int(site_number)
734
- else:
735
- run_number = int(sol)
736
- site_number = 1
737
-
738
- expected_glob = (
739
- package.get_problem_remote_dir('boca') / f'{run_number}-{site_number}.*'
740
- )
741
- expected_glob = expected_glob.relative_to(pathlib.Path.cwd())
742
- for path in pathlib.Path.cwd().glob(str(expected_glob)):
743
- if path.is_file():
744
- relpath = path.resolve().relative_to(pathlib.Path.cwd())
745
- console.console.print(f'Retrieving {href(relpath)} from cache...')
746
- return str(relpath)
747
-
748
- boca_uploader = boca_upload.get_boca_uploader()
749
- boca_uploader.login()
750
- sol_path = boca_uploader.download_run(
751
- run_number, site_number, package.get_problem_remote_dir('boca')
752
- )
753
- console.console.print(f'Downloaded {href(sol_path)} from BOCA...')
754
- return str(sol_path.resolve().relative_to(pathlib.Path.cwd()))
755
-
756
-
757
- def _expand_remote_sols(sols: List[str]) -> List[str]:
758
- res = []
759
- for sol in sols:
760
- if not sol.startswith('@'):
761
- res.append(sol)
762
- continue
763
- expanded = _expand_remote_sol(sol)
764
- if expanded is None:
765
- continue
766
- res.append(expanded)
767
- return res
768
-
769
-
770
- def expand_solutions(sols: List[str]) -> List[Solution]:
723
+ def expand_solutions_with_source(sols: List[str]) -> List[Tuple[Solution, bool]]:
771
724
  pkg = package.find_problem_package_or_die()
772
- seen_sols = {str(sol.path): sol for sol in pkg.solutions}
725
+ pkg_sols = {str(sol.path): sol for sol in pkg.solutions}
773
726
 
774
727
  # Download remote sols.
775
- sols = _expand_remote_sols(sols)
728
+ sols = remote.expand_files(sols)
776
729
 
777
730
  # Ensure sols exist.
778
731
  sols = [sol for sol in sols if pathlib.Path(sol).is_file()]
779
732
 
780
- res = []
733
+ seen_sols = set()
734
+ res: List[Tuple[Solution, bool]] = []
781
735
  for sol in sols:
782
736
  if sol in seen_sols:
783
- res.append(seen_sols[sol])
737
+ # This solution was already added.
738
+ continue
739
+ if sol in pkg_sols:
740
+ # This solution is in the package.
741
+ res.append((pkg_sols[sol], False))
784
742
  else:
743
+ # This solution is fetched from some source.
785
744
  res.append(
786
- Solution(path=pathlib.Path(sol), outcome=ExpectedOutcome.ACCEPTED)
745
+ (Solution(path=pathlib.Path(sol), outcome=ExpectedOutcome.ANY), True)
787
746
  )
747
+ seen_sols.add(sol)
788
748
  return res
789
749
 
790
750
 
751
+ def expand_solutions(sols: List[str]) -> List[Solution]:
752
+ return [sol for sol, _ in expand_solutions_with_source(sols)]
753
+
754
+
791
755
  async def pick_solutions(
792
756
  tracked_solutions: Optional[Set[str]],
793
757
  extra_solutions: Optional[List[str]] = None,
@@ -830,7 +794,7 @@ def get_outcome_style_verdict(outcome: Outcome) -> str:
830
794
  return 'green'
831
795
  if outcome == Outcome.WRONG_ANSWER:
832
796
  return 'red'
833
- if outcome == Outcome.TIME_LIMIT_EXCEEDED:
797
+ if outcome.is_slow():
834
798
  return 'yellow'
835
799
  if outcome == Outcome.RUNTIME_ERROR:
836
800
  return 'blue'
@@ -843,7 +807,7 @@ def get_outcome_markup_verdict(outcome: Outcome) -> str:
843
807
  res = '✓'
844
808
  if outcome != Outcome.ACCEPTED:
845
809
  res = '✗'
846
- if outcome == Outcome.TIME_LIMIT_EXCEEDED:
810
+ if outcome.is_slow():
847
811
  res = '⧖'
848
812
  if outcome == Outcome.RUNTIME_ERROR:
849
813
  res = '✗'
@@ -873,6 +837,19 @@ def get_full_testcase_markup_verdict(eval: Evaluation) -> str:
873
837
  def _get_evals_time_in_ms(evals: List[Evaluation]) -> int:
874
838
  if not evals:
875
839
  return 0
840
+ evals_with_ile = [
841
+ eval for eval in evals if eval.result.outcome == Outcome.IDLENESS_LIMIT_EXCEEDED
842
+ ]
843
+ for eval in evals_with_ile:
844
+ # Try every way of estimating a ILE max timelimit.
845
+ if eval.log.metadata is None:
846
+ continue
847
+ if eval.log.metadata.limits is not None:
848
+ expanded_tl = eval.log.metadata.limits.get_expanded_tl()
849
+ if expanded_tl is not None:
850
+ return expanded_tl
851
+ if eval.log.metadata.timeLimit is not None:
852
+ return eval.log.metadata.timeLimit
876
853
  return max(int((eval.log.time or 0.0) * 1000) for eval in evals)
877
854
 
878
855
 
@@ -893,7 +870,10 @@ def get_capped_evals_formatted_time(
893
870
  pkg = package.find_problem_package_or_die()
894
871
 
895
872
  max_time = _get_evals_time_in_ms(evals)
896
- has_tle = any(eval.result.outcome == Outcome.TIME_LIMIT_EXCEEDED for eval in evals)
873
+ has_tle = any(eval.result.outcome.is_slow() for eval in evals)
874
+ has_ile = any(
875
+ eval.result.outcome == Outcome.IDLENESS_LIMIT_EXCEEDED for eval in evals
876
+ )
897
877
  timelimits = [
898
878
  eval.log.metadata.limits.get_expanded_tl()
899
879
  for eval in evals
@@ -911,7 +891,7 @@ def get_capped_evals_formatted_time(
911
891
  # Using double TL for verification.
912
892
  tl = tl * 2
913
893
 
914
- if has_tle and max_time >= tl:
894
+ if has_tle and max_time >= tl or has_ile:
915
895
  return f'>{tl} ms'
916
896
  return f'{max_time} ms'
917
897
 
@@ -996,8 +976,7 @@ def get_solution_outcome_report(
996
976
  ):
997
977
  no_tle_bad_verdicts.add(eval.result.no_tle_outcome)
998
978
  has_plain_tle = has_plain_tle or (
999
- eval.result.outcome == Outcome.TIME_LIMIT_EXCEEDED
1000
- and eval.result.no_tle_outcome is None
979
+ eval.result.outcome.is_slow() and eval.result.no_tle_outcome is None
1001
980
  )
1002
981
  has_sanitizer_warnings = (
1003
982
  has_sanitizer_warnings or eval.result.sanitizer_warnings
@@ -1030,9 +1009,7 @@ def get_solution_outcome_report(
1030
1009
  report_got_verdicts = {Outcome.ACCEPTED}
1031
1010
 
1032
1011
  evals_time = _get_evals_time_in_ms(evals)
1033
- expected_outcome_is_tle = solution.outcome.match(
1034
- Outcome.TIME_LIMIT_EXCEEDED
1035
- ) and not solution.outcome.match(Outcome.ACCEPTED)
1012
+ expected_outcome_is_tle = solution.outcome.matches_tle_and_is_incorrect()
1036
1013
  if (
1037
1014
  # Running verification with double TL.
1038
1015
  verification.value >= VerificationLevel.FULL.value
rbx/box/stresses.py CHANGED
@@ -8,7 +8,7 @@ import typer
8
8
  from pydantic import BaseModel
9
9
 
10
10
  from rbx import console
11
- from rbx.box import checkers, package, validators
11
+ from rbx.box import checkers, generators, package, validators
12
12
  from rbx.box.code import SanitizationLevel, compile_item, run_item
13
13
  from rbx.box.generators import (
14
14
  GenerationMetadata,
@@ -51,10 +51,10 @@ def _compile_finder(finder: CodeItem) -> str:
51
51
 
52
52
 
53
53
  async def run_stress(
54
- name: str,
55
54
  timeoutInSeconds: int,
55
+ name: Optional[str] = None,
56
56
  finder: Optional[str] = None,
57
- args: Optional[str] = None,
57
+ generator_call: Optional[str] = None,
58
58
  findingsLimit: int = 1,
59
59
  verbose: bool = False,
60
60
  progress: Optional[StatusProgress] = None,
@@ -68,12 +68,23 @@ async def run_stress(
68
68
  raise typer.Exit(1)
69
69
 
70
70
  if finder:
71
+ if generator_call is None:
72
+ console.console.print(
73
+ '[error]Generator arguments are required for stress testing. Specify them through the [item]-g[/item] flag.[/error]'
74
+ )
75
+ raise typer.Exit(1)
76
+ generator = generators.get_call_from_string(generator_call)
71
77
  stress = Stress(
72
- name=f'{name}',
73
- generator=GeneratorCall(name=name, args=args or ''),
78
+ name=f'{generator.name}',
79
+ generator=generator,
74
80
  finder=finder,
75
81
  )
76
82
  else:
83
+ if name is None:
84
+ console.console.print(
85
+ '[error]Invalid stress test paramaters. Either provide a stress test name, or provide a finder expression (-f) and generator arguments (-g).[/error]'
86
+ )
87
+ raise typer.Exit(1)
77
88
  stress = package.get_stress(name)
78
89
 
79
90
  call = stress.generator
@@ -258,7 +269,7 @@ async def run_stress(
258
269
 
259
270
  if internal_error_results:
260
271
  console.console.print(
261
- f'[error]Checkers failed during stress test [item]{name}[/item] with args [info]{expanded_generator_call.name} {expanded_generator_call.args}[/info][/error]'
272
+ f'[error]Checkers failed during stress test [item]{stress.name}[/item] with args [info]{expanded_generator_call.name} {expanded_generator_call.args}[/info][/error]'
262
273
  )
263
274
  for internal_error_result in internal_error_results:
264
275
  assert internal_error_result.checker is not None