rbx.cp 0.5.27__tar.gz → 0.5.28__tar.gz

{rbx_cp-0.5.27 → rbx_cp-0.5.28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rbx.cp
-Version: 0.5.27
+Version: 0.5.28
 Summary: 
 Author: Roberto Sales
 Requires-Python: >=3.9,<4.0

{rbx_cp-0.5.27 → rbx_cp-0.5.28}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "rbx.cp"
-version = "0.5.27"
+version = "0.5.28"
 description = ""
 packages = [
   {include = "rbx"}

{rbx_cp-0.5.27 → rbx_cp-0.5.28}/rbx/box/code.py

@@ -266,6 +266,7 @@ def run_item(
     outputs: Optional[List[GradingFileOutput]] = None,
     extra_args: Optional[str] = None,
     extra_config: Optional[ExecutionConfig] = None,
+    retry_index: Optional[int] = None,
 ) -> Optional[RunLog]:
     language = find_language_name(code)
     execution_options = get_execution_config(language)
@@ -350,6 +351,7 @@ def run_item(
             is_sanitized=sanitized,
             timeLimit=sandbox_params.timeout,
             memoryLimit=sandbox_params.address_space,
+            retryIndex=retry_index,
         ),
     )
 

rbx_cp-0.5.28/rbx/box/retries.py

@@ -0,0 +1,143 @@
+import dataclasses
+import pathlib
+import shutil
+import tempfile
+from contextlib import contextmanager
+from typing import Callable, List, Optional
+
+from rbx.box import package
+from rbx.box.setter_config import RepeatsConfig, get_setter_config
+from rbx.grading.steps import Evaluation, Outcome
+
+
+def _both_accepted(eval_a: Evaluation, eval_b: Evaluation) -> bool:
+    return (
+        eval_a.result.outcome == Outcome.ACCEPTED
+        and eval_b.result.outcome == Outcome.ACCEPTED
+    )
+
+
+def _any_tle(eval_a: Evaluation, eval_b: Evaluation) -> bool:
+    return (
+        eval_a.result.outcome == Outcome.TIME_LIMIT_EXCEEDED
+        or eval_b.result.outcome == Outcome.TIME_LIMIT_EXCEEDED
+    )
+
+
+def _get_faster(eval_a: Evaluation, eval_b: Evaluation) -> Evaluation:
+    if eval_a.log.time is None:
+        return eval_b
+    if eval_b.log.time is None:
+        return eval_a
+    if eval_a.log.time < eval_b.log.time:
+        return eval_a
+    return eval_b
+
+
+def _merge_evaluations(eval_a: Evaluation, eval_b: Evaluation) -> Evaluation:
+    if _both_accepted(eval_a, eval_b) or _any_tle(eval_a, eval_b):
+        return _get_faster(eval_a, eval_b)
+    if eval_a.result.outcome != Outcome.ACCEPTED:
+        return eval_a
+    if eval_b.result.outcome != Outcome.ACCEPTED:
+        return eval_b
+    return _get_faster(eval_a, eval_b)
+
+
+@contextmanager
+def _temp_retry_dir():
+    """Create a temporary directory for retry artifacts."""
+    temp_dir = tempfile.mkdtemp(prefix='rbx_retry_')
+    try:
+        yield pathlib.Path(temp_dir)
+    finally:
+        shutil.rmtree(temp_dir, ignore_errors=True)
+
+
+@dataclasses.dataclass
+class FileToRecover:
+    from_path: pathlib.Path
+    to_path: pathlib.Path
+
+
+def _move_to_temp_dir(path: pathlib.Path, temp_dir: pathlib.Path) -> FileToRecover:
+    problem_path = package.find_problem()
+    path = path.resolve()
+    temp_dir = temp_dir.resolve()
+    relative = path.relative_to(problem_path)
+
+    temp_path = temp_dir / relative
+    temp_path.parent.mkdir(parents=True, exist_ok=True)
+    shutil.move(path, temp_path)
+    return FileToRecover(temp_path, path)
+
+
+def _move_logs_to_temp_dir(
+    eval: Evaluation, temp_dir: pathlib.Path
+) -> List[FileToRecover]:
+    recover = []
+    if (
+        eval.log.stdout_absolute_path is not None
+        and eval.log.stdout_absolute_path.exists()
+    ):
+        recover.append(_move_to_temp_dir(eval.log.stdout_absolute_path, temp_dir))
+    if (
+        eval.log.stderr_absolute_path is not None
+        and eval.log.stderr_absolute_path.exists()
+    ):
+        recover.append(_move_to_temp_dir(eval.log.stderr_absolute_path, temp_dir))
+    if eval.log.log_absolute_path is not None and eval.log.log_absolute_path.exists():
+        recover.append(_move_to_temp_dir(eval.log.log_absolute_path, temp_dir))
+    return recover
+
+
+class Retrier:
+    def __init__(self, config: Optional[RepeatsConfig] = None, is_stress: bool = False):
+        self.config = config or get_setter_config().repeats
+        self.is_stress = is_stress
+
+        self.reset()
+
+    def reset(self):
+        self.reps = self.config.reps - 1
+        self.retries = self.config.retries
+        self.retries_for_stress = self.config.retries_for_stress
+        self.retry_index = 0
+
+    def repeat(
+        self,
+        func: Callable[[int], Evaluation],
+    ) -> Evaluation:
+        self.retry_index += 1
+        eval = func(self.retry_index)
+        if self.should_repeat(eval):
+            with _temp_retry_dir() as temp_dir:
+                # Move files to temp dir to open run for repeat.
+                recover = _move_logs_to_temp_dir(eval, temp_dir)
+                # Actually repeat and choose the best evaluation.
+                next_eval = self.repeat(func)
+                chosen_eval = _merge_evaluations(eval, next_eval)
+
+                if id(chosen_eval) == id(eval):
+                    # Must recover originally moved files.
+                    for file in recover:
+                        file.to_path.parent.mkdir(parents=True, exist_ok=True)
+                        shutil.move(file.from_path, file.to_path)
+                return chosen_eval
+        return eval
+
+    def should_repeat(self, eval: Evaluation) -> bool:
+        if self.is_stress:
+            if (
+                eval.result.outcome == Outcome.TIME_LIMIT_EXCEEDED
+                and self.retries_for_stress > 0
+            ):
+                self.retries_for_stress -= 1
+                return True
+        if eval.result.outcome == Outcome.TIME_LIMIT_EXCEEDED and self.retries > 0:
+            self.retries -= 1
+            return True
+        if self.reps > 0:
+            self.reps -= 1
+            return True
+        return False

{rbx_cp-0.5.27 → rbx_cp-0.5.28}/rbx/box/setter_config.py

@@ -36,6 +36,23 @@ class WarningsConfig(BaseModel):
     )
 
 
+class RepeatsConfig(BaseModel):
+    reps: int = Field(
+        1,
+        description='Number of times to repeat the solution.',
+    )
+
+    retries: int = Field(
+        0,
+        description='Number of times to retry if the solution TLs.',
+    )
+
+    retries_for_stress: int = Field(
+        0,
+        description='Number of times to retry in stress mode if the solution TLs.',
+    )
+
+
 class SetterConfig(BaseModel):
     sanitizers: SanitizersConfig = Field(
         default_factory=SanitizersConfig,  # type: ignore
@@ -46,6 +63,11 @@ class SetterConfig(BaseModel):
         description='Configuration for warnings.',
     )
 
+    repeats: RepeatsConfig = Field(
+        default_factory=RepeatsConfig,  # type: ignore
+        description='Configuration for repeats.',
+    )
+
     command_substitutions: Dict[str, str] = Field(
         {},
         description='Substitutions to apply to commands before running them.',

{rbx_cp-0.5.27 → rbx_cp-0.5.28}/rbx/box/solutions.py

@@ -26,6 +26,7 @@ from rbx.box.environment import (
     VerificationLevel,
 )
 from rbx.box.generators import generate_output_for_testcase, generate_standalone
+from rbx.box.retries import Retrier
 from rbx.box.schema import (
     ExpectedOutcome,
     GeneratorCall,
@@ -169,63 +170,70 @@ def _run_solution_on_testcase(
     verification: VerificationLevel = VerificationLevel.NONE,
     timelimit_override: Optional[int] = None,
 ) -> Evaluation:
-    actual_sandbox = package.get_singleton_sandbox()
+    def run_fn(retry_index: int) -> Evaluation:
+        actual_sandbox = package.get_singleton_sandbox()
 
-    limits = get_limits_for_language(
-        solution.language, verification, timelimit_override
-    )
+        limits = get_limits_for_language(
+            solution.language, verification, timelimit_override
+        )
 
-    sandbox = EnvironmentSandbox()
-    sandbox.timeLimit = limits.time
-    if limits.isDoubleTL and sandbox.timeLimit is not None:
-        # Double TL.
-        sandbox.timeLimit = sandbox.timeLimit * 2
-    sandbox.wallTimeLimit = sandbox.timeLimit
-    if sandbox.timeLimit is not None and actual_sandbox.use_soft_timeout():
-        sandbox.wallTimeLimit = sandbox.timeLimit * 2
-    sandbox.memoryLimit = limits.memory
-    sandbox.fileSizeLimit = limits.output
-    extra_config = ExecutionConfig(sandbox=sandbox)
-
-    output_path = output_dir / testcase.inputPath.with_suffix('.out').name
-    error_path = output_path.with_suffix('.err')
-    log_path = output_path.with_suffix('.log')
-    output_path.parent.mkdir(parents=True, exist_ok=True)
-
-    run_log = run_item(
-        solution,
-        DigestOrSource.create(compiled_digest),
-        stdin=DigestOrSource.create(testcase.inputPath),
-        stdout=DigestOrDest.create(output_path),
-        stderr=DigestOrDest.create(error_path),
-        extra_config=extra_config,
-    )
+        sandbox = EnvironmentSandbox()
+        sandbox.timeLimit = limits.time
+        if limits.isDoubleTL and sandbox.timeLimit is not None:
+            # Double TL.
+            sandbox.timeLimit = sandbox.timeLimit * 2
+        sandbox.wallTimeLimit = sandbox.timeLimit
+        if sandbox.timeLimit is not None and actual_sandbox.use_soft_timeout():
+            sandbox.wallTimeLimit = sandbox.timeLimit * 2
+        sandbox.memoryLimit = limits.memory
+        sandbox.fileSizeLimit = limits.output
+        extra_config = ExecutionConfig(sandbox=sandbox)
+
+        output_path = output_dir / testcase.inputPath.with_suffix('.out').name
+        error_path = output_path.with_suffix('.err')
+        log_path = output_path.with_suffix('.log')
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+
+        run_log = run_item(
+            solution,
+            DigestOrSource.create(compiled_digest),
+            stdin=DigestOrSource.create(testcase.inputPath),
+            stdout=DigestOrDest.create(output_path),
+            stderr=DigestOrDest.create(error_path),
+            extra_config=extra_config,
+            retry_index=retry_index,
+        )
 
-    if checker_digest is not None:
-        checker_result = checkers.check(
-            checker_digest,
-            run_log,
-            testcase,
-            program_output=output_path,
+        if checker_digest is not None:
+            checker_result = checkers.check(
+                checker_digest,
+                run_log,
+                testcase,
+                program_output=output_path,
+            )
+        else:
+            checker_result = checkers.check_with_no_output(run_log)
+
+        eval = Evaluation(
+            result=checker_result,
+            testcase=TestcaseIO(
+                index=testcase_index,
+                input=testcase.inputPath,
+                output=testcase.outputPath,
+            ),
+            log=TestcaseLog(
+                **(run_log.model_dump() if run_log is not None else {}),
+                stdout_absolute_path=output_path.absolute(),
+                stderr_absolute_path=error_path.absolute(),
+                log_absolute_path=log_path.absolute(),
+            ),
         )
-    else:
-        checker_result = checkers.check_with_no_output(run_log)
 
-    eval = Evaluation(
-        result=checker_result,
-        testcase=TestcaseIO(
-            index=testcase_index, input=testcase.inputPath, output=testcase.outputPath
-        ),
-        log=TestcaseLog(
-            **(run_log.model_dump() if run_log is not None else {}),
-            stdout_absolute_path=output_path.absolute(),
-            stderr_absolute_path=error_path.absolute(),
-            log_absolute_path=log_path.absolute(),
-        ),
-    )
+        log_path.write_text(model_to_yaml(eval))
+        return eval
 
-    log_path.write_text(model_to_yaml(eval))
-    return eval
+    retrier = Retrier()
+    return retrier.repeat(run_fn)
 
 
 def _run_solution(

{rbx_cp-0.5.27 → rbx_cp-0.5.28}/rbx/box/stresses.py

@@ -10,13 +10,17 @@ from rbx import console
 from rbx.box import checkers, package, validators
 from rbx.box.code import SanitizationLevel, compile_item, run_item
 from rbx.box.generators import generate_standalone
+from rbx.box.retries import Retrier
 from rbx.box.schema import CodeItem, GeneratorCall, Stress, Testcase
 from rbx.box.solutions import compile_solutions, get_outcome_style_verdict
 from rbx.box.stressing import finder_parser
 from rbx.grading.steps import (
     DigestOrDest,
     DigestOrSource,
+    Evaluation,
     Outcome,
+    TestcaseIO,
+    TestcaseLog,
 )
 from rbx.utils import StatusProgress
 
@@ -131,8 +135,9 @@ def run_stress(
         @functools.cache
         def run_solution_fn(
             solution: str,
+            retry_index: Optional[int] = None,
             input_path=input_path,
-        ) -> finder_parser.FinderSolutionResult:
+        ) -> TestcaseLog:
             index = solution_indices[solution]
             sol = solutions[index]
             output_path = input_path.with_stem(f'{index}').with_suffix('.out')
@@ -144,29 +149,30 @@ def run_stress(
                 stdin=DigestOrSource.create(input_path),
                 stdout=DigestOrDest.create(output_path),
                 stderr=DigestOrDest.create(stderr_path),
+                retry_index=retry_index,
             )
 
-            return finder_parser.FinderSolutionResult(
-                output_path=output_path,
-                stderr_path=stderr_path,
-                run_log=run_log,
+            return TestcaseLog(
+                **(run_log.model_dump() if run_log is not None else {}),
+                stdout_absolute_path=output_path.absolute(),
+                stderr_absolute_path=stderr_path.absolute(),
             )
 
         # Get main solution output.
         expected_output_path = empty_path
         if needs_expected_output:
-            main_result = run_solution_fn(str(solutions[0].path))
-            main_checker_result = checkers.check_with_no_output(main_result.run_log)
+            main_testcase_log = run_solution_fn(str(solutions[0].path))
+            main_checker_result = checkers.check_with_no_output(main_testcase_log)
             if main_checker_result.outcome != Outcome.ACCEPTED:
                 console.console.print(
                     '[error]Error while generating main solution output.[/error]'
                 )
                 console.console.print(f'Input written at [item]{input_path}[/item]')
                 console.console.print(
-                    f'Output written at [item]{main_result.output_path}[/item]'
+                    f'Output written at [item]{main_testcase_log.stdout_absolute_path}[/item]'
                 )
                 console.console.print(
-                    f'Stderr written at [item]{main_result.stderr_path}[/item]'
+                    f'Stderr written at [item]{main_testcase_log.stderr_absolute_path}[/item]'
                 )
                 console.console.print()
                 console.console.print(
@@ -174,7 +180,7 @@ def run_stress(
                     "use the two-way modifier in your finder expression (':2')."
                 )
                 raise typer.Exit(1)
-            expected_output_path = main_result.output_path
+            expected_output_path = main_testcase_log.stdout_absolute_path
 
         @functools.cache
         def run_solution_and_checker_fn(
@@ -182,27 +188,43 @@ def run_stress(
             input_path=input_path,
             expected_output_path=expected_output_path,
         ) -> finder_parser.FinderResult:
-            solution = call.solution
-            checker = call.checker
-
-            solution_result = run_solution_fn(solution)
-
-            if checker is None:
-                checker_result = checkers.check_with_no_output(solution_result.run_log)
-            else:
-                checker_digest = finders_digest[checker.path]
-                checker_result = checkers.check(
-                    checker_digest,
-                    solution_result.run_log,
-                    Testcase(inputPath=input_path, outputPath=expected_output_path),
-                    program_output=solution_result.output_path,
+            def run_fn(retry_index: int) -> Evaluation:
+                solution = call.solution
+                checker = call.checker
+
+                testcase_log = run_solution_fn(solution, retry_index=retry_index)
+                assert testcase_log.stdout_absolute_path is not None
+
+                if checker is None:
+                    checker_result = checkers.check_with_no_output(testcase_log)
+                else:
+                    checker_digest = finders_digest[checker.path]
+                    checker_result = checkers.check(
+                        checker_digest,
+                        testcase_log,
+                        Testcase(inputPath=input_path, outputPath=expected_output_path),
+                        program_output=testcase_log.stdout_absolute_path,
+                    )
+
+                return Evaluation(
+                    result=checker_result,
+                    testcase=TestcaseIO(
+                        index=0,
+                        input=input_path,
+                        output=expected_output_path,
+                    ),
+                    log=testcase_log,
                 )
+
+            retrier = Retrier(is_stress=True)
+            eval = retrier.repeat(run_fn)
+
             return finder_parser.FinderResult(
-                solution=solution,
-                outcome=checker_result.outcome,
-                checker=checker,
-                solution_result=solution_result,
-                checker_result=checker_result,
+                solution=call.solution,
+                outcome=eval.result.outcome,
+                checker=call.checker,
+                solution_log=eval.log,
+                checker_result=eval.result,
             )
 
         runner = finder_parser.FinderTreeRunner(runner=run_solution_and_checker_fn)

{rbx_cp-0.5.27 → rbx_cp-0.5.28}/rbx/box/stressing/finder_parser.py

@@ -10,7 +10,7 @@ import typer
 from rbx import console
 from rbx.box import package
 from rbx.box.schema import CodeItem, ExpectedOutcome
-from rbx.grading.steps import CheckerResult, Outcome, RunLog
+from rbx.grading.steps import CheckerResult, Outcome, RunLog, TestcaseLog
 
 LARK_GRAMMAR = r"""
 // A bunch of words
@@ -102,7 +102,7 @@ class FinderResult:
     checker: Optional[FinderChecker]
 
     # Auxiliary information.
-    solution_result: Optional[FinderSolutionResult] = None
+    solution_log: Optional[TestcaseLog] = None
     checker_result: Optional[CheckerResult] = None
 
 

{rbx_cp-0.5.27 → rbx_cp-0.5.28}/rbx/grading/steps.py

@@ -170,6 +170,7 @@ class RunLogMetadata(BaseModel):
     is_sanitized: bool = False
     timeLimit: Optional[int] = None
     memoryLimit: Optional[int] = None
+    retryIndex: Optional[int] = None
 
 
 class RunLog(BaseModel):

{rbx_cp-0.5.27 → rbx_cp-0.5.28}/rbx/grading/steps_with_caching.py

@@ -46,9 +46,11 @@ def run(
 ) -> Optional[RunLog]:
     artifacts.logs = GradingLogsHolder()
 
-    with dependency_cache(
-        [command], [artifacts], params.get_cacheable_params()
-    ) as is_cached:
+    cacheable_params = params.get_cacheable_params()
+    if metadata is not None and metadata.retryIndex is not None:
+        cacheable_params['__retry_index__'] = metadata.retryIndex
+
+    with dependency_cache([command], [artifacts], cacheable_params) as is_cached:
         if not is_cached:
             steps.run(
                 command=command,

{rbx_cp-0.5.27 → rbx_cp-0.5.28}/rbx/resources/default_setter_config.mac.yml

@@ -14,6 +14,14 @@ command_substitutions:
   python2: python2
   python3: python3
 
+repeats:
+  # Number of times to run the solution.
+  reps: 1
+  # Number of times to retry if the solution TLs.
+  retries: 0
+  # Number of times to retry in stress mode if the solution TLs.
+  retries_for_stress: 0
+
 # Whether sanitizers will be enabled by default
 # when running testlib components.
 # This flag has no effect on running solutions with

{rbx_cp-0.5.27 → rbx_cp-0.5.28}/rbx/resources/default_setter_config.yml

@@ -14,6 +14,14 @@ command_substitutions:
   python2: python2
   python3: python3
 
+repeats:
+  # Number of times to run the solution.
+  reps: 1
+  # Number of times to retry if the solution TLs.
+  retries: 0
+  # Number of times to retry in stress mode if the solution TLs.
+  retries_for_stress: 0
+
 # Whether sanitizers will be enabled by default
 # when running testlib components.
 # This flag has no effect on running solutions with