rbx.cp 0.5.27__py3-none-any.whl → 0.5.29__py3-none-any.whl

rbx/box/main.py

@@ -48,6 +48,7 @@ from rbx.box.solutions import (
     run_solutions,
 )
 from rbx.box.statements import build_statements
+from rbx.box.testcases import TestcaseEntry
 
 app = typer.Typer(no_args_is_help=True, cls=annotations.AliasGroup)
 app.add_typer(
@@ -343,6 +344,14 @@ def irun(
         '-g',
         help='Generator call to use to generate a single test for execution.',
     ),
+    testcase: Optional[str] = typer.Option(
+        None,
+        '--testcase',
+        '--test',
+        '-tc',
+        '-t',
+        help='Testcase to run, in the format "[group]/[index]". If not specified, will run interactively.',
+    ),
     print: bool = typer.Option(
         False, '--print', '-p', help='Whether to print outputs to terminal.'
     ),
@@ -370,13 +379,6 @@ def irun(
         )
         return
 
-    main_solution = package.get_main_solution()
-    if check and main_solution is None:
-        console.console.print(
-            '[warning]No main solution found, running without checkers.[/warning]'
-        )
-        check = False
-
     tracked_solutions = None
     if outcome is not None:
         tracked_solutions = {
@@ -411,6 +413,7 @@ def irun(
                 generator=generators.get_call_from_string(generator)
                 if generator is not None
                 else None,
+                testcase_entry=TestcaseEntry.parse(testcase) if testcase else None,
                 print=print,
                 sanitized=sanitized,
             )

rbx/box/retries.py

@@ -0,0 +1,143 @@
+import dataclasses
+import pathlib
+import shutil
+import tempfile
+from contextlib import contextmanager
+from typing import Callable, List, Optional
+
+from rbx.box import package
+from rbx.box.setter_config import RepeatsConfig, get_setter_config
+from rbx.grading.steps import Evaluation, Outcome
+
+
+def _both_accepted(eval_a: Evaluation, eval_b: Evaluation) -> bool:
+    return (
+        eval_a.result.outcome == Outcome.ACCEPTED
+        and eval_b.result.outcome == Outcome.ACCEPTED
+    )
+
+
+def _any_tle(eval_a: Evaluation, eval_b: Evaluation) -> bool:
+    return (
+        eval_a.result.outcome == Outcome.TIME_LIMIT_EXCEEDED
+        or eval_b.result.outcome == Outcome.TIME_LIMIT_EXCEEDED
+    )
+
+
+def _get_faster(eval_a: Evaluation, eval_b: Evaluation) -> Evaluation:
+    if eval_a.log.time is None:
+        return eval_b
+    if eval_b.log.time is None:
+        return eval_a
+    if eval_a.log.time < eval_b.log.time:
+        return eval_a
+    return eval_b
+
+
+def _merge_evaluations(eval_a: Evaluation, eval_b: Evaluation) -> Evaluation:
+    if _both_accepted(eval_a, eval_b) or _any_tle(eval_a, eval_b):
+        return _get_faster(eval_a, eval_b)
+    if eval_a.result.outcome != Outcome.ACCEPTED:
+        return eval_a
+    if eval_b.result.outcome != Outcome.ACCEPTED:
+        return eval_b
+    return _get_faster(eval_a, eval_b)
+
+
+@contextmanager
+def _temp_retry_dir():
+    """Create a temporary directory for retry artifacts."""
+    temp_dir = tempfile.mkdtemp(prefix='rbx_retry_')
+    try:
+        yield pathlib.Path(temp_dir)
+    finally:
+        shutil.rmtree(temp_dir, ignore_errors=True)
+
+
+@dataclasses.dataclass
+class FileToRecover:
+    from_path: pathlib.Path
+    to_path: pathlib.Path
+
+
+def _move_to_temp_dir(path: pathlib.Path, temp_dir: pathlib.Path) -> FileToRecover:
+    problem_path = package.find_problem()
+    path = path.resolve()
+    temp_dir = temp_dir.resolve()
+    relative = path.relative_to(problem_path)
+
+    temp_path = temp_dir / relative
+    temp_path.parent.mkdir(parents=True, exist_ok=True)
+    shutil.move(path, temp_path)
+    return FileToRecover(temp_path, path)
+
+
+def _move_logs_to_temp_dir(
+    eval: Evaluation, temp_dir: pathlib.Path
+) -> List[FileToRecover]:
+    recover = []
+    if (
+        eval.log.stdout_absolute_path is not None
+        and eval.log.stdout_absolute_path.exists()
+    ):
+        recover.append(_move_to_temp_dir(eval.log.stdout_absolute_path, temp_dir))
+    if (
+        eval.log.stderr_absolute_path is not None
+        and eval.log.stderr_absolute_path.exists()
+    ):
+        recover.append(_move_to_temp_dir(eval.log.stderr_absolute_path, temp_dir))
+    if eval.log.log_absolute_path is not None and eval.log.log_absolute_path.exists():
+        recover.append(_move_to_temp_dir(eval.log.log_absolute_path, temp_dir))
+    return recover
+
+
+class Retrier:
+    def __init__(self, config: Optional[RepeatsConfig] = None, is_stress: bool = False):
+        self.config = config or get_setter_config().repeats
+        self.is_stress = is_stress
+
+        self.reset()
+
+    def reset(self):
+        self.reps = self.config.reps - 1
+        self.retries = self.config.retries
+        self.retries_for_stress = self.config.retries_for_stress
+        self.retry_index = 0
+
+    def repeat(
+        self,
+        func: Callable[[int], Evaluation],
+    ) -> Evaluation:
+        self.retry_index += 1
+        eval = func(self.retry_index)
+        if self.should_repeat(eval):
+            with _temp_retry_dir() as temp_dir:
+                # Move files to temp dir to open run for repeat.
+                recover = _move_logs_to_temp_dir(eval, temp_dir)
+                # Actually repeat and choose the best evaluation.
+                next_eval = self.repeat(func)
+                chosen_eval = _merge_evaluations(eval, next_eval)
+
+                if id(chosen_eval) == id(eval):
+                    # Must recover originally moved files.
+                    for file in recover:
+                        file.to_path.parent.mkdir(parents=True, exist_ok=True)
+                        shutil.move(file.from_path, file.to_path)
+                return chosen_eval
+        return eval
+
+    def should_repeat(self, eval: Evaluation) -> bool:
+        if self.is_stress:
+            if (
+                eval.result.outcome == Outcome.TIME_LIMIT_EXCEEDED
+                and self.retries_for_stress > 0
+            ):
+                self.retries_for_stress -= 1
+                return True
+        if eval.result.outcome == Outcome.TIME_LIMIT_EXCEEDED and self.retries > 0:
+            self.retries -= 1
+            return True
+        if self.reps > 0:
+            self.reps -= 1
+            return True
+        return False

rbx/box/schema.py

@@ -20,6 +20,12 @@ def NameField(**kwargs):
     )
 
 
+def FNameField(**kwargs):
+    return Field(
+        pattern=r'^[a-zA-Z0-9][a-zA-Z0-9\-_]*$', min_length=3, max_length=128, **kwargs
+    )
+
+
 def _check_oneof(model_obj: BaseModel, fields: List[str]):
     has = []
     for field in fields:
@@ -100,9 +106,9 @@ class ExpectedOutcome(AutoEnum):
         if self.match(Outcome.TIME_LIMIT_EXCEEDED):
             return 'yellow'
         if self.match(Outcome.RUNTIME_ERROR):
-            return 'lnumber'
+            return 'blue'
         if self.match(Outcome.MEMORY_LIMIT_EXCEEDED):
-            return 'cyan'
+            return 'yellow'
         return 'magenta'
 
     def is_slow(self) -> bool:
@@ -173,14 +179,14 @@ class Testcase(BaseModel):
     inputPath: pathlib.Path = Field(description="""The path of the input file.""")
 
     outputPath: Optional[pathlib.Path] = Field(
-        None, description="""The path of the output file."""
+        default=None, description="""The path of the output file."""
     )
 
 
 class GeneratorCall(BaseModel):
     model_config = ConfigDict(extra='forbid')
 
-    name: str = NameField(description='The name of the generator to call.')
+    name: str = FNameField(description='The name of the generator to call.')
 
     args: Optional[str] = Field(
         None, description='The arguments to pass to the generator.'
@@ -403,12 +409,15 @@ that is correct and used as reference -- and should have the `accepted` outcome.
         return res
 
     @model_validator(mode='after')
-    def check_first_solution_is_main(self):
+    def check_first_solution_is_main_if_there_is_ac(self):
+        if all(sol.outcome != Outcome.ACCEPTED for sol in self.solutions):
+            # No main solution.
+            return self
         if self.solutions:
             if self.solutions[0].outcome != ExpectedOutcome.ACCEPTED:
                 raise PydanticCustomError(
                     'MISSING_MAIN_SOLUTION',
-                    'The first solution in the package must have the "ACCEPTED" outcome.',
+                    'The first solution in the package must have the "ACCEPTED" outcome if there are ACCEPTED solutions.',
                 )
         return self
 

rbx/box/setter_config.py

@@ -36,6 +36,23 @@ class WarningsConfig(BaseModel):
     )
 
 
+class RepeatsConfig(BaseModel):
+    reps: int = Field(
+        1,
+        description='Number of times to repeat the solution.',
+    )
+
+    retries: int = Field(
+        0,
+        description='Number of times to retry if the solution TLs.',
+    )
+
+    retries_for_stress: int = Field(
+        0,
+        description='Number of times to retry in stress mode if the solution TLs.',
+    )
+
+
 class SetterConfig(BaseModel):
     sanitizers: SanitizersConfig = Field(
         default_factory=SanitizersConfig,  # type: ignore
@@ -46,6 +63,11 @@ class SetterConfig(BaseModel):
         description='Configuration for warnings.',
     )
 
+    repeats: RepeatsConfig = Field(
+        default_factory=RepeatsConfig,  # type: ignore
+        description='Configuration for repeats.',
+    )
+
     command_substitutions: Dict[str, str] = Field(
         {},
         description='Substitutions to apply to commands before running them.',

rbx/box/solutions.py

@@ -16,7 +16,7 @@ import rich.text
 import typer
 from pydantic import BaseModel
 
-from rbx import console
+from rbx import console, utils
 from rbx.box import checkers, package
 from rbx.box.code import SanitizationLevel, compile_item, find_language_name, run_item
 from rbx.box.deferred import Deferred
@@ -25,7 +25,14 @@ from rbx.box.environment import (
     ExecutionConfig,
     VerificationLevel,
 )
-from rbx.box.generators import generate_output_for_testcase, generate_standalone
+from rbx.box.generators import (
+    GenerationMetadata,
+    expand_generator_call,
+    extract_generation_testcases,
+    generate_output_for_testcase,
+    generate_standalone,
+)
+from rbx.box.retries import Retrier
 from rbx.box.schema import (
     ExpectedOutcome,
     GeneratorCall,
@@ -34,7 +41,7 @@ from rbx.box.schema import (
     Testcase,
     TestcaseGroup,
 )
-from rbx.box.testcases import find_built_testcases
+from rbx.box.testcases import TestcaseEntry, find_built_testcases
 from rbx.grading.steps import (
     DigestOrDest,
     DigestOrSource,
@@ -169,63 +176,70 @@ def _run_solution_on_testcase(
     verification: VerificationLevel = VerificationLevel.NONE,
     timelimit_override: Optional[int] = None,
 ) -> Evaluation:
-    actual_sandbox = package.get_singleton_sandbox()
+    def run_fn(retry_index: int) -> Evaluation:
+        actual_sandbox = package.get_singleton_sandbox()
 
-    limits = get_limits_for_language(
-        solution.language, verification, timelimit_override
-    )
+        limits = get_limits_for_language(
+            solution.language, verification, timelimit_override
+        )
 
-    sandbox = EnvironmentSandbox()
-    sandbox.timeLimit = limits.time
-    if limits.isDoubleTL and sandbox.timeLimit is not None:
-        # Double TL.
-        sandbox.timeLimit = sandbox.timeLimit * 2
-    sandbox.wallTimeLimit = sandbox.timeLimit
-    if sandbox.timeLimit is not None and actual_sandbox.use_soft_timeout():
-        sandbox.wallTimeLimit = sandbox.timeLimit * 2
-    sandbox.memoryLimit = limits.memory
-    sandbox.fileSizeLimit = limits.output
-    extra_config = ExecutionConfig(sandbox=sandbox)
-
-    output_path = output_dir / testcase.inputPath.with_suffix('.out').name
-    error_path = output_path.with_suffix('.err')
-    log_path = output_path.with_suffix('.log')
-    output_path.parent.mkdir(parents=True, exist_ok=True)
-
-    run_log = run_item(
-        solution,
-        DigestOrSource.create(compiled_digest),
-        stdin=DigestOrSource.create(testcase.inputPath),
-        stdout=DigestOrDest.create(output_path),
-        stderr=DigestOrDest.create(error_path),
-        extra_config=extra_config,
-    )
+        sandbox = EnvironmentSandbox()
+        sandbox.timeLimit = limits.time
+        if limits.isDoubleTL and sandbox.timeLimit is not None:
+            # Double TL.
+            sandbox.timeLimit = sandbox.timeLimit * 2
+        sandbox.wallTimeLimit = sandbox.timeLimit
+        if sandbox.timeLimit is not None and actual_sandbox.use_soft_timeout():
+            sandbox.wallTimeLimit = sandbox.timeLimit * 2
+        sandbox.memoryLimit = limits.memory
+        sandbox.fileSizeLimit = limits.output
+        extra_config = ExecutionConfig(sandbox=sandbox)
+
+        output_path = output_dir / testcase.inputPath.with_suffix('.out').name
+        error_path = output_path.with_suffix('.err')
+        log_path = output_path.with_suffix('.log')
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+
+        run_log = run_item(
+            solution,
+            DigestOrSource.create(compiled_digest),
+            stdin=DigestOrSource.create(testcase.inputPath),
+            stdout=DigestOrDest.create(output_path),
+            stderr=DigestOrDest.create(error_path),
+            extra_config=extra_config,
+            retry_index=retry_index,
+        )
 
-    if checker_digest is not None:
-        checker_result = checkers.check(
-            checker_digest,
-            run_log,
-            testcase,
-            program_output=output_path,
+        if checker_digest is not None:
+            checker_result = checkers.check(
+                checker_digest,
+                run_log,
+                testcase,
+                program_output=output_path,
+            )
+        else:
+            checker_result = checkers.check_with_no_output(run_log)
+
+        eval = Evaluation(
+            result=checker_result,
+            testcase=TestcaseIO(
+                index=testcase_index,
+                input=testcase.inputPath,
+                output=testcase.outputPath,
+            ),
+            log=TestcaseLog(
+                **(run_log.model_dump() if run_log is not None else {}),
+                stdout_absolute_path=output_path.absolute(),
+                stderr_absolute_path=error_path.absolute(),
+                log_absolute_path=log_path.absolute(),
+            ),
         )
-    else:
-        checker_result = checkers.check_with_no_output(run_log)
 
-    eval = Evaluation(
-        result=checker_result,
-        testcase=TestcaseIO(
-            index=testcase_index, input=testcase.inputPath, output=testcase.outputPath
-        ),
-        log=TestcaseLog(
-            **(run_log.model_dump() if run_log is not None else {}),
-            stdout_absolute_path=output_path.absolute(),
-            stderr_absolute_path=error_path.absolute(),
-            log_absolute_path=log_path.absolute(),
-        ),
-    )
+        log_path.write_text(model_to_yaml(eval))
+        return eval
 
-    log_path.write_text(model_to_yaml(eval))
-    return eval
+    retrier = Retrier()
+    return retrier.repeat(run_fn)
 
 
 def _run_solution(
@@ -411,26 +425,97 @@ def run_solutions(
     )
 
 
-def _run_interactive_solutions(
+async def _generate_testcase_interactively(
     progress: Optional[StatusProgress] = None,
-    tracked_solutions: Optional[Set[str]] = None,
-    verification: VerificationLevel = VerificationLevel.NONE,
     generator: Optional[GeneratorCall] = None,
+    testcase_entry: Optional[TestcaseEntry] = None,
     check: bool = True,
-    print: bool = False,
     sanitized: bool = False,
-) -> Iterator[EvaluationItem]:
-    pkg = package.find_problem_package_or_die()
+    print: bool = False,
+) -> Testcase:
     main_solution = package.get_main_solution()
-    check = check and main_solution is not None
-
-    checker_digest = checkers.compile_checker() if check else None
-    compiled_solutions = compile_solutions(
-        progress=progress, tracked_solutions=tracked_solutions, sanitized=sanitized
+    irun_dir = package.get_problem_iruns_dir()
+    inputs_dir = irun_dir / 'inputs'
+    inputs_dir.mkdir(parents=True, exist_ok=True)
+    testcase = Testcase(
+        inputPath=inputs_dir / '000.in',
+        outputPath=(inputs_dir / '000.out') if check else None,
     )
 
+    is_manual = False
+    generation_metadata = None
+    if generator is not None:
+        generation_metadata = GenerationMetadata(
+            generator_call=expand_generator_call(generator),
+            copied_to=testcase,
+        )
+    elif testcase_entry is not None:
+        extracted = extract_generation_testcases([testcase_entry])
+        if not extracted:
+            console.console.print(
+                f'[error]Failed searching for testcase [item]{testcase_entry}[/item].[/error]'
+            )
+            raise typer.Exit(1)
+        generation_metadata = extracted[0].metadata
+        # Replace destination with the irun testcase we're using.
+        generation_metadata.copied_to = testcase
+    else:
+        with utils.no_progress(progress):
+            input = console.multiline_prompt('Testcase input')
+        testcase.inputPath.write_text(input)
+        console.console.print()
+
+        if (
+            testcase.outputPath is not None
+            and not testcase.outputPath.is_file()
+            and main_solution is None
+        ):
+            with utils.no_progress(progress):
+                output = console.multiline_prompt('Testcase output')
+                testcase.outputPath.write_text(output)
+                console.console.print()
+
+        generation_metadata = GenerationMetadata(
+            copied_to=testcase,
+        )
+        is_manual = True
+
+    # 1. Generate testcase.
+    if generation_metadata is not None:
+        generate_standalone(
+            generation_metadata,
+            progress=progress,
+            validate=True,
+        )
+        if testcase_entry is not None:
+            console.console.print(
+                f'Using input from testcase [item]{testcase_entry}[/item].'
+            )
+        elif generation_metadata.generator_call is not None:
+            console.console.print(
+                f'Using input from generator call [item]{generation_metadata.generator_call.name} {generation_metadata.generator_call.args}[/item].'
+            )
+        if print and not is_manual:
+            console.console.print(testcase.inputPath.read_text())
+        else:
+            console.console.print(
+                f'Input was written to [item]{testcase.inputPath.resolve()}[/item]'
+            )
+        console.console.print()
+
+    # 2. Generate test output from reference
     main_solution_digest = None
-    if check and main_solution is not None:
+    if check and not (
+        testcase.outputPath is not None and testcase.outputPath.is_file()
+    ):
+        if main_solution is None:
+            console.console.print(
+                '[error]Checking is enabled but no main solution or custom output was specified.[/error]'
+            )
+            raise typer.Exit(1)
+
+        if progress:
+            progress.update('Compiling main solution...')
         try:
             main_solution_digest = compile_item(
                 main_solution,
@@ -444,6 +529,42 @@ def _run_interactive_solutions(
             )
             raise
 
+    if main_solution_digest is not None:
+        if progress:
+            progress.update('Generating output for test...')
+        # TODO: Add stderr path
+        generate_output_for_testcase(main_solution_digest, testcase)
+
+    if check and testcase.outputPath is not None and not testcase.outputPath.is_file():
+        # Output was not created, throw an error.
+        console.console.print(
+            '[error]Checking is enabled but no output could be generated for this testcase.[/error]'
+        )
+        console.console.print(
+            '[error]Either specify it explicitly or provide a main solution.[/error]'
+        )
+        raise typer.Exit(1)
+
+    return testcase
+
+
+def _run_interactive_solutions(
+    testcase: Testcase,
+    progress: Optional[StatusProgress] = None,
+    tracked_solutions: Optional[Set[str]] = None,
+    verification: VerificationLevel = VerificationLevel.NONE,
+    check: bool = True,
+    sanitized: bool = False,
+) -> Iterator[EvaluationItem]:
+    pkg = package.find_problem_package_or_die()
+
+    if check and progress:
+        progress.update('Compiling checker...')
+    checker_digest = checkers.compile_checker() if check else None
+    compiled_solutions = compile_solutions(
+        progress=progress, tracked_solutions=tracked_solutions, sanitized=sanitized
+    )
+
     solutions = list(enumerate(pkg.solutions))
     if tracked_solutions is not None:
         solutions = [
@@ -451,33 +572,9 @@ def _run_interactive_solutions(
         ]
 
     irun_dir = package.get_problem_iruns_dir()
-    shutil.rmtree(str(irun_dir), ignore_errors=True)
-    irun_dir.mkdir(parents=True, exist_ok=True)
-    inputs_dir = irun_dir / 'inputs'
-    inputs_dir.mkdir(parents=True, exist_ok=True)
-    input_path = inputs_dir / '000.in'
-    output_path = input_path.with_suffix('.out')
 
-    if generator is not None:
-        expanded_call = generate_standalone(generator, input_path)
-        console.console.print(
-            f'Using input from generator call [item]{expanded_call.name} {expanded_call.args}[/item].'
-        )
-        if print:
-            console.console.print(input_path.read_text())
-        else:
-            console.console.print(
-                f'Input was written to [item]{input_path.resolve()}[/item]'
-            )
-        console.console.print()
-    else:
-        input = console.multiline_prompt('Testcase input')
-        input_path.write_text(input)
-    testcase = Testcase(inputPath=input_path, outputPath=output_path if check else None)
-
-    if main_solution_digest is not None:
-        # TODO: Add stderr path
-        generate_output_for_testcase(main_solution_digest, testcase)
+    if progress:
+        progress.update('Running solutions...')
 
     for i, solution in solutions:
         output_dir = irun_dir / f'{i}'
@@ -505,27 +602,38 @@ async def run_and_print_interactive_solutions(
     tracked_solutions: Optional[Set[str]] = None,
     verification: VerificationLevel = VerificationLevel.NONE,
     generator: Optional[GeneratorCall] = None,
+    testcase_entry: Optional[TestcaseEntry] = None,
     check: bool = True,
     print: bool = False,
     sanitized: bool = False,
 ):
+    # Ensure path is new.
+    irun_dir = package.get_problem_iruns_dir()
+    shutil.rmtree(str(irun_dir), ignore_errors=True)
+    irun_dir.mkdir(parents=True, exist_ok=True)
+
     pkg = package.find_problem_package_or_die()
+    testcase = await _generate_testcase_interactively(
+        progress=progress,
+        generator=generator,
+        testcase_entry=testcase_entry,
+        check=check,
+        sanitized=sanitized,
+        print=print,
+    )
     items = _run_interactive_solutions(
+        testcase,
         progress=progress,
         tracked_solutions=tracked_solutions,
         verification=verification,
         check=check,
-        generator=generator,
         sanitized=sanitized,
-        print=print,
     )
 
-    if progress:
-        progress.stop()
-
     for item in items:
         sol = pkg.solutions[item.solution_index]
-        _print_solution_header(sol, console.console, is_irun=True)
+        with utils.no_progress(progress):
+            _print_solution_header(sol, console.console, is_irun=True)
 
         eval = await item.eval()
 
@@ -549,7 +657,7 @@ async def run_and_print_interactive_solutions(
 
 
 def _get_solution_repr(sol: Solution) -> List[Tuple[str, str]]:
-    fg_color = sol.outcome.style().replace('lnumber', 'cyan')
+    fg_color = sol.outcome.style()
     return [
         ('', f'{str(sol.path)} '),
         (f'fg:{fg_color}', sol.outcome.name),
@@ -582,9 +690,9 @@ def get_outcome_style_verdict(outcome: Outcome) -> str:
     if outcome == Outcome.TIME_LIMIT_EXCEEDED:
         return 'yellow'
     if outcome == Outcome.RUNTIME_ERROR:
-        return 'lnumber'
+        return 'blue'
     if outcome == Outcome.MEMORY_LIMIT_EXCEEDED:
-        return 'cyan'
+        return 'yellow'
     return 'magenta'