rbx.cp 0.5.39__py3-none-any.whl → 0.5.42__py3-none-any.whl

rbx/box/solutions.py

@@ -17,11 +17,13 @@ from pydantic import BaseModel
 
 from rbx import console, utils
 from rbx.box import checkers, package
-from rbx.box.code import SanitizationLevel, compile_item, find_language_name, run_item
+from rbx.box.code import (
+    SanitizationLevel,
+    compile_item,
+    find_language_name,
+)
 from rbx.box.deferred import Deferred
 from rbx.box.environment import (
-    EnvironmentSandbox,
-    ExecutionConfig,
     VerificationLevel,
 )
 from rbx.box.formatting import get_formatted_memory, get_formatted_time
@@ -31,26 +33,26 @@ from rbx.box.generators import (
     generate_output_for_testcase,
     generate_standalone,
 )
-from rbx.box.retries import Retrier
 from rbx.box.schema import (
     ExpectedOutcome,
     GeneratorCall,
     Limits,
     Solution,
+    TaskType,
     Testcase,
     TestcaseGroup,
 )
+from rbx.box.tasks import (
+    get_limits_for_language,
+    run_solution_on_testcase,
+)
 from rbx.box.testcase_extractors import extract_generation_testcases
 from rbx.box.testcase_utils import TestcaseEntry, find_built_testcases
 from rbx.grading.steps import (
-    DigestOrDest,
-    DigestOrSource,
     Evaluation,
     Outcome,
-    TestcaseIO,
-    TestcaseLog,
 )
-from rbx.utils import StatusProgress, model_to_yaml
+from rbx.utils import StatusProgress
 
 StructuredEvaluation = Dict[str, Dict[str, List[Optional[Deferred[Evaluation]]]]]
 
@@ -152,102 +154,13 @@ def compile_solutions(
     return compiled_solutions
 
 
-def get_limits_for_language(
-    lang: Optional[str],
-    verification: VerificationLevel,
-    timelimit_override: Optional[int],
-) -> Limits:
-    pkg = package.find_problem_package_or_die()
-    time = timelimit_override or pkg.timelimit_for_language(lang)
-    isDoubleTL = verification.value >= VerificationLevel.FULL.value
-    memory = pkg.memorylimit_for_language(lang)
-    return Limits(
-        time=time, memory=memory, output=pkg.outputLimit, isDoubleTL=isDoubleTL
-    )
-
-
-def _run_solution_on_testcase(
-    solution: Solution,
-    compiled_digest: str,
-    checker_digest: Optional[str],
-    testcase: Testcase,
-    output_dir: pathlib.Path,
-    testcase_index: int = 0,
-    verification: VerificationLevel = VerificationLevel.NONE,
-    timelimit_override: Optional[int] = None,
-) -> Evaluation:
-    def run_fn(retry_index: int) -> Evaluation:
-        actual_sandbox = package.get_singleton_sandbox()
-
-        limits = get_limits_for_language(
-            solution.language, verification, timelimit_override
-        )
-
-        sandbox = EnvironmentSandbox()
-        sandbox.timeLimit = limits.time
-        if limits.isDoubleTL and sandbox.timeLimit is not None:
-            # Double TL.
-            sandbox.timeLimit = sandbox.timeLimit * 2
-        sandbox.wallTimeLimit = sandbox.timeLimit
-        if sandbox.timeLimit is not None and actual_sandbox.use_soft_timeout():
-            sandbox.wallTimeLimit = sandbox.timeLimit * 2
-        sandbox.memoryLimit = limits.memory
-        sandbox.fileSizeLimit = limits.output
-        extra_config = ExecutionConfig(sandbox=sandbox)
-
-        output_path = output_dir / testcase.inputPath.with_suffix('.out').name
-        error_path = output_path.with_suffix('.err')
-        log_path = output_path.with_suffix('.log')
-        output_path.parent.mkdir(parents=True, exist_ok=True)
-
-        run_log = run_item(
-            solution,
-            DigestOrSource.create(compiled_digest),
-            stdin=DigestOrSource.create(testcase.inputPath),
-            stdout=DigestOrDest.create(output_path),
-            stderr=DigestOrDest.create(error_path),
-            extra_config=extra_config,
-            retry_index=retry_index,
-        )
-
-        if checker_digest is not None:
-            checker_result = checkers.check(
-                checker_digest,
-                run_log,
-                testcase,
-                program_output=output_path,
-            )
-        else:
-            checker_result = checkers.check_with_no_output(run_log)
-
-        eval = Evaluation(
-            result=checker_result,
-            testcase=TestcaseIO(
-                index=testcase_index,
-                input=testcase.inputPath,
-                output=testcase.outputPath,
-            ),
-            log=TestcaseLog(
-                **(run_log.model_dump() if run_log is not None else {}),
-                stdout_absolute_path=output_path.absolute(),
-                stderr_absolute_path=error_path.absolute(),
-                log_absolute_path=log_path.absolute(),
-            ),
-        )
-
-        log_path.write_text(model_to_yaml(eval))
-        return eval
-
-    retrier = Retrier()
-    return retrier.repeat(run_fn)
-
-
 def _run_solution(
     solution: Solution,
     compiled_digest: str,
     checker_digest: Optional[str],
     solution_index: int,
     group_name: str,
+    interactor_digest: Optional[str] = None,
     progress: Optional[StatusProgress] = None,
     verification: VerificationLevel = VerificationLevel.NONE,
     timelimit_override: Optional[int] = None,
@@ -267,12 +180,13 @@ def _run_solution(
             )
 
         async def run_fn(i=i, testcase=testcase, output_path=output_path):
-            return _run_solution_on_testcase(
+            return await run_solution_on_testcase(
                 solution,
                 compiled_digest,
                 checker_digest,
                 testcase,
                 output_path,
+                interactor_digest=interactor_digest,
                 testcase_index=i,
                 verification=verification,
                 timelimit_override=timelimit_override,
@@ -343,7 +257,15 @@ def _produce_solution_items(
 ) -> List[EvaluationItem]:
     pkg = package.find_problem_package_or_die()
 
-    checker_digest = checkers.compile_checker() if check else None
+    if pkg.type == TaskType.COMMUNICATION:
+        checker_digest = (
+            checkers.compile_checker() if check and pkg.checker is not None else None
+        )
+        interactor_digest = checkers.compile_interactor()
+    else:
+        checker_digest = checkers.compile_checker() if check else None
+        interactor_digest = None
+
     compiled_solutions = compile_solutions(
         progress=progress, tracked_solutions=tracked_solutions, sanitized=sanitized
     )
@@ -374,6 +296,7 @@ def _produce_solution_items(
                 checker_digest,
                 solution_index,
                 group_name,
+                interactor_digest=interactor_digest,
                 progress=progress,
                 verification=verification,
                 timelimit_override=timelimit_override,
@@ -451,7 +374,7 @@ async def _generate_testcase_interactively(
             copied_to=testcase,
         )
     elif testcase_entry is not None:
-        extracted = extract_generation_testcases([testcase_entry])
+        extracted = await extract_generation_testcases([testcase_entry])
         if not extracted:
             console.console.print(
                 f'[error]Failed searching for testcase [item]{testcase_entry}[/item].[/error]'
@@ -483,7 +406,7 @@ async def _generate_testcase_interactively(
 
     # 1. Generate testcase.
     if generation_metadata is not None:
-        generate_standalone(
+        await generate_standalone(
             generation_metadata,
             progress=progress,
             validate=True,
@@ -531,10 +454,20 @@ async def _generate_testcase_interactively(
             raise
 
     if main_solution_digest is not None:
+        pkg = package.find_problem_package_or_die()
+        if pkg.type == TaskType.COMMUNICATION:
+            interactor_digest = checkers.compile_interactor(progress)
+        else:
+            interactor_digest = None
+
         if progress:
             progress.update('Generating output for test...')
         # TODO: Add stderr path
-        generate_output_for_testcase(main_solution_digest, testcase)
+        await generate_output_for_testcase(
+            main_solution_digest,
+            testcase,
+            interactor_digest=interactor_digest,
+        )
 
     if check and testcase.outputPath is not None and not testcase.outputPath.is_file():
         # Output was not created, throw an error.
@@ -559,9 +492,13 @@ def _run_interactive_solutions(
 ) -> Iterator[EvaluationItem]:
     pkg = package.find_problem_package_or_die()
 
-    if check and progress:
-        progress.update('Compiling checker...')
-    checker_digest = checkers.compile_checker() if check else None
+    if pkg.type == TaskType.COMMUNICATION:
+        checker_digest = checkers.compile_checker() if check else None
+        interactor_digest = checkers.compile_interactor()
+    else:
+        checker_digest = checkers.compile_checker() if check else None
+        interactor_digest = None
+
     compiled_solutions = compile_solutions(
         progress=progress, tracked_solutions=tracked_solutions, sanitized=sanitized
     )
@@ -581,12 +518,13 @@ def _run_interactive_solutions(
         output_dir = irun_dir / f'{i}'
 
         async def run_fn(solution=solution, output_dir=output_dir):
-            return _run_solution_on_testcase(
+            return await run_solution_on_testcase(
                 solution,
                 compiled_solutions[solution.path],
                 checker_digest,
                 testcase,
                 output_dir,
+                interactor_digest=interactor_digest,
                 verification=verification,
             )
 

rbx/box/solutions_test.py

@@ -1,4 +1,3 @@
-import asyncio
 import pathlib
 
 import pytest
@@ -17,15 +16,15 @@ from rbx.grading.steps import Outcome
 
 
 @pytest.mark.test_pkg('box1')
-def test_solutions(pkg_from_testdata: pathlib.Path):
-    generate_testcases()
+async def test_solutions(pkg_from_testdata: pathlib.Path):
+    await generate_testcases()
     entries = [
-        entry.group_entry for entry in extract_generation_testcases_from_groups()
+        entry.group_entry for entry in await extract_generation_testcases_from_groups()
     ]
-    generate_outputs_for_testcases(entries)
+    await generate_outputs_for_testcases(entries)
 
     result = run_solutions(verification=VerificationLevel.FULL)
-    res = asyncio.run(convert_list_of_solution_evaluations_to_dict(result.items))
+    res = await convert_list_of_solution_evaluations_to_dict(result.items)
 
     # First solution should pass all tests.
     assert all(chk.result.outcome == Outcome.ACCEPTED for chk in res[0]['gen1'])

rbx/box/statements/build_statements.py

@@ -3,6 +3,7 @@ import tempfile
 import typing
 from typing import Annotated, Dict, List, Optional, Tuple
 
+import syncer
 import typer
 
 from rbx import annotations, console
@@ -307,7 +308,8 @@ def build_statement(
 
 @app.command('build, b', help='Build statements.')
 @package.within_problem
-def build(
+@syncer.sync
+async def build(
     verification: environment.VerificationParam,
     languages: Annotated[
         Optional[List[str]],
@@ -335,7 +337,7 @@ def build(
     if samples:
         from rbx.box import builder
 
-        if not builder.build(
+        if not await builder.build(
             verification=verification,
             groups=set(['samples']),
             output=None,

rbx/box/stresses.py

@@ -3,6 +3,7 @@ import time
 from shutil import rmtree
 from typing import List, Optional
 
+import syncer
 import typer
 from pydantic import BaseModel
 
@@ -15,7 +16,7 @@ from rbx.box.generators import (
     generate_standalone,
 )
 from rbx.box.retries import Retrier
-from rbx.box.schema import CodeItem, GeneratorCall, Stress, Testcase
+from rbx.box.schema import CodeItem, GeneratorCall, Stress, TaskType, Testcase
 from rbx.box.solutions import compile_solutions, get_outcome_style_verdict
 from rbx.box.stressing import finder_parser
 from rbx.grading.steps import (
@@ -49,7 +50,8 @@ def _compile_finder(finder: CodeItem) -> str:
     return digest
 
 
-def run_stress(
+@syncer.sync
+async def run_stress(
     name: str,
     timeoutInSeconds: int,
     finder: Optional[str] = None,
@@ -59,6 +61,13 @@ def run_stress(
     progress: Optional[StatusProgress] = None,
     sanitized: bool = False,
 ) -> StressReport:
+    pkg = package.find_problem_package_or_die()
+    if pkg.type == TaskType.COMMUNICATION:
+        console.console.print(
+            '[error]Communication problems do not support stress testing.[/error]'
+        )
+        raise typer.Exit(1)
+
     if finder:
         stress = Stress(
             name=f'{name}',
@@ -128,7 +137,7 @@ def run_stress(
         input_path.parent.mkdir(parents=True, exist_ok=True)
 
         expanded_generator_call = expand_generator_call(stress.generator)
-        generate_standalone(
+        await generate_standalone(
             GenerationMetadata(
                 generator_call=expanded_generator_call,
                 copied_to=Testcase(inputPath=input_path),
@@ -140,7 +149,7 @@ def run_stress(
         )
 
         @functools.cache
-        def run_solution_fn(
+        async def run_solution_fn(
             solution: str,
             retry_index: Optional[int] = None,
             input_path=input_path,
@@ -150,7 +159,7 @@ def run_stress(
             output_path = input_path.with_stem(f'{index}').with_suffix('.out')
             stderr_path = output_path.with_suffix('.err')
 
-            run_log = run_item(
+            run_log = await run_item(
                 sol,
                 DigestOrSource.create(solutions_digest[sol.path]),
                 stdin=DigestOrSource.create(input_path),
@@ -168,7 +177,7 @@ def run_stress(
         # Get main solution output.
         expected_output_path = empty_path
         if needs_expected_output:
-            main_testcase_log = run_solution_fn(str(solutions[0].path))
+            main_testcase_log = await run_solution_fn(str(solutions[0].path))
             main_checker_result = checkers.check_with_no_output(main_testcase_log)
             if main_checker_result.outcome != Outcome.ACCEPTED:
                 console.console.print(
@@ -190,23 +199,23 @@ def run_stress(
             expected_output_path = main_testcase_log.stdout_absolute_path
 
         @functools.cache
-        def run_solution_and_checker_fn(
+        async def run_solution_and_checker_fn(
             call: finder_parser.FinderCall,
             input_path=input_path,
             expected_output_path=expected_output_path,
         ) -> finder_parser.FinderResult:
-            def run_fn(retry_index: int) -> Evaluation:
+            async def run_fn(retry_index: int) -> Evaluation:
                 solution = call.solution
                 checker = call.checker
 
-                testcase_log = run_solution_fn(solution, retry_index=retry_index)
+                testcase_log = await run_solution_fn(solution, retry_index=retry_index)
                 assert testcase_log.stdout_absolute_path is not None
 
                 if checker is None:
                     checker_result = checkers.check_with_no_output(testcase_log)
                 else:
                     checker_digest = finders_digest[checker.path]
-                    checker_result = checkers.check(
+                    checker_result = await checkers.check(
                         checker_digest,
                         testcase_log,
                         Testcase(inputPath=input_path, outputPath=expected_output_path),
@@ -224,7 +233,7 @@ def run_stress(
                 )
 
             retrier = Retrier(is_stress=True)
-            eval = retrier.repeat(run_fn)
+            eval = await retrier.repeat(run_fn)
 
             return finder_parser.FinderResult(
                 solution=call.solution,
@@ -234,7 +243,9 @@ def run_stress(
                 checker_result=eval.result,
             )
 
-        runner = finder_parser.FinderTreeRunner(runner=run_solution_and_checker_fn)
+        runner = finder_parser.FinderTreeRunner(
+            runner=syncer.sync(run_solution_and_checker_fn)
+        )
         finder_outcome: finder_parser.FinderOutcome = runner.transform(parsed_finder)
 
         internal_error_results = [

rbx/box/tasks.py

@@ -0,0 +1,258 @@
+import pathlib
+from typing import Optional
+
+from rbx.box import checkers, package
+from rbx.box.code import CommunicationItem, run_communication, run_item
+from rbx.box.environment import EnvironmentSandbox, ExecutionConfig, VerificationLevel
+from rbx.box.retries import Retrier
+from rbx.box.schema import Limits, Solution, Testcase
+from rbx.grading.judge.sandbox import SandboxBase
+from rbx.grading.steps import (
+    DigestOrDest,
+    DigestOrSource,
+    Evaluation,
+    GradingFileInput,
+    GradingFileOutput,
+    TestcaseIO,
+    TestcaseLog,
+)
+from rbx.utils import model_to_yaml
+
+
+def get_limits_for_language(
+    lang: Optional[str],
+    verification: VerificationLevel,
+    timelimit_override: Optional[int],
+    use_timelimit: bool = True,
+) -> Limits:
+    pkg = package.find_problem_package_or_die()
+    time = timelimit_override or pkg.timelimit_for_language(lang)
+    isDoubleTL = verification.value >= VerificationLevel.FULL.value
+    memory = pkg.memorylimit_for_language(lang)
+    return Limits(
+        time=time if use_timelimit else None,
+        memory=memory,
+        output=pkg.outputLimit,
+        isDoubleTL=isDoubleTL,
+    )
+
+
+async def run_solution_on_testcase(
+    solution: Solution,
+    compiled_digest: str,
+    checker_digest: Optional[str],
+    testcase: Testcase,
+    output_dir: pathlib.Path,
+    interactor_digest: Optional[str] = None,
+    testcase_index: int = 0,
+    verification: VerificationLevel = VerificationLevel.NONE,
+    timelimit_override: Optional[int] = None,
+    use_retries: bool = True,
+    use_timelimit: bool = True,
+) -> Evaluation:
+    if interactor_digest is not None:
+        return await _run_communication_solution_on_testcase(
+            solution,
+            compiled_digest,
+            interactor_digest,
+            checker_digest,
+            testcase,
+            output_dir,
+            testcase_index=testcase_index,
+            verification=verification,
+            timelimit_override=timelimit_override,
+            use_retries=use_retries,
+            use_timelimit=use_timelimit,
+        )
+
+    async def run_fn(retry_index: int) -> Evaluation:
+        actual_sandbox = package.get_singleton_sandbox()
+
+        limits = get_limits_for_language(
+            solution.language,
+            verification,
+            timelimit_override,
+            use_timelimit=use_timelimit,
+        )
+        extra_config = _get_execution_config(limits, actual_sandbox)
+
+        output_path = output_dir / testcase.inputPath.with_suffix('.out').name
+        error_path = output_path.with_suffix('.err')
+        log_path = output_path.with_suffix('.log')
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+
+        run_log = await run_item(
+            solution,
+            DigestOrSource.create(compiled_digest),
+            stdin=DigestOrSource.create(testcase.inputPath),
+            stdout=DigestOrDest.create(output_path),
+            stderr=DigestOrDest.create(error_path),
+            extra_config=extra_config,
+            retry_index=retry_index,
+        )
+
+        if checker_digest is not None:
+            checker_result = await checkers.check(
+                checker_digest,
+                run_log,
+                testcase,
+                program_output=output_path,
+            )
+        else:
+            checker_result = checkers.check_with_no_output(run_log)
+
+        eval = Evaluation(
+            result=checker_result,
+            testcase=TestcaseIO(
+                index=testcase_index,
+                input=testcase.inputPath,
+                output=testcase.outputPath,
+            ),
+            log=TestcaseLog(
+                **(run_log.model_dump() if run_log is not None else {}),
+                stdout_absolute_path=output_path.absolute(),
+                stderr_absolute_path=error_path.absolute(),
+                log_absolute_path=log_path.absolute(),
+            ),
+        )
+
+        log_path.write_text(model_to_yaml(eval))
+        return eval
+
+    if not use_retries:
+        return await run_fn(0)
+
+    retrier = Retrier()
+    return await retrier.repeat(run_fn)
+
+
+def _get_execution_config(
+    limits: Limits,
+    actual_sandbox: SandboxBase,
+) -> ExecutionConfig:
+    sandbox = EnvironmentSandbox()
+    sandbox.timeLimit = limits.time
+    if limits.isDoubleTL and sandbox.timeLimit is not None:
+        # Double TL.
+        sandbox.timeLimit = sandbox.timeLimit * 2
+    sandbox.wallTimeLimit = sandbox.timeLimit
+    if sandbox.timeLimit is not None and actual_sandbox.use_soft_timeout():
+        sandbox.wallTimeLimit = sandbox.timeLimit * 2
+    sandbox.memoryLimit = limits.memory
+    sandbox.fileSizeLimit = limits.output
+    return ExecutionConfig(sandbox=sandbox)
+
+
+async def _run_communication_solution_on_testcase(
+    solution: Solution,
+    compiled_digest: str,
+    interactor_digest: str,
+    checker_digest: Optional[str],
+    testcase: Testcase,
+    output_dir: pathlib.Path,
+    testcase_index: int = 0,
+    verification: VerificationLevel = VerificationLevel.NONE,
+    timelimit_override: Optional[int] = None,
+    use_retries: bool = True,
+    use_timelimit: bool = True,
+) -> Evaluation:
+    async def run_fn(retry_index: int) -> Evaluation:
+        actual_sandbox = package.get_singleton_sandbox()
+        interactor_sandbox = package.get_singleton_interactor_sandbox()
+
+        limits = get_limits_for_language(
+            solution.language,
+            verification,
+            timelimit_override,
+            use_timelimit=use_timelimit,
+        )
+
+        extra_config = _get_execution_config(limits, actual_sandbox)
+        interactor_extra_config = _get_execution_config(limits, interactor_sandbox)
+        if (
+            interactor_extra_config.sandbox is not None
+            and interactor_extra_config.sandbox.wallTimeLimit is not None
+            and extra_config.sandbox is not None
+            and extra_config.sandbox.wallTimeLimit is not None
+        ):
+            interactor_extra_config.sandbox.wallTimeLimit += (
+                extra_config.sandbox.wallTimeLimit
+            )
+        # TODO: maybe combine wall time limits?
+
+        output_path = output_dir / testcase.inputPath.with_suffix('.out').name
+        error_path = output_path.with_suffix('.err')
+        log_path = output_path.with_suffix('.log')
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+
+        interactor_item = CommunicationItem(
+            code=package.get_interactor(),
+            executable=DigestOrSource.create(interactor_digest),
+            stderr=DigestOrDest.create(error_path),
+            extra_config=interactor_extra_config,
+            extra_args='interactor.in interactor.out',
+            inputs=[
+                GradingFileInput(
+                    src=testcase.inputPath,
+                    dest=pathlib.PosixPath('interactor.in'),
+                )
+            ],
+            outputs=[
+                GradingFileOutput(
+                    src=pathlib.PosixPath('interactor.out'),
+                    dest=output_path,
+                    touch=True,
+                )
+            ],
+        )
+        solution_item = CommunicationItem(
+            code=solution,
+            executable=DigestOrSource.create(compiled_digest),
+            extra_config=extra_config,
+        )
+
+        interactor_run_log, run_log = await run_communication(
+            interactor=interactor_item,
+            solution=solution_item,
+            retry_index=retry_index,
+        )
+
+        checker_result = await checkers.check_communication(
+            checker_digest,
+            run_log,
+            interactor_run_log,
+            error_path,
+            testcase,
+            output_path,
+        )
+
+        eval = Evaluation(
+            result=checker_result,
+            testcase=TestcaseIO(
+                index=testcase_index,
+                input=testcase.inputPath,
+                output=testcase.outputPath,
+            ),
+            log=TestcaseLog(
+                **(run_log.model_dump() if run_log is not None else {}),
+                stdout_absolute_path=output_path.absolute(),
+                stderr_absolute_path=error_path.absolute(),
+                log_absolute_path=log_path.absolute(),
+            ),
+        )
+
+        log_path.write_text(model_to_yaml(eval))
+
+        if interactor_run_log is not None:
+            interactor_log_path = output_path.with_suffix('.int.log')
+            interactor_log_path.write_text(model_to_yaml(interactor_run_log))
+        if run_log is not None:
+            solution_log_path = output_path.with_suffix('.sol.log')
+            solution_log_path.write_text(model_to_yaml(run_log))
+        return eval
+
+    if not use_retries:
+        return await run_fn(0)
+
+    retrier = Retrier()
+    return await retrier.repeat(run_fn)