rbx.cp 0.5.42__py3-none-any.whl → 0.5.46__py3-none-any.whl

rbx/box/cli.py

@@ -103,6 +103,13 @@ def main(
         help='Whether to compile and run testlib components with sanitizers enabled. '
         'If you want to run the solutions with sanitizers enabled, use the "-s" flag in the corresponding run command.',
     ),
+    debug_logs: bool = typer.Option(
+        False,
+        '--debug-logs',
+        '--debug',
+        '-d',
+        help='Whether to save extra debug logs along with the evaluation results.',
+    ),
 ):
     state.STATE.run_through_cli = True
     state.STATE.sanitized = sanitized
@@ -111,6 +118,7 @@ def main(
             '[warning]Sanitizers are running just for testlib components.\n'
             'If you want to run the solutions with sanitizers enabled, use the [item]-s[/item] flag in the corresponding run command.[/warning]'
         )
+    state.STATE.debug_logs = debug_logs
 
 
 # @app.command('ui', hidden=True)

rbx/box/code.py

@@ -216,6 +216,52 @@ class PreparedRun:
     metadata: RunLogMetadata
 
 
+@dataclasses.dataclass
+class CaptureSpec:
+    prefix: str
+    output: Optional[DigestOrDest] = None
+    merged_capture: Optional[pathlib.Path] = None
+
+
+def _prepare_for_communication(
+    run: PreparedRun,
+    stdin: pathlib.Path,
+    stdout: pathlib.Path,
+    reverse_io: bool = False,
+    capture: Optional[CaptureSpec] = None,
+):
+    run.sandbox_params.set_stdio(
+        stdin=stdin,
+        stdout=stdout,
+    )
+    run.sandbox_params.reverse_io = reverse_io
+    if capture is not None:
+        run.sandbox_params.timeit_prefix = capture.prefix
+
+        if capture.output is not None:
+            output_path = PosixPath('capture')
+            run.sandbox_params.timeit_dups['do'].append(output_path)
+
+            run.artifacts.outputs.append(
+                GradingFileOutput(
+                    src=output_path,
+                    **capture.output.expand(),
+                    touch=True,
+                )
+            )
+
+        if capture.merged_capture is not None:
+            merged_output_path = package.get_merged_capture_path().resolve()
+            run.sandbox_params.timeit_dups['Do'].append(merged_output_path)
+
+            run.artifacts.outputs.append(
+                GradingFileOutput(
+                    src=merged_output_path,
+                    dest=capture.merged_capture,
+                )
+            )
+
+
 def _prepare_run(
     code: CodeItem,
     executable: DigestOrSource,
@@ -484,6 +530,7 @@ class CommunicationItem:
     outputs: Optional[List[GradingFileOutput]] = None
     extra_args: Optional[str] = None
     extra_config: Optional[ExecutionConfig] = None
+    capture: Optional[DigestOrDest] = None
 
     def prepare(self) -> PreparedRun:
         return _prepare_run(
@@ -500,6 +547,7 @@ class CommunicationItem:
 async def run_communication(
     interactor: CommunicationItem,
     solution: CommunicationItem,
+    merged_capture: Optional[pathlib.Path] = None,
     retry_index: Optional[int] = None,
 ):
     fifo_in, fifo_out = package.get_fifos()
@@ -510,10 +558,35 @@ async def run_communication(
     interactor_prepared.metadata.retryIndex = retry_index
     solution_prepared.metadata.retryIndex = retry_index
 
-    interactor_prepared.sandbox_params.set_stdio(stdin=fifo_out, stdout=fifo_in)
-    solution_prepared.sandbox_params.set_stdio(stdin=fifo_in, stdout=fifo_out)
+    interactor_prefix = 'INTERACTOR:'
+    solution_prefix = 'SOLUTION:'
+
+    if merged_capture is not None:
+        package.get_merged_capture_path().write_text(
+            f'{interactor_prefix}\n{solution_prefix}\n'
+        )
 
-    solution_prepared.sandbox_params.reverse_io = True
+    _prepare_for_communication(
+        interactor_prepared,
+        fifo_out,
+        fifo_in,
+        capture=CaptureSpec(
+            prefix=interactor_prefix,
+            output=interactor.capture,
+            merged_capture=merged_capture,
+        ),
+    )
+    _prepare_for_communication(
+        solution_prepared,
+        fifo_in,
+        fifo_out,
+        reverse_io=True,
+        capture=CaptureSpec(
+            prefix=solution_prefix,
+            output=solution.capture,
+            merged_capture=merged_capture,
+        ),
+    )
 
     interactor_run_params = steps.CoordinatedRunParams(
         command=interactor_prepared.command,

rbx/box/generators.py

@@ -1,6 +1,5 @@
 import pathlib
 import shutil
-import tempfile
 from typing import Dict, List, Optional, Set
 
 import typer
@@ -63,6 +62,47 @@ def _copy_testcase_over(
         )
 
 
+def _copy_testcase_output_over(
+    src_output_path: pathlib.Path, dest_output_path: pathlib.Path, suffix: str
+) -> bool:
+    dest_output_path.parent.mkdir(parents=True, exist_ok=True)
+
+    src_path = src_output_path.with_suffix(suffix)
+    if not src_path.is_file():
+        return False
+
+    shutil.copy(str(src_path), str(dest_output_path.with_suffix(suffix)))
+    return True
+
+
+def _copy_testcase_outputs_over(
+    testcase: Testcase, dest: Testcase, pipes: bool = False
+):
+    assert dest.outputPath is not None
+    dest.outputPath.parent.mkdir(parents=True, exist_ok=True)
+
+    has_copied = False
+
+    if testcase.outputPath is not None and testcase.outputPath.is_file():
+        shutil.copy(str(testcase.outputPath), str(dest.outputPath))
+        has_copied = True
+
+    if not pipes:
+        return has_copied
+
+    reference_path = testcase.outputPath or testcase.inputPath
+    if _copy_testcase_output_over(reference_path, dest.outputPath, '.pin'):
+        has_copied = True
+
+    if _copy_testcase_output_over(reference_path, dest.outputPath, '.pout'):
+        has_copied = True
+
+    if _copy_testcase_output_over(reference_path, dest.outputPath, '.pio'):
+        has_copied = True
+
+    return has_copied
+
+
 def get_all_built_testcases() -> Dict[str, List[Testcase]]:
     pkg = package.find_problem_package_or_die()
     res = {group.name: find_built_testcases(group) for group in pkg.testcases}
@@ -256,59 +296,46 @@ async def generate_testcases(
 async def generate_output_for_testcase(
     main_solution_digest: str,
     testcase: Testcase,
-    stderr_path: Optional[pathlib.Path] = None,
     interactor_digest: Optional[str] = None,
 ):
     assert testcase.outputPath is not None
     testcase.inputPath.parent.mkdir(parents=True, exist_ok=True)
     testcase.outputPath.parent.mkdir(parents=True, exist_ok=True)
 
-    if testcase.outputPath.is_file():
-        # Output file was already copied over from manual tests.
-        return
-
     main_solution = package.get_main_solution()
     if main_solution is None:
         return
 
-    with tempfile.TemporaryDirectory() as dir:
-        output_dir = pathlib.Path(dir)
+    eval: Evaluation = await run_solution_on_testcase(
+        main_solution,
+        main_solution_digest,
+        None,
+        testcase,
+        interactor_digest=interactor_digest,
+        use_retries=False,
+        use_timelimit=False,
+        capture_pipes=True,
+    )
 
-        eval: Evaluation = await run_solution_on_testcase(
-            main_solution,
-            main_solution_digest,
-            None,
-            testcase,
-            output_dir,
-            interactor_digest=interactor_digest,
-            use_retries=False,
-            use_timelimit=False,
+    if eval.result.outcome != Outcome.ACCEPTED:
+        console.console.print(
+            f'[error]Failed generating output for [item]{testcase.inputPath}[/item][/error]',
         )
-
-        if eval.log.stdout_absolute_path is not None:
-            shutil.copy(eval.log.stdout_absolute_path, testcase.outputPath)
-        if eval.log.stderr_absolute_path is not None and stderr_path is not None:
-            shutil.copy(eval.log.stderr_absolute_path, stderr_path)
-
-        if eval.result.outcome != Outcome.ACCEPTED:
-            console.console.print(
-                f'[error]Failed generating output for [item]{testcase.inputPath}[/item][/error]',
-            )
-            console.console.print(f'[error]Summary:[/error] {eval.log.get_summary()}')
-            console.console.print(
-                f'[warning]Verdict: [item]{eval.result.outcome.value}[/item][/warning]',
-            )
-            console.console.print(
-                f'[warning]Message: [info]{eval.result.message}[/info][/warning]',
-            )
-            console.console.print(f'Input written at [item]{testcase.inputPath}[/item]')
+        console.console.print(f'[error]Summary:[/error] {eval.log.get_summary()}')
+        console.console.print(
+            f'[warning]Verdict: [item]{eval.result.outcome.value}[/item][/warning]',
+        )
+        console.console.print(
+            f'[warning]Message: [info]{eval.result.message}[/info][/warning]',
+        )
+        console.console.print(f'Input written at [item]{testcase.inputPath}[/item]')
+        console.console.print(f'Output written at [item]{testcase.outputPath}[/item]')
+        if eval.log.stderr_absolute_path is not None:
             console.console.print(
-                f'Output written at [item]{testcase.outputPath}[/item]'
+                f'Stderr written at [item]{eval.log.stderr_absolute_path}[/item]'
             )
-            if stderr_path is not None:
-                console.console.print(f'Stderr written at [item]{stderr_path}[/item]')
 
-            raise typer.Exit(1)
+        raise typer.Exit(1)
 
 
 async def generate_outputs_for_testcases(
@@ -350,6 +377,14 @@ async def generate_outputs_for_testcases(
             return
         assert tc.outputPath is not None
 
+        if entry.metadata.copied_from is not None and _copy_testcase_outputs_over(
+            entry.metadata.copied_from, tc
+        ):
+            # Copy remaining pipe files.
+            _copy_testcase_outputs_over(entry.metadata.copied_from, tc, pipes=True)
+            step()
+            continue
+
         if (
             main_solution is None or solution_digest is None
         ) and not tc.outputPath.is_file():
@@ -362,7 +397,9 @@ async def generate_outputs_for_testcases(
         await generate_output_for_testcase(
             solution_digest,
             tc,
-            gen_runs_dir / 'main.stderr',
             interactor_digest=interactor_digest,
         )
+        if entry.metadata.copied_from is not None:
+            # Copy remaining pipe files.
+            _copy_testcase_outputs_over(entry.metadata.copied_from, tc, pipes=True)
         step()

rbx/box/main.py

@@ -1,6 +1,2 @@
 # flake8: noqa
-from gevent import monkey
-
-monkey.patch_all()
-
 from rbx.box.cli import app

rbx/box/package.py

@@ -393,16 +393,23 @@ def get_compilation_files(code: CodeItem) -> List[Tuple[pathlib.Path, pathlib.Pa
     return res
 
 
+@functools.cache
+def get_shared_dir(root: pathlib.Path = pathlib.Path()) -> pathlib.Path:
+    shared_dir = get_problem_cache_dir(root) / '.shared'
+    shared_dir.mkdir(parents=True, exist_ok=True)
+    return shared_dir
+
+
 @functools.cache
 def get_empty_sentinel_path(root: pathlib.Path = pathlib.Path()) -> pathlib.Path:
-    path = get_problem_cache_dir(root) / '.empty'
+    path = get_shared_dir(root) / '.empty'
     path.write_text('')
     return path
 
 
 @functools.cache
 def get_fifos(root: pathlib.Path = pathlib.Path()) -> Tuple[pathlib.Path, pathlib.Path]:
-    path = get_problem_cache_dir(root) / '.fifos'
+    path = get_shared_dir(root) / '.fifos'
     shutil.rmtree(path, ignore_errors=True)
     path.mkdir(parents=True, exist_ok=True)
     fifo_in = path / 'fifo.in'
@@ -412,6 +419,13 @@ def get_fifos(root: pathlib.Path = pathlib.Path()) -> Tuple[pathlib.Path, pathli
     return fifo_in, fifo_out
 
 
+@functools.cache
+def get_merged_capture_path(root: pathlib.Path = pathlib.Path()) -> pathlib.Path:
+    path = get_shared_dir(root) / '.merged_capture'
+    path.write_text('')
+    return path
+
+
 def clear_package_cache():
     pkgs = [sys.modules[__name__]]
 

rbx/box/solutions.py

@@ -185,7 +185,7 @@ def _run_solution(
                 compiled_digest,
                 checker_digest,
                 testcase,
-                output_path,
+                output_dir=output_path,
                 interactor_digest=interactor_digest,
                 testcase_index=i,
                 verification=verification,
@@ -367,6 +367,7 @@ async def _generate_testcase_interactively(
     )
 
     is_manual = False
+    is_output_manual = False
     generation_metadata = None
     if generator is not None:
         generation_metadata = GenerationMetadata(
@@ -398,6 +399,7 @@ async def _generate_testcase_interactively(
                 output = console.multiline_prompt('Testcase output')
                 testcase.outputPath.write_text(output)
                 console.console.print()
+            is_output_manual = True
 
         generation_metadata = GenerationMetadata(
             copied_to=testcase,
@@ -453,7 +455,7 @@ async def _generate_testcase_interactively(
             )
             raise
 
-    if main_solution_digest is not None:
+    if main_solution_digest is not None and not is_output_manual:
         pkg = package.find_problem_package_or_die()
         if pkg.type == TaskType.COMMUNICATION:
             interactor_digest = checkers.compile_interactor(progress)
@@ -523,7 +525,7 @@ def _run_interactive_solutions(
                 compiled_solutions[solution.path],
                 checker_digest,
                 testcase,
-                output_dir,
+                output_dir=output_dir,
                 interactor_digest=interactor_digest,
                 verification=verification,
             )

rbx/box/state.py

@@ -5,6 +5,7 @@ import dataclasses
 class State:
     run_through_cli: bool = False
     sanitized: bool = False
+    debug_logs: bool = False
 
 
 STATE = State()

rbx/box/statements/builders.py

@@ -25,6 +25,7 @@ from rbx.box.statements.schema import (
     TexToPDF,
     rbxToTeX,
 )
+from rbx.box.testcase_utils import TestcaseInteraction, parse_interaction
 
 
 @dataclasses.dataclass
@@ -63,13 +64,31 @@ class StatementSample(BaseModel):
     inputPath: pathlib.Path
     outputPath: pathlib.Path
     hasOutput: bool = True
+    interaction: Optional[TestcaseInteraction] = None
 
     @staticmethod
     def from_testcase(testcase: Testcase) -> 'StatementSample':
+        input_path = testcase.inputPath
+        output_path = testcase.outputPath
+
+        pin_path = input_path.with_suffix('.pin')
+        pout_path = input_path.with_suffix('.pout')
+        pio_path = input_path.with_suffix('.pio')
+
+        if pin_path.is_file():
+            input_path = pin_path
+        if pout_path.is_file():
+            output_path = pout_path
+
+        interaction = None
+        if pio_path.is_file():
+            interaction = parse_interaction(pio_path)
+
         return StatementSample(
-            inputPath=testcase.inputPath,
-            outputPath=testcase.outputPath or utils.get_empty_sentinel_path(),
-            hasOutput=testcase.outputPath is not None,
+            inputPath=input_path,
+            outputPath=output_path or utils.get_empty_sentinel_path(),
+            hasOutput=output_path is not None,
+            interaction=interaction,
         )
 
     @staticmethod

rbx/box/tasks.py

@@ -1,7 +1,7 @@
 import pathlib
 from typing import Optional
 
-from rbx.box import checkers, package
+from rbx.box import checkers, package, state
 from rbx.box.code import CommunicationItem, run_communication, run_item
 from rbx.box.environment import EnvironmentSandbox, ExecutionConfig, VerificationLevel
 from rbx.box.retries import Retrier
@@ -42,13 +42,14 @@ async def run_solution_on_testcase(
     compiled_digest: str,
     checker_digest: Optional[str],
     testcase: Testcase,
-    output_dir: pathlib.Path,
+    output_dir: Optional[pathlib.Path] = None,
     interactor_digest: Optional[str] = None,
     testcase_index: int = 0,
     verification: VerificationLevel = VerificationLevel.NONE,
     timelimit_override: Optional[int] = None,
     use_retries: bool = True,
     use_timelimit: bool = True,
+    capture_pipes: bool = False,
 ) -> Evaluation:
     if interactor_digest is not None:
         return await _run_communication_solution_on_testcase(
@@ -63,6 +64,7 @@ async def run_solution_on_testcase(
             timelimit_override=timelimit_override,
             use_retries=use_retries,
             use_timelimit=use_timelimit,
+            capture_pipes=capture_pipes,
         )
 
     async def run_fn(retry_index: int) -> Evaluation:
@@ -76,7 +78,11 @@ async def run_solution_on_testcase(
         )
         extra_config = _get_execution_config(limits, actual_sandbox)
 
-        output_path = output_dir / testcase.inputPath.with_suffix('.out').name
+        if output_dir is None:
+            assert testcase.outputPath is not None
+            output_path = testcase.outputPath
+        else:
+            output_path = output_dir / testcase.inputPath.with_suffix('.out').name
         error_path = output_path.with_suffix('.err')
         log_path = output_path.with_suffix('.log')
         output_path.parent.mkdir(parents=True, exist_ok=True)
@@ -149,13 +155,16 @@ async def _run_communication_solution_on_testcase(
     interactor_digest: str,
     checker_digest: Optional[str],
     testcase: Testcase,
-    output_dir: pathlib.Path,
+    output_dir: Optional[pathlib.Path] = None,
     testcase_index: int = 0,
     verification: VerificationLevel = VerificationLevel.NONE,
     timelimit_override: Optional[int] = None,
     use_retries: bool = True,
     use_timelimit: bool = True,
+    capture_pipes: bool = False,
 ) -> Evaluation:
+    capture_pipes = capture_pipes or state.STATE.debug_logs
+
     async def run_fn(retry_index: int) -> Evaluation:
         actual_sandbox = package.get_singleton_sandbox()
         interactor_sandbox = package.get_singleton_interactor_sandbox()
@@ -180,11 +189,18 @@ async def _run_communication_solution_on_testcase(
             )
         # TODO: maybe combine wall time limits?
 
-        output_path = output_dir / testcase.inputPath.with_suffix('.out').name
+        if output_dir is None:
+            assert testcase.outputPath is not None
+            output_path = testcase.outputPath
+        else:
+            output_path = output_dir / testcase.inputPath.with_suffix('.out').name
         error_path = output_path.with_suffix('.err')
         log_path = output_path.with_suffix('.log')
         output_path.parent.mkdir(parents=True, exist_ok=True)
 
+        interactor_capture_path = (
+            output_path.with_suffix('.pin') if capture_pipes else None
+        )
         interactor_item = CommunicationItem(
             code=package.get_interactor(),
             executable=DigestOrSource.create(interactor_digest),
@@ -204,17 +220,28 @@ async def _run_communication_solution_on_testcase(
                     touch=True,
                 )
             ],
+            capture=DigestOrDest.create(interactor_capture_path)
+            if interactor_capture_path
+            else None,
+        )
+        solution_capture_path = (
+            output_path.with_suffix('.pout') if capture_pipes else None
         )
         solution_item = CommunicationItem(
             code=solution,
             executable=DigestOrSource.create(compiled_digest),
             extra_config=extra_config,
+            capture=DigestOrDest.create(solution_capture_path)
+            if solution_capture_path
+            else None,
         )
 
+        merged_capture_path = output_path.with_suffix('.pio') if capture_pipes else None
         interactor_run_log, run_log = await run_communication(
             interactor=interactor_item,
             solution=solution_item,
             retry_index=retry_index,
+            merged_capture=merged_capture_path,
         )
 
         checker_result = await checkers.check_communication(

rbx/box/testcase_utils.py

@@ -97,6 +97,16 @@ class TestcaseData(BaseModel):
     output: str
 
 
+class TestcaseInteractionEntry(BaseModel):
+    data: str
+    pipe: int
+
+
+class TestcaseInteraction(BaseModel):
+    entries: List[TestcaseInteractionEntry]
+    prefixes: Tuple[str, str]
+
+
 def find_built_testcases(group: TestcaseGroup) -> List[Testcase]:
     inputs = find_built_testcase_inputs(group)
 
@@ -143,3 +153,59 @@ def fill_output_for_defined_testcase(testcase: Testcase) -> Testcase:
     if output_path.is_file():
         res.outputPath = output_path
     return res
+
+
+def parse_interaction(file: pathlib.Path) -> TestcaseInteraction:
+    entries = []
+    with file.open('r') as f:
+        try:
+            interactor_prefix = f.readline().strip()
+            solution_prefix = f.readline().strip()
+        except Exception:
+            console.console.print(
+                f'[error]Failed to read interaction file [item]{file}[/item]. Expected the first two lines to be the interactor and solution prefixes.[/error]'
+            )
+            raise typer.Exit(1) from None
+
+        rest = f.read()
+        start = 0
+
+        def _find_next_prefix(start: int) -> Optional[Tuple[int, int]]:
+            interactor_idx = rest.find(interactor_prefix, start)
+            solution_idx = rest.find(solution_prefix, start)
+            if interactor_idx == -1 and solution_idx == -1:
+                return None
+            if interactor_idx == -1:
+                return (solution_idx, solution_idx + len(solution_prefix))
+            if solution_idx == -1:
+                return (interactor_idx, interactor_idx + len(interactor_prefix))
+            if interactor_idx < solution_idx:
+                return (interactor_idx, interactor_idx + len(interactor_prefix))
+            return (solution_idx, solution_idx + len(solution_prefix))
+
+        def _find_next_block() -> Optional[Tuple[int, Tuple[int, int]]]:
+            prefix = _find_next_prefix(start)
+            if prefix is None:
+                return None
+            prefix_start, prefix_end = prefix
+            prefix = rest[prefix_start:prefix_end]
+            pipe = 1 if prefix == solution_prefix else 0
+
+            nxt = _find_next_prefix(prefix_end)
+            if nxt is None:
+                return (pipe, (prefix_end, len(rest)))
+            nxt_start, _ = nxt
+            return (pipe, (prefix_end, nxt_start))
+
+        while True:
+            block = _find_next_block()
+            if block is None:
+                break
+            pipe, (st, nd) = block
+            entries.append(TestcaseInteractionEntry(data=rest[st:nd], pipe=pipe))
+            start = nd
+
+    return TestcaseInteraction(
+        prefixes=(interactor_prefix, solution_prefix),
+        entries=entries,
+    )

rbx/grading/judge/cacher.py

@@ -9,8 +9,6 @@ import tempfile
 import typing
 from typing import IO, List, Optional
 
-import gevent
-
 from rbx.grading.judge import digester, storage
 
 logger = logging.getLogger(__name__)
@@ -318,8 +316,6 @@ class FileCacher:
                 d.update(buf)
                 while len(buf) > 0:
                     written = dst.write(buf)
-                    # Cooperative yield.
-                    gevent.sleep(0)
                     if written is None:
                         break
                     buf = buf[written:]

rbx/grading/judge/digester.py

@@ -2,8 +2,6 @@ import hashlib
 import pathlib
 from typing import IO
 
-import gevent
-
 
 class Digester:
     """Simple wrapper of hashlib using our preferred hasher."""
@@ -26,7 +24,6 @@ def digest_cooperatively_into_digester(
     buf = f.read(chunk_size)
     while len(buf) > 0:
         digester.update(buf)
-        gevent.sleep(0)
         buf = f.read(chunk_size)
 
 

rbx/grading/judge/sandbox.py

@@ -1,4 +1,5 @@
 import abc
+import collections
 import dataclasses
 import io
 import logging
@@ -114,6 +115,12 @@ class SandboxParams(pydantic.BaseModel):
     extra_timeout: Optional[int] = None  # ms
     reverse_io: bool = False
 
+    # For timeit
+    timeit_dups: Dict[str, List[pathlib.Path]] = dataclasses.field(
+        default_factory=lambda: collections.defaultdict(list)
+    )
+    timeit_prefix: Optional[str] = None
+
     def get_cacheable_params(self) -> Dict[str, Any]:
         return self.model_dump(mode='json', exclude_unset=True, exclude_none=True)