rbx.cp 0.5.39__py3-none-any.whl → 0.5.42__py3-none-any.whl

rbx/box/testcase_extractors.py

@@ -30,7 +30,7 @@ def _get_group_output(
     return group_path / f'{subgroup_prefix}{i:03d}.out'
 
 
-def _run_generator_script(testcase: TestcaseSubgroup) -> str:
+async def _run_generator_script(testcase: TestcaseSubgroup) -> str:
     assert testcase.generatorScript is not None
 
     cacher = package.get_file_cacher()
@@ -54,7 +54,7 @@ def _run_generator_script(testcase: TestcaseSubgroup) -> str:
             raise
 
         run_stderr = DigestHolder()
-        run_log = run_item(
+        run_log = await run_item(
             testcase.generatorScript,
             DigestOrSource.create(compiled_digest),
             stdout=DigestOrDest.create(script_digest),
@@ -116,7 +116,7 @@ class GenerationTestcaseEntry(BaseModel):
 
 class TestcaseVisitor(abc.ABC):
     @abc.abstractmethod
-    def visit(self, entry: GenerationTestcaseEntry):
+    async def visit(self, entry: GenerationTestcaseEntry):
         pass
 
     def should_visit_group(self, group_name: str) -> bool:
@@ -139,10 +139,10 @@ class TestcaseGroupVisitor(TestcaseVisitor):
         return self.groups is None or group_name in self.groups
 
 
-def run_testcase_visitor(visitor: TestcaseVisitor):
+async def run_testcase_visitor(visitor: TestcaseVisitor):
     pkg = package.find_problem_package_or_die()
 
-    def _explore_subgroup(
+    async def _explore_subgroup(
         subgroup: TestcaseSubgroup,
         subgroup_index: Optional[int],
         prefix: List[str],
@@ -179,7 +179,7 @@ def run_testcase_visitor(visitor: TestcaseVisitor):
         i = 0
         # Individual testcases.
         for tc in subgroup.testcases or []:
-            visitor.visit(
+            await visitor.visit(
                 GenerationTestcaseEntry(
                     group_entry=_entry(i),
                     subgroup_entry=_sub_entry(i),
@@ -202,7 +202,7 @@ def run_testcase_visitor(visitor: TestcaseVisitor):
                     continue
 
                 tc = Testcase(inputPath=input_path)
-                visitor.visit(
+                await visitor.visit(
                     GenerationTestcaseEntry(
                         group_entry=_entry(i),
                         subgroup_entry=_sub_entry(i),
@@ -218,7 +218,7 @@ def run_testcase_visitor(visitor: TestcaseVisitor):
 
         # Single generators.
         for generator_call in subgroup.generators:
-            visitor.visit(
+            await visitor.visit(
                 GenerationTestcaseEntry(
                     group_entry=_entry(i),
                     subgroup_entry=_sub_entry(i),
@@ -237,12 +237,12 @@ def run_testcase_visitor(visitor: TestcaseVisitor):
 
         # Run generator script.
         if subgroup.generatorScript is not None:
-            script = _run_generator_script(subgroup)
+            script = await _run_generator_script(subgroup)
 
             # Run each line from generator script.
             for generator_name, args, line_number in _extract_script_lines(script):
                 call = GeneratorCall(name=generator_name, args=args)
-                visitor.visit(
+                await visitor.visit(
                     GenerationTestcaseEntry(
                         group_entry=_entry(i),
                         subgroup_entry=_sub_entry(i),
@@ -269,7 +269,7 @@ def run_testcase_visitor(visitor: TestcaseVisitor):
             group_validator = group.validator
 
         extra_validators = group.extraValidators
-        _explore_subgroup(
+        await _explore_subgroup(
             group,
             0 if group.subgroups else None,
             [group.name],
@@ -278,7 +278,7 @@ def run_testcase_visitor(visitor: TestcaseVisitor):
         )
 
         for i, subgroup in enumerate(group.subgroups):
-            _explore_subgroup(
+            await _explore_subgroup(
                 subgroup,
                 i + 1,
                 [group.name, subgroup.name],
@@ -287,7 +287,7 @@ def run_testcase_visitor(visitor: TestcaseVisitor):
             )
 
 
-def extract_generation_testcases(
+async def extract_generation_testcases(
     entries: List[TestcaseEntry],
 ) -> List[GenerationTestcaseEntry]:
     # TODO: support subgroups.
@@ -300,30 +300,30 @@ def extract_generation_testcases(
         def should_visit_group(self, group_name: str) -> bool:
             return group_name in groups
 
-        def visit(self, entry: GenerationTestcaseEntry):
+        async def visit(self, entry: GenerationTestcaseEntry):
             # TODO: support subgroups.
             if entry.group_entry.key() not in entry_keys:
                 return
             res.append(entry)
 
-    run_testcase_visitor(ExtractGenerationTestcasesVisitor())
+    await run_testcase_visitor(ExtractGenerationTestcasesVisitor())
     return res
 
 
-def extract_generation_testcases_from_groups(
+async def extract_generation_testcases_from_groups(
     groups: Optional[Set[str]] = None,
 ) -> List[GenerationTestcaseEntry]:
     res: List[GenerationTestcaseEntry] = []
 
     class ExtractGenerationTestcasesVisitor(TestcaseGroupVisitor):
-        def visit(self, entry: GenerationTestcaseEntry):
+        async def visit(self, entry: GenerationTestcaseEntry):
             res.append(entry)
 
-    run_testcase_visitor(ExtractGenerationTestcasesVisitor(groups))
+    await run_testcase_visitor(ExtractGenerationTestcasesVisitor(groups))
     return res
 
 
-def extract_generation_testcases_from_patterns(
+async def extract_generation_testcases_from_patterns(
     patterns: List[TestcasePattern],
 ) -> List[GenerationTestcaseEntry]:
     res: List[GenerationTestcaseEntry] = []
@@ -337,12 +337,12 @@ def extract_generation_testcases_from_patterns(
                 pattern.intersecting_group(subgroup_path) for pattern in patterns
             )
 
-        def visit(self, entry: GenerationTestcaseEntry):
+        async def visit(self, entry: GenerationTestcaseEntry):
             if not any(
                 pattern.match(entry.group_entry) for pattern in patterns
             ) and not any(pattern.match(entry.subgroup_entry) for pattern in patterns):
                 return
             res.append(entry)
 
-    run_testcase_visitor(ExtractGenerationTestcasesVisitor())
+    await run_testcase_visitor(ExtractGenerationTestcasesVisitor())
     return res

rbx/box/testcases/main.py

@@ -1,6 +1,7 @@
 import pathlib
 from typing import Annotated, List, Optional
 
+import syncer
 import typer
 
 from rbx import annotations, config, utils
@@ -20,8 +21,8 @@ from rbx.console import console
 app = typer.Typer(no_args_is_help=True, cls=annotations.AliasGroup)
 
 
-def _find_testcase(entry: TestcaseEntry) -> GenerationTestcaseEntry:
-    extracted = extract_generation_testcases([entry])
+async def _find_testcase(entry: TestcaseEntry) -> GenerationTestcaseEntry:
+    extracted = await extract_generation_testcases([entry])
     if not extracted:
         console.print(f'[error]Testcase [item]{entry}[/item] not found.[/error]')
         raise typer.Exit(1)
@@ -35,7 +36,7 @@ def _should_generate_output(entry: GenerationTestcaseEntry) -> bool:
     ) and package.get_main_solution() is not None
 
 
-def _generate_input_for_editing(
+async def _generate_input_for_editing(
     entry: GenerationTestcaseEntry,
     output: bool = True,
     progress: Optional[utils.StatusProgress] = None,
@@ -43,7 +44,7 @@ def _generate_input_for_editing(
     if (
         output and _should_generate_output(entry)
     ) or entry.metadata.copied_from is None:
-        generate_standalone(
+        await generate_standalone(
             entry.metadata,
             validate=False,
             group_entry=entry.group_entry,
@@ -54,7 +55,7 @@ def _generate_input_for_editing(
     return entry.metadata.copied_to.inputPath
 
 
-def _generate_output_for_editing(
+async def _generate_output_for_editing(
     entry: GenerationTestcaseEntry,
     progress: Optional[utils.StatusProgress] = None,
 ) -> Optional[pathlib.Path]:
@@ -65,29 +66,32 @@ def _generate_output_for_editing(
         return entry.metadata.copied_from.outputPath
     if not _should_generate_output(entry):
         return None
-    generate_outputs_for_testcases([entry.group_entry], progress=progress)
+    await generate_outputs_for_testcases([entry.group_entry], progress=progress)
     return entry.metadata.copied_to.outputPath
 
 
-def _generate_for_editing(
+async def _generate_for_editing(
     entry: GenerationTestcaseEntry,
     input: bool,
     output: bool,
     progress: Optional[utils.StatusProgress] = None,
 ) -> List[pathlib.Path]:
     res = []
-    input_path = _generate_input_for_editing(entry, output=output, progress=progress)
+    input_path = await _generate_input_for_editing(
+        entry, output=output, progress=progress
+    )
     if input:
         res.append(input_path)
     if output:
-        output_path = _generate_output_for_editing(entry, progress=progress)
+        output_path = await _generate_output_for_editing(entry, progress=progress)
         if output_path is not None:
             res.append(output_path)
     return res
 
 
 @app.command('view, v', help='View a testcase in your default editor.')
-def view(
+@syncer.sync
+async def view(
     tc: Annotated[
         str,
         typer.Argument(help='Testcase to view. Format: [group]/[index].'),
@@ -112,17 +116,18 @@ def view(
         raise typer.Exit(1)
 
     entry = TestcaseEntry.parse(tc)
-    testcase = _find_testcase(entry)
+    testcase = await _find_testcase(entry)
 
     with utils.StatusProgress('Preparing testcase...') as s:
-        items = _generate_for_editing(
+        items = await _generate_for_editing(
             testcase, input=not output_only, output=not input_only, progress=s
         )
     config.edit_multiple(items, readonly=True)
 
 
 @app.command('info, i', help='Show information about testcases.')
-def info(
+@syncer.sync
+async def info(
     pattern: Annotated[
         Optional[str],
         typer.Argument(
@@ -131,7 +136,7 @@ def info(
     ] = None,
 ):
     tc_pattern = TestcasePattern.parse(pattern or '*')
-    testcases = extract_generation_testcases_from_patterns([tc_pattern])
+    testcases = await extract_generation_testcases_from_patterns([tc_pattern])
     if not testcases:
         console.print(
             f'[error]No testcases found matching pattern [item]{pattern}[/item].[/error]'

rbx/box/unit.py

@@ -0,0 +1,116 @@
+from typing import List, Optional
+
+import syncer
+
+from rbx import console
+from rbx.box import checkers, package, validators
+from rbx.box.schema import CodeItem, Testcase, ValidatorOutcome, ValidatorTest
+from rbx.utils import StatusProgress
+
+
+def _get_validator_for_test(test: ValidatorTest) -> Optional[CodeItem]:
+    pkg = package.find_problem_package_or_die()
+    if test.validator is not None:
+        return test.validator
+    return pkg.validator
+
+
+async def run_validator_unit_tests(progress: StatusProgress):
+    pkg = package.find_problem_package_or_die()
+
+    vals: List[CodeItem] = []
+    for test in pkg.unitTests.validator:
+        val = _get_validator_for_test(test)
+        if val is not None:
+            vals.append(val)
+
+    compiled_validators = validators.compile_validators(vals, progress=progress)
+
+    if progress:
+        progress.update('Running validator unit tests...')
+
+    console.console.rule('Validator tests', style='info')
+
+    for i, test in enumerate(pkg.unitTests.validator):
+        val = _get_validator_for_test(test)
+        if val is None:
+            console.console.print(
+                f'[warning]No validator found for test [item]#{i + 1}[/item], skipping.[/warning]'
+            )
+            continue
+
+        compiled_digest = compiled_validators[str(val.path)]
+        info = await validators.validate_one_off(
+            test.input,
+            val,
+            compiled_digest,
+        )
+
+        is_valid = test.outcome == ValidatorOutcome.VALID
+
+        markup = (
+            '[success]OK[/success]' if info.ok == is_valid else '[error]FAIL[/error]'
+        )
+
+        console.console.print(
+            f'{markup} Unit test [item]#{i + 1}[/item] for [item]{test.input}[/item]'
+        )
+        console.console.print(f'  [status]Expected[/status] {test.outcome.value}')
+        if info.ok != is_valid:
+            if info.ok:
+                console.console.print('  [status]Actual[/status] VALID')
+            else:
+                console.console.print(f'  [status]Actual[/status] {info.message}')
+
+
+async def run_checker_unit_tests(progress: StatusProgress):
+    pkg = package.find_problem_package_or_die()
+    if not pkg.unitTests.checker:
+        return
+
+    if not package.get_checker():
+        console.console.print(
+            '[warning]No checker found, skipping checker unit tests.[/warning]'
+        )
+        return
+
+    compiled_digest = checkers.compile_checker(progress=progress)
+
+    if progress:
+        progress.update('Running checker unit tests...')
+
+    console.console.rule('Checker tests', style='info')
+
+    empty_file = package.get_empty_sentinel_path()
+
+    for i, test in enumerate(pkg.unitTests.checker):
+        result = await checkers.check(
+            compiled_digest,
+            run_log=None,
+            testcase=Testcase(
+                inputPath=test.input or empty_file,
+                outputPath=test.answer or empty_file,
+            ),
+            program_output=test.output or empty_file,
+            skip_run_log=True,
+        )
+
+        markup = (
+            '[success]OK[/success]'
+            if test.outcome.match(result.outcome)
+            else '[error]FAIL[/error]'
+        )
+
+        console.console.print(f'{markup} Unit test [item]#{i + 1}[/item]')
+        console.console.print(f'  [status]Expected[/status] {test.outcome.name}')
+
+        if not test.outcome.match(result.outcome):
+            console.console.print(f'  [status]Actual[/status] {result.outcome.name}')
+            if result.message:
+                console.console.print(f'  [status]Message[/status] {result.message}')
+
+
+@syncer.sync
+async def run_unit_tests(progress: StatusProgress):
+    await run_validator_unit_tests(progress)
+    await run_checker_unit_tests(progress)

rbx/box/validators.py

@@ -86,7 +86,7 @@ def _has_group_specific_validator() -> bool:
     return any(group.validator is not None for group in pkg.testcases)
 
 
-def _validate_testcase(
+async def _validate_testcase(
     testcase: pathlib.Path,
     validator: CodeItem,
     validator_digest: str,
@@ -103,7 +103,7 @@ def _validate_testcase(
 
     message_digest = DigestHolder()
     log_digest = DigestHolder()
-    run_log = run_item(
+    run_log = await run_item(
         validator,
         DigestOrSource.create(validator_digest),
         stdin=DigestOrSource.create(testcase),
@@ -140,13 +140,13 @@ def _validate_testcase(
     )
 
 
-def _validate_test(
+async def _validate_test(
     testcase: pathlib.Path,
     validator: CodeItem,
     validator_digest: str,
 ) -> Tuple[bool, Optional[str], HitBounds]:
     pkg = package.find_problem_package_or_die()
-    return _validate_testcase(
+    return await _validate_testcase(
         testcase, validator, validator_digest, vars=pkg.expanded_vars
     )
 
@@ -159,12 +159,12 @@ def compile_main_validator() -> Optional[Tuple[CodeItem, str]]:
     return pkg.validator, _compile_validator(pkg.validator)
 
 
-def validate_one_off(
+async def validate_one_off(
     testcase: pathlib.Path,
     validator: CodeItem,
     validator_digest: str,
 ) -> TestcaseValidationInfo:
-    ok, message, _ = _validate_test(testcase, validator, validator_digest)
+    ok, message, _ = await _validate_test(testcase, validator, validator_digest)
     info = TestcaseValidationInfo(
         validator=validator,
         group='interactive',
@@ -177,15 +177,10 @@ def validate_one_off(
 
 
 def compile_validators(
-    validation_entries: List[GenerationTestcaseEntry],
+    validators: List[CodeItem],
     progress: Optional[StatusProgress] = None,
 ) -> Dict[str, str]:
-    validators = []
-
-    for entry in validation_entries:
-        if entry.validator is not None:
-            validators.append(entry.validator)
-        validators.extend(entry.extra_validators)
+    validator_to_compiled_digest = {}
 
     validator_to_compiled_digest = {}
 
@@ -202,7 +197,21 @@ def compile_validators(
     return validator_to_compiled_digest
 
 
-def validate_testcases(
+def compile_validators_for_entries(
+    validation_entries: List[GenerationTestcaseEntry],
+    progress: Optional[StatusProgress] = None,
+) -> Dict[str, str]:
+    validators = []
+
+    for entry in validation_entries:
+        if entry.validator is not None:
+            validators.append(entry.validator)
+        validators.extend(entry.extra_validators)
+
+    return compile_validators(validators, progress=progress)
+
+
+async def validate_testcases(
     progress: Optional[StatusProgress] = None,
     groups: Optional[Set[str]] = None,
 ) -> List[TestcaseValidationInfo]:
@@ -210,8 +219,8 @@ def validate_testcases(
         if progress is not None:
             progress.step()
 
-    validation_entries = extract_generation_testcases_from_groups(groups)
-    validator_to_compiled_digest = compile_validators(
+    validation_entries = await extract_generation_testcases_from_groups(groups)
+    validator_to_compiled_digest = compile_validators_for_entries(
         validation_entries, progress=progress
     )
 
@@ -225,7 +234,7 @@ def validate_testcases(
         # Main validation.
         if entry.validator is not None:
             compiled_digest = validator_to_compiled_digest[str(entry.validator.path)]
-            ok, message, hit_bounds = _validate_test(
+            ok, message, hit_bounds = await _validate_test(
                 input_path, entry.validator, compiled_digest
             )
             validation_info.append(
@@ -241,7 +250,7 @@ def validate_testcases(
 
         for extra_validator in entry.extra_validators:
             compiled_digest = validator_to_compiled_digest[str(extra_validator.path)]
-            ok, message, hit_bounds = _validate_test(
+            ok, message, hit_bounds = await _validate_test(
                 input_path, extra_validator, compiled_digest
             )
             validation_info.append(

rbx/box/validators_test.py

@@ -7,9 +7,9 @@ from rbx.box.validators import validate_testcases
 
 
 @pytest.mark.test_pkg('box1')
-def test_validators(pkg_from_testdata: pathlib.Path):
-    generate_testcases()
-    validation_infos = validate_testcases()
+async def test_validators(pkg_from_testdata: pathlib.Path):
+    await generate_testcases()
+    validation_infos = await validate_testcases()
 
     for info in validation_infos:
         assert info.ok

rbx/grading/judge/sandbox.py

@@ -112,6 +112,7 @@ class SandboxParams(pydantic.BaseModel):
     timeout: Optional[int] = None  # ms
     wallclock_timeout: Optional[int] = None  # ms
     extra_timeout: Optional[int] = None  # ms
+    reverse_io: bool = False
 
     def get_cacheable_params(self) -> Dict[str, Any]:
         return self.model_dump(mode='json', exclude_unset=True, exclude_none=True)
@@ -393,6 +394,13 @@ class SandboxBase(abc.ABC):
             return None
         return real_path
 
+    def create_fifo(self, path: pathlib.Path, override: bool = False):
+        real_path = self.relative_path(path)
+        if override:
+            real_path.unlink(missing_ok=True)
+        os.mkfifo(str(real_path))
+        return real_path
+
     def create_file_from_storage(
         self,
         path: pathlib.Path,

rbx/grading/judge/sandboxes/stupid_sandbox.py

@@ -91,17 +91,22 @@ class StupidSandbox(SandboxBase):
             args.append(f'-w{walltimeout_in_s:.3f}')
         if self.params.address_space:
             args.append(f'-m{self.params.address_space}')
-        if self.params.stdin_file:
-            args.append(f'-i{self.params.stdin_file}')
-        if self.params.stdout_file:
-            args.append(f'-o{self.params.stdout_file}')
-        if self.params.stderr_file:
-            args.append(f'-e{self.params.stderr_file}')
         if self.params.fsize:
             args.append(f'-f{self.params.fsize}')
         if self.chdir:
             args.append(f'-c{self.chdir}')
-        return args
+
+        file_args = []
+        if self.params.stdin_file:
+            file_args.append(f'-i{self.params.stdin_file}')
+        if self.params.stdout_file:
+            file_args.append(f'-o{self.params.stdout_file}')
+        if self.params.stderr_file:
+            file_args.append(f'-e{self.params.stderr_file}')
+        if self.params.reverse_io:
+            file_args.reverse()
+
+        return args + file_args
 
     def get_root_path(self) -> pathlib.Path:
         """Return the toplevel path of the sandbox.

rbx/grading/judge/sandboxes/timeit.py

@@ -9,7 +9,7 @@ from time import monotonic
 from typing import List, Optional
 
 
-@dataclasses.dataclass()
+@dataclasses.dataclass
 class Options:
     output_file: str
     argv: List[str]
@@ -21,6 +21,7 @@ class Options:
     wall_time_limit: Optional[float] = None  # seconds
     memory_limit: Optional[int] = None  # kb, but passed in args as mb
     fs_limit: Optional[int] = None  # kb
+    files_to_open: List[int] = dataclasses.field(default_factory=list)
 
 
 def exit_with(code: int):
@@ -29,6 +30,7 @@ def exit_with(code: int):
 
 def parse_opts() -> Options:
     options = Options(output_file=sys.argv[1], argv=[])
+    options.files_to_open = []
     num_opts = 0
     while num_opts + 2 < len(sys.argv) and sys.argv[num_opts + 2].startswith('-'):
         # Process option
@@ -41,10 +43,13 @@ def parse_opts() -> Options:
             options.memory_limit = int(opt[2:]) * 1024
         elif opt.startswith('-i'):
             options.stdin_file = opt[2:]
+            options.files_to_open.append(0)
         elif opt.startswith('-o'):
             options.stdout_file = opt[2:]
+            options.files_to_open.append(1)
         elif opt.startswith('-e'):
             options.stderr_file = opt[2:]
+            options.files_to_open.append(2)
         elif opt.startswith('-c'):
             options.chdir = opt[2:]
         elif opt.startswith('-f'):
@@ -92,7 +97,8 @@ def set_rlimits(options: Options):
 def redirect_fds(options: Options):
     files = [options.stdin_file, options.stdout_file, options.stderr_file]
 
-    for i, file in enumerate(files):
+    for i in options.files_to_open:
+        file = files[i]
         if file is None:
             continue
         open_args = [