rbx.cp 0.5.36__py3-none-any.whl → 0.5.38__py3-none-any.whl

rbx/box/solutions.py

@@ -7,7 +7,6 @@ import shutil
 from collections.abc import Iterator
 from typing import Dict, Iterable, List, Optional, Set, Tuple
 
-import questionary
 import rich
 import rich.live
 import rich.markup
@@ -29,7 +28,6 @@ from rbx.box.formatting import get_formatted_memory, get_formatted_time
 from rbx.box.generators import (
     GenerationMetadata,
     expand_generator_call,
-    extract_generation_testcases,
     generate_output_for_testcase,
     generate_standalone,
 )
@@ -42,6 +40,7 @@ from rbx.box.schema import (
     Testcase,
     TestcaseGroup,
 )
+from rbx.box.testcase_extractors import extract_generation_testcases
 from rbx.box.testcase_utils import TestcaseEntry, find_built_testcases
 from rbx.grading.steps import (
     DigestOrDest,
@@ -674,6 +673,8 @@ def pick_solutions(tracked_solutions: Optional[Set[str]]) -> List[str]:
         tracked_solutions = set(str(sol.path) for sol in pkg.solutions)
 
     # Store in a separate list to maintain order with the package declaration.
+    import questionary
+
     choices = [
         questionary.Choice(title=_get_solution_repr(sol), value=str(sol.path))
         for sol in pkg.solutions

rbx/box/solutions_test.py

@@ -5,7 +5,6 @@ import pytest
 
 from rbx.box.environment import VerificationLevel
 from rbx.box.generators import (
-    extract_generation_testcases_from_groups,
     generate_outputs_for_testcases,
     generate_testcases,
 )
@@ -13,6 +12,7 @@ from rbx.box.solutions import (
     convert_list_of_solution_evaluations_to_dict,
     run_solutions,
 )
+from rbx.box.testcase_extractors import extract_generation_testcases_from_groups
 from rbx.grading.steps import Outcome
 
 

rbx/box/statements/build_statements.py

@@ -6,7 +6,7 @@ from typing import Annotated, Dict, List, Optional, Tuple
 import typer
 
 from rbx import annotations, console
-from rbx.box import builder, environment, package
+from rbx.box import environment, package
 from rbx.box.schema import Package
 from rbx.box.statements.builders import (
     BUILDER_LIST,
@@ -333,6 +333,8 @@ def build(
 ):
     # At most run the validators, only in samples.
     if samples:
+        from rbx.box import builder
+
         if not builder.build(
             verification=verification,
             groups=set(['samples']),

rbx/box/statements/builders.py

@@ -11,12 +11,6 @@ from pydantic import BaseModel
 
 from rbx import console, utils
 from rbx.box.schema import Package, Primitive, Testcase
-from rbx.box.statements.latex import (
-    MAX_PDFLATEX_RUNS,
-    Latex,
-    decode_latex_output,
-    should_rerun,
-)
 from rbx.box.statements.latex_jinja import (
     JinjaDictWrapper,
     render_latex_template,
@@ -355,6 +349,13 @@ class TeX2PDFBuilder(StatementBuilder):
         item: StatementBuilderItem,
         verbose: bool = False,
     ) -> bytes:
+        from rbx.box.statements.latex import (
+            MAX_PDFLATEX_RUNS,
+            Latex,
+            decode_latex_output,
+            should_rerun,
+        )
+
         latex = Latex(input.decode())
         latex_result = latex.build_pdf(context.root)
         pdf = latex_result.pdf

rbx/box/statements/joiners.py

@@ -10,12 +10,6 @@ from rbx.box.statements.builders import (
     StatementBuilderContest,
     StatementCodeLanguage,
 )
-from rbx.box.statements.latex import (
-    MAX_PDFLATEX_RUNS,
-    Latex,
-    decode_latex_output,
-    should_rerun,
-)
 from rbx.box.statements.schema import Joiner, JoinerType, JoinTexToPDF, StatementType
 
 
@@ -84,6 +78,13 @@ class TeX2PDFJoiner(StatementJoiner):
         contest: StatementBuilderContest,
         verbose: bool = False,
     ) -> bytes:
+        from rbx.box.statements.latex import (
+            MAX_PDFLATEX_RUNS,
+            Latex,
+            decode_latex_output,
+            should_rerun,
+        )
+
         latex = Latex(input.decode())
         latex_result = latex.build_pdf(context.root)
         pdf = latex_result.pdf

rbx/box/testcase_extractors.py

@@ -0,0 +1,348 @@
+import abc
+import pathlib
+import shlex
+from typing import Iterable, List, Optional, Set, Tuple
+
+import typer
+from pydantic import BaseModel
+
+from rbx import console
+from rbx.box import package
+from rbx.box.code import compile_item, run_item
+from rbx.box.schema import CodeItem, GeneratorCall, Testcase, TestcaseSubgroup
+from rbx.box.testcase_utils import (
+    TestcaseEntry,
+    TestcasePattern,
+    fill_output_for_defined_testcase,
+)
+from rbx.grading.steps import DigestHolder, DigestOrDest, DigestOrSource
+
+
+def _get_group_input(
+    group_path: pathlib.Path, subgroup_prefix: str, i: int
+) -> pathlib.Path:
+    return group_path / f'{subgroup_prefix}{i:03d}.in'
+
+
+def _get_group_output(
+    group_path: pathlib.Path, subgroup_prefix: str, i: int
+) -> pathlib.Path:
+    return group_path / f'{subgroup_prefix}{i:03d}.out'
+
+
+def _run_generator_script(testcase: TestcaseSubgroup) -> str:
+    assert testcase.generatorScript is not None
+
+    cacher = package.get_file_cacher()
+
+    if not testcase.generatorScript.path.is_file():
+        console.console.print(
+            f'[error]Generator script not found: [item]{testcase.generatorScript.path}[/item][/error]'
+        )
+        raise typer.Exit(1)
+
+    script_digest = DigestHolder()
+    if testcase.generatorScript.path.suffix == '.txt':
+        script_digest.value = cacher.put_file_from_path(testcase.generatorScript.path)
+    else:
+        try:
+            compiled_digest = compile_item(testcase.generatorScript)
+        except:
+            console.console.print(
+                f'[error]Failed compiling generator script for group [item]{testcase.name}[/item].[/error]'
+            )
+            raise
+
+        run_stderr = DigestHolder()
+        run_log = run_item(
+            testcase.generatorScript,
+            DigestOrSource.create(compiled_digest),
+            stdout=DigestOrDest.create(script_digest),
+            stderr=DigestOrDest.create(run_stderr),
+        )
+
+        if run_log is None or run_log.exitcode != 0:
+            console.console.print(
+                f'Could not run generator script for group {testcase.name}'
+            )
+            if run_log is not None:
+                console.console.print(
+                    f'[error]Summary:[/error] {run_log.get_summary()}'
+                )
+            if run_stderr.value is not None:
+                console.console.print('[error]Stderr:[/error]')
+                console.console.print(
+                    package.get_digest_as_string(run_stderr.value) or ''
+                )
+            raise typer.Exit(1)
+
+    assert script_digest.value
+    script = cacher.get_file_content(script_digest.value).decode()
+    return script
+
+
+def _extract_script_lines(script: str) -> Iterable[Tuple[str, str, int]]:
+    lines = script.splitlines()
+    for i, line in enumerate(lines):
+        line = line.strip()
+        if not line:
+            continue
+        if line.startswith('#'):
+            continue
+        yield shlex.split(line)[0], shlex.join(shlex.split(line)[1:]), i + 1
+
+
+class GeneratorScriptEntry(BaseModel):
+    path: pathlib.Path
+    line: int
+
+
+class GenerationMetadata(BaseModel):
+    copied_to: Testcase
+
+    copied_from: Optional[Testcase] = None
+    generator_call: Optional[GeneratorCall] = None
+    generator_script: Optional[GeneratorScriptEntry] = None
+
+
+class GenerationTestcaseEntry(BaseModel):
+    group_entry: TestcaseEntry
+    subgroup_entry: TestcaseEntry
+
+    metadata: GenerationMetadata
+    validator: Optional[CodeItem] = None
+    extra_validators: List[CodeItem] = []
+
+
+class TestcaseVisitor(abc.ABC):
+    @abc.abstractmethod
+    def visit(self, entry: GenerationTestcaseEntry):
+        pass
+
+    def should_visit_group(self, group_name: str) -> bool:
+        return True
+
+    def should_visit_subgroup(self, subgroup_path: str) -> bool:
+        return True
+
+    def should_visit_generator_scripts(
+        self, group_name: str, subgroup_path: str
+    ) -> bool:
+        return True
+
+
+class TestcaseGroupVisitor(TestcaseVisitor):
+    def __init__(self, groups: Optional[Set[str]] = None):
+        self.groups = groups
+
+    def should_visit_group(self, group_name: str) -> bool:
+        return self.groups is None or group_name in self.groups
+
+
+def run_testcase_visitor(visitor: TestcaseVisitor):
+    pkg = package.find_problem_package_or_die()
+
+    def _explore_subgroup(
+        subgroup: TestcaseSubgroup,
+        subgroup_index: Optional[int],
+        prefix: List[str],
+        validator: Optional[CodeItem] = None,
+        extra_validators: Optional[List[CodeItem]] = None,
+    ):
+        extra_validators = extra_validators or []
+
+        assert prefix and len(prefix) >= 1 and len(prefix) <= 2
+        group_path = prefix[0]
+        subgroup_path = '/'.join(prefix)
+        if not visitor.should_visit_subgroup(subgroup_path):
+            return
+
+        def _entry(i: int) -> TestcaseEntry:
+            return TestcaseEntry(group=group_path, index=i)
+
+        def _sub_entry(i: int) -> TestcaseEntry:
+            return TestcaseEntry(group=subgroup_path, index=i)
+
+        def _copied_to(i: int) -> Testcase:
+            group_fs_path = package.get_build_testgroup_path(group_path)
+            group_prefix = ''
+            if subgroup_index is not None:
+                group_prefix = f'{subgroup_index}-'
+            if len(prefix) == 2:
+                group_prefix += f'{prefix[1]}-'
+            return Testcase(
+                inputPath=_get_group_input(group_fs_path, group_prefix, i),
+                outputPath=_get_group_output(group_fs_path, group_prefix, i),
+            )
+
+        # Go through testcases.
+        i = 0
+        # Individual testcases.
+        for tc in subgroup.testcases or []:
+            visitor.visit(
+                GenerationTestcaseEntry(
+                    group_entry=_entry(i),
+                    subgroup_entry=_sub_entry(i),
+                    metadata=GenerationMetadata(
+                        copied_from=fill_output_for_defined_testcase(tc),
+                        copied_to=_copied_to(i),
+                    ),
+                    validator=validator,
+                    extra_validators=extra_validators,
+                )
+            )
+            i += 1
+
+        # Glob testcases.
+        if subgroup.testcaseGlob:
+            matched_inputs = sorted(pathlib.PosixPath().glob(subgroup.testcaseGlob))
+
+            for input_path in matched_inputs:
+                if not input_path.is_file() or input_path.suffix != '.in':
+                    continue
+
+                tc = Testcase(inputPath=input_path)
+                visitor.visit(
+                    GenerationTestcaseEntry(
+                        group_entry=_entry(i),
+                        subgroup_entry=_sub_entry(i),
+                        metadata=GenerationMetadata(
+                            copied_from=fill_output_for_defined_testcase(tc),
+                            copied_to=_copied_to(i),
+                        ),
+                        validator=validator,
+                        extra_validators=extra_validators,
+                    )
+                )
+                i += 1
+
+        # Single generators.
+        for generator_call in subgroup.generators:
+            visitor.visit(
+                GenerationTestcaseEntry(
+                    group_entry=_entry(i),
+                    subgroup_entry=_sub_entry(i),
+                    metadata=GenerationMetadata(
+                        generator_call=generator_call,
+                        copied_to=_copied_to(i),
+                    ),
+                    validator=validator,
+                    extra_validators=extra_validators,
+                )
+            )
+            i += 1
+
+        if not visitor.should_visit_generator_scripts(group_path, subgroup_path):
+            return
+
+        # Run generator script.
+        if subgroup.generatorScript is not None:
+            script = _run_generator_script(subgroup)
+
+            # Run each line from generator script.
+            for generator_name, args, line_number in _extract_script_lines(script):
+                call = GeneratorCall(name=generator_name, args=args)
+                visitor.visit(
+                    GenerationTestcaseEntry(
+                        group_entry=_entry(i),
+                        subgroup_entry=_sub_entry(i),
+                        metadata=GenerationMetadata(
+                            generator_call=call,
+                            generator_script=GeneratorScriptEntry(
+                                path=subgroup.generatorScript.path,
+                                line=line_number,
+                            ),
+                            copied_to=_copied_to(i),
+                        ),
+                        validator=validator,
+                        extra_validators=extra_validators,
+                    )
+                )
+                i += 1
+
+    for group in pkg.testcases:
+        if not visitor.should_visit_group(group.name):
+            continue
+
+        group_validator = pkg.validator
+        if group.validator is not None:
+            group_validator = group.validator
+
+        extra_validators = group.extraValidators
+        _explore_subgroup(
+            group,
+            0 if group.subgroups else None,
+            [group.name],
+            validator=group_validator,
+            extra_validators=extra_validators,
+        )
+
+        for i, subgroup in enumerate(group.subgroups):
+            _explore_subgroup(
+                subgroup,
+                i + 1,
+                [group.name, subgroup.name],
+                validator=group_validator,
+                extra_validators=extra_validators + subgroup.extraValidators,
+            )
+
+
+def extract_generation_testcases(
+    entries: List[TestcaseEntry],
+) -> List[GenerationTestcaseEntry]:
+    # TODO: support subgroups.
+    groups = set(entry.group for entry in entries)
+    entry_keys = set(entry.key() for entry in entries)
+
+    res: List[GenerationTestcaseEntry] = []
+
+    class ExtractGenerationTestcasesVisitor(TestcaseVisitor):
+        def should_visit_group(self, group_name: str) -> bool:
+            return group_name in groups
+
+        def visit(self, entry: GenerationTestcaseEntry):
+            # TODO: support subgroups.
+            if entry.group_entry.key() not in entry_keys:
+                return
+            res.append(entry)
+
+    run_testcase_visitor(ExtractGenerationTestcasesVisitor())
+    return res
+
+
+def extract_generation_testcases_from_groups(
+    groups: Optional[Set[str]] = None,
+) -> List[GenerationTestcaseEntry]:
+    res: List[GenerationTestcaseEntry] = []
+
+    class ExtractGenerationTestcasesVisitor(TestcaseGroupVisitor):
+        def visit(self, entry: GenerationTestcaseEntry):
+            res.append(entry)
+
+    run_testcase_visitor(ExtractGenerationTestcasesVisitor(groups))
+    return res
+
+
+def extract_generation_testcases_from_patterns(
+    patterns: List[TestcasePattern],
+) -> List[GenerationTestcaseEntry]:
+    res: List[GenerationTestcaseEntry] = []
+
+    class ExtractGenerationTestcasesVisitor(TestcaseVisitor):
+        def should_visit_group(self, group_name: str) -> bool:
+            return any(pattern.intersecting_group(group_name) for pattern in patterns)
+
+        def should_visit_subgroup(self, subgroup_path: str) -> bool:
+            return any(
+                pattern.intersecting_group(subgroup_path) for pattern in patterns
+            )
+
+        def visit(self, entry: GenerationTestcaseEntry):
+            if not any(
+                pattern.match(entry.group_entry) for pattern in patterns
+            ) and not any(pattern.match(entry.subgroup_entry) for pattern in patterns):
+                return
+            res.append(entry)
+
+    run_testcase_visitor(ExtractGenerationTestcasesVisitor())
+    return res

rbx/box/testcase_utils.py

@@ -133,3 +133,13 @@ def get_samples() -> List[Testcase]:
         )
         for tc in tcs
     ]
+
+
+def fill_output_for_defined_testcase(testcase: Testcase) -> Testcase:
+    res = testcase.model_copy()
+    if res.outputPath is not None:
+        return res
+    output_path = res.inputPath.with_suffix('.ans')
+    if output_path.is_file():
+        res.outputPath = output_path
+    return res

rbx/box/testcases/main.py

@@ -7,11 +7,13 @@ from rbx import annotations, config, utils
 from rbx.box import package
 from rbx.box.generators import (
     GenerationTestcaseEntry,
-    extract_generation_testcases,
-    extract_generation_testcases_from_patterns,
     generate_outputs_for_testcases,
     generate_standalone,
 )
+from rbx.box.testcase_extractors import (
+    extract_generation_testcases,
+    extract_generation_testcases_from_patterns,
+)
 from rbx.box.testcase_utils import TestcaseEntry, TestcasePattern
 from rbx.console import console
 

rbx/box/validators.py

@@ -9,7 +9,10 @@ from rbx import console
 from rbx.box import package
 from rbx.box.code import SanitizationLevel, compile_item, run_item
 from rbx.box.schema import CodeItem, Primitive
-from rbx.box.testcase_utils import find_built_testcase_inputs
+from rbx.box.testcase_extractors import (
+    GenerationTestcaseEntry,
+    extract_generation_testcases_from_groups,
+)
 from rbx.grading.judge.sandbox import SandboxBase
 from rbx.grading.steps import (
     DigestHolder,
@@ -23,6 +26,7 @@ HitBounds = Dict[str, Tuple[bool, bool]]
 
 
 class TestcaseValidationInfo(BaseModel):
+    validator: CodeItem
     group: str
     path: pathlib.Path
     ok: bool
@@ -136,7 +140,7 @@ def _validate_testcase(
     )
 
 
-def validate_test(
+def _validate_test(
     testcase: pathlib.Path,
     validator: CodeItem,
     validator_digest: str,
@@ -160,8 +164,9 @@ def validate_one_off(
     validator: CodeItem,
     validator_digest: str,
 ) -> TestcaseValidationInfo:
-    ok, message, _ = validate_test(testcase, validator, validator_digest)
+    ok, message, _ = _validate_test(testcase, validator, validator_digest)
     info = TestcaseValidationInfo(
+        validator=validator,
         group='interactive',
         path=testcase,
         ok=ok,
@@ -172,23 +177,29 @@ def validate_one_off(
 
 
 def compile_validators(
+    validation_entries: List[GenerationTestcaseEntry],
     progress: Optional[StatusProgress] = None,
 ) -> Dict[str, str]:
-    pkg = package.find_problem_package_or_die()
+    validators = []
+
+    for entry in validation_entries:
+        if entry.validator is not None:
+            validators.append(entry.validator)
+        validators.extend(entry.extra_validators)
 
-    group_to_compiled_digest = {}
+    validator_to_compiled_digest = {}
 
-    for group in pkg.testcases:
-        validator = group.validator or pkg.validator
-        if validator is None:
+    for validator in validators:
+        if str(validator.path) in validator_to_compiled_digest:
             continue
+
         if progress:
-            progress.update(
-                f'Compiling validator for group [item]{group.name}[/item]...'
-            )
-        group_to_compiled_digest[group.name] = _compile_validator(validator)
+            progress.update(f'Compiling validator [item]{validator.path}[/item]...')
+        validator_to_compiled_digest[str(validator.path)] = _compile_validator(
+            validator
+        )
 
-    return group_to_compiled_digest
+    return validator_to_compiled_digest
 
 
 def validate_testcases(
@@ -199,38 +210,52 @@ def validate_testcases(
         if progress is not None:
             progress.step()
 
-    pkg = package.find_problem_package_or_die()
-
-    group_to_compiled_digest = compile_validators(progress)
+    validation_entries = extract_generation_testcases_from_groups(groups)
+    validator_to_compiled_digest = compile_validators(
+        validation_entries, progress=progress
+    )
 
     validation_info = []
 
-    for group in pkg.testcases:
-        validator = group.validator or pkg.validator
-        if validator is None:
-            continue
-        if group.name not in group_to_compiled_digest:
-            continue
-        if groups is not None and group.name not in groups:
+    for entry in validation_entries:
+        input_path = entry.metadata.copied_to.inputPath
+        if not input_path.is_file():
             continue
-        compiled_digest = group_to_compiled_digest[group.name]
 
-        testcases = find_built_testcase_inputs(group)
+        # Main validation.
+        if entry.validator is not None:
+            compiled_digest = validator_to_compiled_digest[str(entry.validator.path)]
+            ok, message, hit_bounds = _validate_test(
+                input_path, entry.validator, compiled_digest
+            )
+            validation_info.append(
+                TestcaseValidationInfo(
+                    validator=entry.validator,
+                    group=entry.group_entry.group,
+                    path=input_path,
+                    ok=ok,
+                    hit_bounds=hit_bounds,
+                    message=message,
+                )
+            )
 
-        for testcase in testcases:
-            ok, message, hit_bounds = validate_test(
-                testcase, validator, compiled_digest
+        for extra_validator in entry.extra_validators:
+            compiled_digest = validator_to_compiled_digest[str(extra_validator.path)]
+            ok, message, hit_bounds = _validate_test(
+                input_path, extra_validator, compiled_digest
             )
             validation_info.append(
                 TestcaseValidationInfo(
-                    group=group.name,
-                    path=testcase,
+                    validator=extra_validator,
+                    group=entry.group_entry.group,
+                    path=input_path,
                     ok=ok,
                     hit_bounds=hit_bounds,
                     message=message,
                 )
             )
-            step()
+
+        step()
 
     return validation_info
 
@@ -242,11 +267,14 @@ def has_validation_errors(infos: List[TestcaseValidationInfo]) -> bool:
 def print_validation_report(infos: List[TestcaseValidationInfo]):
     console.console.rule('Validation report', style='status')
     hit_bounds_per_group: Dict[Optional[str], HitBounds] = {}
+    any_failure = False
     for info in infos:
         if not info.ok:
             console.console.print(
-                f'[error]Testcase [item]{info.path}[/item] failed verification:[/error]\n{info.message}'
+                f'[error]Testcase [item]{info.path}[/item] failed verification on validator [item]{info.validator.path}[/item]:[/error]'
             )
+            console.console.print(info.message)
+            any_failure = True
             continue
 
         if info.group not in hit_bounds_per_group:
@@ -278,7 +306,7 @@ def print_validation_report(infos: List[TestcaseValidationInfo]):
         # If there's only the samples group, do not check for hit bounds.
         hit_bounds_per_group = {}
 
-    if not hit_bounds_per_group:
+    if not hit_bounds_per_group and not any_failure:
         console.console.print('[info]No validation issues found.[/info]')
         return