rbx.cp 0.5.61__py3-none-any.whl → 0.5.63__py3-none-any.whl

rbx/box/solutions.py

@@ -16,7 +16,7 @@ import typer
 from pydantic import BaseModel
 
 from rbx import console, utils
-from rbx.box import checkers, environment, package
+from rbx.box import checkers, environment, package, state
 from rbx.box.code import (
     SanitizationLevel,
     compile_item,
@@ -64,9 +64,8 @@ StructuredEvaluation = Dict[str, Dict[str, List[Optional[Deferred[Evaluation]]]]
 
 @dataclasses.dataclass(frozen=True)
 class EvaluationItem:
-    solution_index: int
-    group_name: str
-    testcase_index: int
+    solution: Solution
+    testcase_entry: TestcaseEntry
     eval: Deferred[Evaluation]
 
 
@@ -75,11 +74,20 @@ class GroupSkeleton(BaseModel):
     testcases: List[Testcase]
 
 
+class SolutionSkeleton(Solution):
+    runs_dir: pathlib.Path
+
+    def get_entry_prefix(self, entry: TestcaseEntry) -> pathlib.Path:
+        return self.runs_dir / entry.group / f'{entry.index:03d}'
+
+
 class SolutionReportSkeleton(BaseModel):
-    solutions: List[Solution]
+    solutions: List[SolutionSkeleton]
     entries: List[TestcaseEntry]
     groups: List[GroupSkeleton]
     limits: Dict[str, Limits]
+    verification: VerificationLevel
+    capture_pipes: bool = False
 
     def find_group_skeleton(self, group_name: str) -> Optional[GroupSkeleton]:
         groups = [group for group in self.groups if group.name == group_name]
@@ -87,6 +95,21 @@ class SolutionReportSkeleton(BaseModel):
             return None
         return groups[0]
 
+    def find_solution_skeleton(self, solution: Solution) -> Optional[SolutionSkeleton]:
+        for sol in self.solutions:
+            if sol.path == solution.path:
+                return sol
+        return None
+
+    def find_solution_skeleton_index(self, solution: Solution) -> Optional[int]:
+        for i, sol in enumerate(self.solutions):
+            if sol.path == solution.path:
+                return i
+        return None
+
+    def get_solution_path_set(self) -> Set[str]:
+        return set(str(sol.path) for sol in self.solutions)
+
     def empty_structured_evaluation(self) -> StructuredEvaluation:
         res: StructuredEvaluation = {}
         for solution in self.solutions:
@@ -164,21 +187,20 @@ def _run_solution(
     solution: Solution,
     compiled_digest: str,
     checker_digest: Optional[str],
-    solution_index: int,
+    runs_dir: pathlib.Path,
     group_name: str,
     interactor_digest: Optional[str] = None,
     progress: Optional[StatusProgress] = None,
     verification: VerificationLevel = VerificationLevel.NONE,
     timelimit_override: Optional[int] = None,
 ) -> List[Deferred[Evaluation]]:
-    runs_dir = package.get_problem_runs_dir()
-
     group = package.get_testgroup(group_name)
     testcases = find_built_testcases(group)
     res: List[Deferred[Evaluation]] = []
     for i, testcase in enumerate(testcases):
         assert testcase.outputPath is not None
-        output_path = runs_dir / f'{solution_index}' / group.name
+        output_path = runs_dir / group.name
+        output_path.mkdir(parents=True, exist_ok=True)
 
         if progress:
             progress.update(
@@ -204,6 +226,7 @@ def _run_solution(
 
 
 async def convert_list_of_solution_evaluations_to_dict(
+    skeleton: SolutionReportSkeleton,
     items: Iterable[EvaluationItem],
 ) -> List[Dict[str, List[Evaluation]]]:
     pkg = package.find_problem_package_or_die()
@@ -212,16 +235,18 @@ async def convert_list_of_solution_evaluations_to_dict(
     ]
 
     for item in items:
-        res[item.solution_index][item.group_name].append(await item.eval())
+        sol_idx = skeleton.find_solution_skeleton_index(item.solution)
+        if sol_idx is not None:
+            to_append = await item.eval()
+            res[sol_idx][item.testcase_entry.group].append(to_append)
 
     return res
 
 
-def _get_report_skeleton(
+def _get_solutions_for_skeleton(
     tracked_solutions: Optional[Set[str]] = None,
     verification: VerificationLevel = VerificationLevel.NONE,
-    timelimit_override: Optional[int] = None,
-) -> SolutionReportSkeleton:
+) -> List[Solution]:
     pkg = package.find_problem_package_or_die()
     solutions = [
         sol
@@ -234,6 +259,16 @@ def _get_report_skeleton(
             for solution in solutions
             if str(solution.path) in tracked_solutions
         ]
+    return solutions
+
+
+def _get_report_skeleton(
+    tracked_solutions: Optional[Set[str]] = None,
+    verification: VerificationLevel = VerificationLevel.NONE,
+    timelimit_override: Optional[int] = None,
+) -> SolutionReportSkeleton:
+    pkg = package.find_problem_package_or_die()
+    solutions = _get_solutions_for_skeleton(tracked_solutions, verification)
 
     langs = set(find_language_name(solution) for solution in solutions)
     limits = {
@@ -251,17 +286,36 @@ def _get_report_skeleton(
         for group in groups
         for i in range(len(group.testcases))
     ]
-    return SolutionReportSkeleton(
-        solutions=solutions,
+
+    # Prepare directory.
+    runs_dir = package.get_problem_runs_dir()
+    shutil.rmtree(str(runs_dir), ignore_errors=True)
+    runs_dir.mkdir(parents=True, exist_ok=True)
+
+    skeleton = SolutionReportSkeleton(
+        solutions=[
+            SolutionSkeleton(
+                **solution.model_dump(),
+                runs_dir=package.get_problem_runs_dir() / f'{i}',
+            )
+            for i, solution in enumerate(solutions)
+        ],
         groups=groups,
         limits=limits,
         entries=entries,
+        verification=verification,
+        capture_pipes=state.STATE.debug_logs,
     )
 
+    skeleton_file = runs_dir / 'skeleton.yml'
+    skeleton_file.write_text(utils.model_to_yaml(skeleton))
+
+    return skeleton
+
 
 def _produce_solution_items(
+    skeleton: SolutionReportSkeleton,
     progress: Optional[StatusProgress] = None,
-    tracked_solutions: Optional[Set[str]] = None,
     verification: VerificationLevel = VerificationLevel.NONE,
     check: bool = True,
     timelimit_override: Optional[int] = None,
@@ -279,26 +333,13 @@ def _produce_solution_items(
         interactor_digest = None
 
     compiled_solutions = compile_solutions(
-        progress=progress, tracked_solutions=tracked_solutions, sanitized=sanitized
-    )
-
-    # Clear run directory and rely on cache to
-    # repopulate it.
-    runs_dir = package.get_problem_runs_dir()
-    shutil.rmtree(str(runs_dir), ignore_errors=True)
-    runs_dir.mkdir(parents=True, exist_ok=True)
-    solutions = list(
-        (i, sol)
-        for i, sol in enumerate(pkg.solutions)
-        if verification.value >= VerificationLevel.ALL_SOLUTIONS.value or is_fast(sol)
+        progress=progress,
+        tracked_solutions=skeleton.get_solution_path_set(),
+        sanitized=sanitized,
     )
-    if tracked_solutions is not None:
-        solutions = [
-            (i, sol) for i, sol in solutions if str(sol.path) in tracked_solutions
-        ]
 
     def yield_items(
-        solution_index: int, solution: Solution, group_name: str
+        solution: SolutionSkeleton, group_name: str
     ) -> List[EvaluationItem]:
         res: List[EvaluationItem] = []
         for i, eval in enumerate(
@@ -306,7 +347,7 @@ def _produce_solution_items(
                 solution,
                 compiled_solutions[solution.path],
                 checker_digest,
-                solution_index,
+                solution.runs_dir,
                 group_name,
                 interactor_digest=interactor_digest,
                 progress=progress,
@@ -316,9 +357,8 @@ def _produce_solution_items(
         ):
             res.append(
                 EvaluationItem(
-                    solution_index=solution_index,
-                    group_name=group_name,
-                    testcase_index=i,
+                    solution=solution,
+                    testcase_entry=TestcaseEntry(group=group_name, index=i),
                     eval=eval,
                 )
             )
@@ -328,9 +368,9 @@ def _produce_solution_items(
     res: List[EvaluationItem] = []
 
     groups = pkg.testcases
-    for i, solution in solutions:
+    for solution in skeleton.solutions:
         for group in groups:
-            res.extend(yield_items(i, solution, group.name))
+            res.extend(yield_items(solution, group.name))
 
     return res
 
@@ -356,24 +396,22 @@ def run_solutions(
     timelimit_override: Optional[int] = None,
     sanitized: bool = False,
 ) -> RunSolutionResult:
+    skeleton = _get_report_skeleton(
+        tracked_solutions,
+        verification=verification,
+        timelimit_override=timelimit_override,
+    )
     result = RunSolutionResult(
-        skeleton=_get_report_skeleton(
-            tracked_solutions,
-            verification=verification,
-            timelimit_override=timelimit_override,
-        ),
+        skeleton=skeleton,
         items=_produce_solution_items(
+            skeleton=skeleton,
             progress=progress,
-            tracked_solutions=tracked_solutions,
             verification=verification,
             check=check,
             timelimit_override=timelimit_override,
             sanitized=sanitized,
         ),
     )
-    skeleton_file = package.get_problem_runs_dir() / 'skeleton.yml'
-    skeleton_file.parent.mkdir(parents=True, exist_ok=True)
-    skeleton_file.write_text(utils.model_to_yaml(result.skeleton))
     return result
 
 
@@ -515,8 +553,8 @@ async def _generate_testcase_interactively(
 
 def _run_interactive_solutions(
     testcase: Testcase,
+    skeleton: SolutionReportSkeleton,
     progress: Optional[StatusProgress] = None,
-    tracked_solutions: Optional[Set[str]] = None,
     verification: VerificationLevel = VerificationLevel.NONE,
     check: bool = True,
     sanitized: bool = False,
@@ -531,22 +569,16 @@ def _run_interactive_solutions(
         interactor_digest = None
 
     compiled_solutions = compile_solutions(
-        progress=progress, tracked_solutions=tracked_solutions, sanitized=sanitized
+        progress=progress,
+        tracked_solutions=skeleton.get_solution_path_set(),
+        sanitized=sanitized,
     )
 
-    solutions = list(enumerate(pkg.solutions))
-    if tracked_solutions is not None:
-        solutions = [
-            (i, sol) for i, sol in solutions if str(sol.path) in tracked_solutions
-        ]
-
-    irun_dir = package.get_problem_iruns_dir()
-
     if progress:
         progress.update('Running solutions...')
 
-    for i, solution in solutions:
-        output_dir = irun_dir / f'{i}'
+    for solution in skeleton.solutions:
+        output_dir = solution.runs_dir
 
         async def run_fn(solution=solution, output_dir=output_dir):
             return await run_solution_on_testcase(
@@ -561,13 +593,51 @@ def _run_interactive_solutions(
             )
 
         yield EvaluationItem(
-            solution_index=i,
-            group_name='irun',
-            testcase_index=0,
+            solution=solution,
+            testcase_entry=TestcaseEntry(group='irun', index=0),
             eval=Deferred(run_fn),
         )
 
 
+def _get_interactive_skeleton(
+    tracked_solutions: Optional[Set[str]] = None,
+    verification: VerificationLevel = VerificationLevel.NONE,
+) -> SolutionReportSkeleton:
+    solutions = _get_solutions_for_skeleton(tracked_solutions, verification)
+
+    langs = set(find_language_name(solution) for solution in solutions)
+    limits = {
+        lang: get_limits_for_language(lang, verification, timelimit_override=None)
+        for lang in langs
+        if lang is not None
+    }
+
+    # Ensure path is new.
+    irun_dir = package.get_problem_iruns_dir()
+    shutil.rmtree(str(irun_dir), ignore_errors=True)
+    irun_dir.mkdir(parents=True, exist_ok=True)
+
+    skeleton = SolutionReportSkeleton(
+        solutions=[
+            SolutionSkeleton(
+                **solution.model_dump(),
+                runs_dir=irun_dir / f'{i}',
+            )
+            for i, solution in enumerate(solutions)
+        ],
+        groups=[],
+        limits=limits,
+        entries=[],
+        verification=verification,
+        capture_pipes=True,
+    )
+
+    skeleton_file = irun_dir / 'skeleton.yml'
+    skeleton_file.write_text(utils.model_to_yaml(skeleton))
+
+    return skeleton
+
+
 async def run_and_print_interactive_solutions(
     progress: Optional[StatusProgress] = None,
     tracked_solutions: Optional[Set[str]] = None,
@@ -579,12 +649,11 @@ async def run_and_print_interactive_solutions(
     print: bool = False,
     sanitized: bool = False,
 ):
-    # Ensure path is new.
-    irun_dir = package.get_problem_iruns_dir()
-    shutil.rmtree(str(irun_dir), ignore_errors=True)
-    irun_dir.mkdir(parents=True, exist_ok=True)
-
     pkg = package.find_problem_package_or_die()
+    skeleton = _get_interactive_skeleton(
+        tracked_solutions,
+        verification=verification,
+    )
     testcase = await _generate_testcase_interactively(
         progress=progress,
         generator=generator,
@@ -596,15 +665,16 @@ async def run_and_print_interactive_solutions(
     )
     items = _run_interactive_solutions(
         testcase,
+        skeleton=skeleton,
         progress=progress,
-        tracked_solutions=tracked_solutions,
         verification=verification,
         check=check,
         sanitized=sanitized,
     )
 
     for item in items:
-        sol = pkg.solutions[item.solution_index]
+        sol = skeleton.find_solution_skeleton(item.solution)
+        assert sol is not None
 
         if progress:
             progress.update(f'Running [item]{sol.path}[/item]...')
@@ -613,7 +683,7 @@ async def run_and_print_interactive_solutions(
 
         with utils.no_progress(progress):
             console.console.print(get_testcase_markup_verdict(eval), end=' ')
-            _print_solution_header(sol, console.console, is_irun=True)
+            _print_solution_header(sol, console.console)
             _print_solution_outcome(
                 sol, [eval], console.console, verification, subset=True
             )
@@ -703,6 +773,12 @@ def get_outcome_markup_verdict(outcome: Outcome) -> str:
     return res
 
 
+def get_full_outcome_markup_verdict(outcome: Outcome) -> str:
+    style = get_outcome_style_verdict(outcome)
+    res = f'[{style}]{outcome.name}[/{style}]'
+    return res
+
+
 def get_testcase_markup_verdict(eval: Evaluation) -> str:
     # if eval.log.stdout_absolute_path:
     #     output_path = eval.log.stdout_absolute_path.resolve()
@@ -711,6 +787,10 @@ def get_testcase_markup_verdict(eval: Evaluation) -> str:
     return get_outcome_markup_verdict(eval.result.outcome)
 
 
+def get_full_testcase_markup_verdict(eval: Evaluation) -> str:
+    return get_full_outcome_markup_verdict(eval.result.outcome)
+
+
 def _get_evals_time_in_ms(evals: List[Evaluation]) -> int:
     if not evals:
         return 0
@@ -766,13 +846,60 @@ def get_worst_outcome(evals: List[Evaluation]) -> Outcome:
     return Outcome.worst_outcome(eval.result.outcome for eval in evals)
 
 
-def _print_solution_outcome(
+class SolutionOutcomeReport(BaseModel):
+    solution: Solution
+    evals: List[Evaluation]
+    ok: bool
+    expectedOutcome: Optional[ExpectedOutcome]
+    gotVerdicts: Set[Outcome]
+    runUnderDoubleTl: bool
+    doubleTlVerdicts: Set[Outcome]
+    sanitizerWarnings: bool
+    verification: VerificationLevel
+
+    def get_verdict_markup(self, incomplete: bool = False) -> str:
+        success_str = '[bold green]OK[/bold green]'
+        if not self.ok:
+            success_str = '[bold red]FAILED[/bold red]'
+        if incomplete:
+            success_str = '[bold yellow]INCOMPLETE[/bold yellow]'
+
+        gotVerdicts = self.gotVerdicts if not incomplete else {}
+
+        got_verdict_names = ' '.join(v.name for v in self.gotVerdicts)
+        verdict_str = ''
+        if self.expectedOutcome is not None:
+            verdict_str = f'Expected: {self.expectedOutcome}'
+            if gotVerdicts:
+                verdict_str += f', got: {got_verdict_names}'
+        elif gotVerdicts:
+            verdict_str = f'Got: {got_verdict_names}'
+        return f'{success_str} {verdict_str}'
+
+    def get_verdict_markup_with_warnings(self) -> str:
+        res = self.get_verdict_markup()
+        if self.runUnderDoubleTl:
+            if self.doubleTlVerdicts:
+                res += f'\n[bold yellow]WARNING[/bold yellow] The solution still passed in double TL, but failed with [item]{" ".join(v.name for v in self.doubleTlVerdicts)}[/item].'
+            else:
+                res += '\n[bold yellow]WARNING[/bold yellow] The solution still passed in double TL.'
+        if self.sanitizerWarnings:
+            res += '\n[bold yellow]WARNING[/bold yellow] The solution had sanitizer errors or warnings, marked with [bold yellow]*[/bold yellow]. See their stderr for more details.'
+        return res
+
+    def get_outcome_markup(self) -> str:
+        res = self.get_verdict_markup_with_warnings()
+        res += f'\nTime: {get_capped_evals_formatted_time(self.solution, self.evals, self.verification)}'
+        res += f'\nMemory: {get_evals_formatted_memory(self.evals)}'
+        return res
+
+
+def get_solution_outcome_report(
     solution: Solution,
     evals: List[Evaluation],
-    console: rich.console.Console,
     verification: VerificationLevel = VerificationLevel.NONE,
     subset: bool = False,
-) -> bool:
+) -> SolutionOutcomeReport:
     pkg = package.find_problem_package_or_die()
 
     has_plain_tle = False
@@ -805,26 +932,24 @@ def _print_solution_outcome(
     has_failed = unmatched_bad_verdicts or (
         expected_outcome_is_bad and not matched_bad_verdicts and not subset
     )
-    if has_failed:
-        console.print('[error]FAILED[/error]', end=' ')
-    else:
-        console.print('[success]OK[/success]', end=' ')
 
+    report_expected_outcome = None
+    report_got_verdicts = set()
+    report_run_under_double_tl = False
+    report_double_tl_verdicts = set()
+    report_sanitizer_warnings = False
     if has_failed or not subset:
-        console.print(f'Expected: {solution.outcome}', end='')
+        report_expected_outcome = solution.outcome
     elif subset:
-        all_verdicts_names = ' '.join(v.name for v in all_verdicts)
-        console.print(f'Got: {all_verdicts_names}', end='')
+        report_got_verdicts = all_verdicts
 
     if has_failed or not subset:
         # Only print verdicts if not subset.
         if unmatched_bad_verdicts:
-            unmatched_bad_verdicts_names = set(v.name for v in unmatched_bad_verdicts)
-            console.print(f', got: {" ".join(unmatched_bad_verdicts_names)}', end='')
+            report_got_verdicts = unmatched_bad_verdicts
         elif expected_outcome_is_bad and not matched_bad_verdicts and not subset:
-            console.print(f', got: {Outcome.ACCEPTED.name}', end='')
+            report_got_verdicts = {Outcome.ACCEPTED}
 
-    console.print()
     evals_time = _get_evals_time_in_ms(evals)
     expected_outcome_is_tle = solution.outcome.match(
         Outcome.TIME_LIMIT_EXCEEDED
@@ -846,26 +971,37 @@ def _print_solution_outcome(
         }
         if not other_verdicts:
             # The solution has no other bad verdicts except for TLEs in double TL.
-            console.print(
-                '[yellow]WARNING[/yellow] The solution still passed in double TL.'
-            )
+            report_run_under_double_tl = True
         elif not (bad_verdicts - {Outcome.TIME_LIMIT_EXCEEDED}):
             # The solution has other bad soft TLE outcomes.
-            other_verdicts_names = ' '.join(v.name for v in other_verdicts)
-            console.print(
-                f'[yellow]WARNING[/yellow] The solution could still run under double TL, but failed with [item]{other_verdicts_names}[/item].'
-            )
+            report_double_tl_verdicts = other_verdicts
 
     if has_sanitizer_warnings:
-        console.print(
-            '[warning]WARNING[/warning] The solution had sanitizer errors or warnings, marked with [warning]*[/warning]. See their stderr for more details.'
-        )
-
-    console.print(
-        f'Time: {get_capped_evals_formatted_time(solution, evals, verification)}'
+        report_sanitizer_warnings = True
+
+    return SolutionOutcomeReport(
+        solution=solution,
+        evals=evals,
+        ok=not has_failed,
+        expectedOutcome=report_expected_outcome,
+        gotVerdicts=report_got_verdicts,
+        runUnderDoubleTl=report_run_under_double_tl,
+        doubleTlVerdicts=report_double_tl_verdicts,
+        sanitizerWarnings=report_sanitizer_warnings,
+        verification=verification,
     )
-    console.print(f'Memory: {get_evals_formatted_memory(evals)}')
-    return len(unmatched_bad_verdicts) == 0
+
+
+def _print_solution_outcome(
+    solution: Solution,
+    evals: List[Evaluation],
+    console: rich.console.Console,
+    verification: VerificationLevel = VerificationLevel.NONE,
+    subset: bool = False,
+) -> bool:
+    report = get_solution_outcome_report(solution, evals, verification, subset)
+    console.print(report.get_outcome_markup())
+    return report.ok
 
 
 def _consume_and_key_evaluation_items(
@@ -876,30 +1012,22 @@ def _consume_and_key_evaluation_items(
     Consumes EvaluationItems from a run_solutions call and build a view
     with them, possibly marking with optional unprocessed items.
     """
-    pkg = package.find_problem_package_or_die()
     res = skeleton.empty_structured_evaluation()
 
     for item in items:
-        solution = pkg.solutions[item.solution_index]
-        res[str(solution.path)][item.group_name][item.testcase_index] = item.eval
+        res[str(item.solution.path)][item.testcase_entry.group][
+            item.testcase_entry.index
+        ] = item.eval
 
     return res
 
 
 def _print_solution_header(
-    solution: Solution, console: rich.console.Console, is_irun: bool = False
+    solution: SolutionSkeleton,
+    console: rich.console.Console,
 ):
-    solutions = package.get_solutions()
-    solution_index = [
-        i for i, sol in enumerate(solutions) if sol.path == solution.path
-    ][0]
-    solution_testdir = (
-        package.get_problem_iruns_dir() / f'{solution_index}'
-        if is_irun
-        else package.get_problem_runs_dir() / f'{solution_index}'
-    )
     console.print(f'[item]{solution.path}[/item]', end=' ')
-    console.print(f'({solution_testdir})')
+    console.print(f'({solution.runs_dir})')
 
 
 @dataclasses.dataclass

rbx/box/solutions_test.py

@@ -24,7 +24,9 @@ async def test_solutions(pkg_from_testdata: pathlib.Path):
     await generate_outputs_for_testcases(entries)
 
     result = run_solutions(verification=VerificationLevel.FULL)
-    res = await convert_list_of_solution_evaluations_to_dict(result.items)
+    res = await convert_list_of_solution_evaluations_to_dict(
+        result.skeleton, result.items
+    )
 
     # First solution should pass all tests.
     assert all(chk.result.outcome == Outcome.ACCEPTED for chk in res[0]['gen1'])

rbx/box/tasks.py

@@ -86,6 +86,7 @@ async def run_solution_on_testcase(
             output_path = output_dir / testcase.inputPath.with_suffix('.out').name
         error_path = output_path.with_suffix('.err')
         log_path = output_path.with_suffix('.log')
+        eval_path = output_path.with_suffix('.eval')
         output_path.parent.mkdir(parents=True, exist_ok=True)
 
         run_log = await run_item(
@@ -120,10 +121,12 @@ async def run_solution_on_testcase(
                 stdout_absolute_path=output_path.absolute(),
                 stderr_absolute_path=error_path.absolute(),
                 log_absolute_path=log_path.absolute(),
+                eval_absolute_path=eval_path.absolute(),
             ),
         )
 
         log_path.write_text(model_to_yaml(eval))
+        eval_path.write_text(model_to_yaml(eval))
         return eval
 
     if not use_retries:
@@ -198,6 +201,7 @@ async def _run_communication_solution_on_testcase(
         solution_error_path = output_path.with_suffix('.sol.err')
         interactor_error_path = output_path.with_suffix('.int.err')
         log_path = output_path.with_suffix('.log')
+        eval_path = output_path.with_suffix('.eval')
         output_path.parent.mkdir(parents=True, exist_ok=True)
 
         interactor_capture_path = (
@@ -268,11 +272,12 @@ async def _run_communication_solution_on_testcase(
                 stdout_absolute_path=output_path.absolute(),
                 stderr_absolute_path=solution_error_path.absolute(),
                 log_absolute_path=log_path.absolute(),
+                eval_absolute_path=eval_path.absolute(),
             ),
         )
 
         log_path.write_text(model_to_yaml(eval))
-
+        eval_path.write_text(model_to_yaml(eval))
         interactor_log_path = output_path.with_suffix('.int.log')
         interactor_log_path.unlink(missing_ok=True)
         if interactor_run_log is not None:

rbx/box/testcase_utils.py

@@ -33,6 +33,9 @@ class TestcaseEntry(BaseModel):
         group, index = spec.split('/')
         return TestcaseEntry(group=group.strip(), index=int(index))
 
+    def get_prefix_path(self) -> pathlib.Path:
+        return package.get_build_testgroup_path(self.group) / f'{self.index:03d}'
+
 
 class TestcasePattern(BaseModel):
     group_prefix: List[str]