rbx.cp 0.13.8__py3-none-any.whl → 0.16.0__py3-none-any.whl

rbx/box/remote.py

@@ -8,6 +8,7 @@ import typer
 from rbx import console, utils
 from rbx.box import cd, package
 from rbx.box.formatting import href, ref
+from rbx.box.tooling.boca.scraper import BocaRun
 
 PathLike = Union[str, pathlib.Path]
 
@@ -69,11 +70,10 @@ class BocaExpander(Expander):
             return None
         run_number, site_number = match
 
+        run = BocaRun.from_run_number(run_number, site_number)
         boca_uploader = boca_upload.get_boca_scraper()
         boca_uploader.login()
-        sol_path = boca_uploader.download_run(
-            run_number, site_number, self.get_boca_folder()
-        )
+        sol_path = boca_uploader.download_run(run, self.get_boca_folder())
         console.console.print(f'Downloaded {href(sol_path)} from BOCA...')
         return sol_path
 

rbx/box/sanitizers/issue_stack.py

@@ -0,0 +1,124 @@
+import contextvars
+import enum
+from collections import OrderedDict
+from typing import Callable, Optional, Tuple, Union
+
+from rbx import console
+
+
+class IssueLevel(enum.Enum):
+    OVERVIEW = enum.auto()
+    DETAILED = enum.auto()
+
+
+class Issue:
+    def get_detailed_section(self) -> Optional[Tuple[str, ...]]:
+        return None
+
+    def get_overview_section(self) -> Optional[Tuple[str, ...]]:
+        return None
+
+    def get_detailed_message(self) -> str:
+        return ''
+
+    def get_overview_message(self) -> str:
+        return ''
+
+
+IssueSection = OrderedDict[str, Union[Issue, 'IssueSection']]
+
+
+class IssueAccumulator:
+    def __init__(self):
+        self.issues = []
+
+    def add_issue(self, issue: Issue):
+        self.issues.append(issue)
+
+    def get_sections_by(
+        self, key: Callable[[Issue], Optional[Tuple[str, ...]]]
+    ) -> IssueSection:
+        sections = OrderedDict()
+        for issue in self.issues:
+            section_key = key(issue)
+            if section_key is None:
+                continue
+            current = sections
+            for k in section_key[:-1]:
+                current = current.setdefault(k, OrderedDict())
+            current[section_key[-1]] = issue
+        return sections
+
+    def get_detailed_sections(self) -> IssueSection:
+        return self.get_sections_by(lambda issue: issue.get_detailed_section())
+
+    def get_overview_sections(self) -> IssueSection:
+        return self.get_sections_by(lambda issue: issue.get_overview_section())
+
+    def _print_report_by(
+        self,
+        section_fn: Callable[[], IssueSection],
+        message_fn: Callable[[Issue], str],
+    ):
+        from rich.tree import Tree
+
+        tree = Tree('Issues')
+        sections = section_fn()
+
+        def print_section(section: IssueSection, tree: Tree):
+            for key, value in section.items():
+                child = tree.add(key)
+                if isinstance(value, OrderedDict):
+                    print_section(value, child)
+                else:
+                    child.add(f'[error]{message_fn(value)}[/error]')
+
+        print_section(sections, tree)
+
+        if tree.children:
+            console.console.rule('Issues', style='error')
+            for child in tree.children:
+                console.console.print(child)
+
+    def print_detailed_report(self):
+        self._print_report_by(
+            self.get_detailed_sections, lambda issue: issue.get_detailed_message()
+        )
+
+    def print_overview_report(self):
+        self._print_report_by(
+            self.get_overview_sections, lambda issue: issue.get_overview_message()
+        )
+
+
+issue_stack_var = contextvars.ContextVar('issue_stack', default=[IssueAccumulator()])
+issue_level_var = contextvars.ContextVar('issue_level', default=IssueLevel.DETAILED)
+
+
+def get_issue_stack() -> list[IssueAccumulator]:
+    return issue_stack_var.get()
+
+
+def push_issue_accumulator():
+    issue_stack_var.set(get_issue_stack() + [IssueAccumulator()])
+
+
+def pop_issue_accumulator():
+    issue_stack_var.set(get_issue_stack()[:-1])
+
+
+def get_issue_accumulator() -> IssueAccumulator:
+    return get_issue_stack()[-1]
+
+
+def add_issue(issue: Issue):
+    for acc in get_issue_stack():
+        acc.add_issue(issue)
+
+
+def print_current_report():
+    acc = get_issue_accumulator()
+    if issue_level_var.get() == IssueLevel.OVERVIEW:
+        acc.print_overview_report()
+    else:
+        acc.print_detailed_report()

rbx/box/schema.py

@@ -3,13 +3,14 @@ from __future__ import annotations
 import os
 import pathlib
 import re
+import typing
 from typing import Annotated, Any, Dict, List, Optional
 
 from pydantic import AfterValidator, BaseModel, ConfigDict, Field, model_validator
 from pydantic_core import PydanticCustomError
 
 from rbx.autoenum import AutoEnum, alias
-from rbx.box.fields import NameField, Primitive, expand_vars
+from rbx.box.fields import NameField, Primitive, RecVars, Vars, expand_vars
 from rbx.box.statements.expander import expand_statements
 from rbx.box.statements.schema import Statement
 from rbx.grading.steps import Outcome
@@ -52,7 +53,7 @@ def convert_to_primitive(value: Any) -> Primitive:
 
 def expand_any_vars(vars: Dict[str, Any]) -> Dict[str, Primitive]:
     converted_vars = {key: convert_to_primitive(value) for key, value in vars.items()}
-    return expand_vars(converted_vars)
+    return expand_vars(typing.cast(RecVars, converted_vars))
 
 
 def is_unique_by_name(statements: List['Statement']) -> List['Statement']:
@@ -330,6 +331,26 @@ problems that have points.
 """,
     )
 
+    model_solution: Optional[CodeItem] = Field(
+        default=None,
+        description="""
+The solution to be used to generate outputs for this testgroup.
+
+Can only be set for the "samples" testgroup.
+""",
+    )
+
+    @model_validator(mode='after')
+    def check_model_solution_for_samples(self):
+        if self.name == 'samples':
+            return self
+        if self.model_solution is not None:
+            raise PydanticCustomError(
+                'MODEL_SOLUTION_NOT_ALLOWED',
+                'Model solution can only be set for the "samples" testgroup.',
+            )
+        return self
+
 
 class Generator(CodeItem):
     model_config = ConfigDict(extra='forbid')
@@ -419,6 +440,68 @@ class UnitTests(BaseModel):
     )
 
 
+class LimitsProfile(BaseModel):
+    model_config = ConfigDict(extra='forbid')
+
+    inheritFromPackage: bool = Field(
+        default=False,
+        description="""
+Whether to inherit limits from the package.
+""",
+    )
+
+    timeLimit: Optional[int] = Field(
+        default=None, description='Time limit of the problem, in milliseconds.'
+    )
+
+    memoryLimit: Optional[int] = Field(
+        default=None, description='Memory limit of the problem, in MB.'
+    )
+
+    outputLimit: Optional[int] = Field(
+        default=None, description='Output limit of the problem, in KB.'
+    )
+
+    modifiers: Dict[str, LimitModifiers] = Field(
+        default={},
+        description="""
+    Limit modifiers that can be specified per language.
+    """,
+    )
+
+    formula: Optional[str] = Field(
+        default=None,
+        description="""
+A formula to estimate the time limit for the problem.
+""",
+    )
+
+    def timelimit_for_language(self, language: Optional[str] = None) -> int:
+        assert self.timeLimit is not None
+        res = self.timeLimit
+        if language is not None and language in self.modifiers:
+            modifier = self.modifiers[language]
+            if modifier.time is not None:
+                res = modifier.time
+            if modifier.timeMultiplier is not None:
+                res = int(res * float(modifier.timeMultiplier))
+        if 'RBX_TIME_MULTIPLIER' in os.environ:
+            res = int(res * float(os.environ['RBX_TIME_MULTIPLIER']))
+        return res
+
+    def memorylimit_for_language(self, language: Optional[str] = None) -> int:
+        assert self.memoryLimit is not None
+        res = self.memoryLimit
+        if language is None:
+            return res
+        if language not in self.modifiers:
+            return res
+        modifier = self.modifiers[language]
+        if modifier.memory is not None:
+            return modifier.memory
+        return res
+
+
 class Package(BaseModel):
     model_config = ConfigDict(extra='forbid')
 
@@ -486,7 +569,7 @@ that is correct and used as reference -- and should have the `accepted` outcome.
     # Vars to be re-used across the package.
     #   - It will be passed as --key=value arguments to the validator.
     #   - It will be available as \VAR{key} variables in the rbx statement.
-    vars: Dict[str, Primitive] = Field(
+    vars: RecVars = Field(
         default={}, description='Variables to be re-used across the package.'
     )
 
@@ -500,32 +583,9 @@ that is correct and used as reference -- and should have the `accepted` outcome.
         return expand_statements(self.statements)
 
     @property
-    def expanded_vars(self) -> Dict[str, Primitive]:
+    def expanded_vars(self) -> Vars:
         return expand_vars(self.vars)
 
-    def timelimit_for_language(self, language: Optional[str]) -> int:
-        res = self.timeLimit
-        if language is not None and language in self.modifiers:
-            modifier = self.modifiers[language]
-            if modifier.time is not None:
-                res = modifier.time
-            if modifier.timeMultiplier is not None:
-                res = int(res * float(modifier.timeMultiplier))
-        if 'RBX_TIME_MULTIPLIER' in os.environ:
-            res = int(res * float(os.environ['RBX_TIME_MULTIPLIER']))
-        return res
-
-    def memorylimit_for_language(self, language: Optional[str]) -> int:
-        res = self.memoryLimit
-        if language is None:
-            return res
-        if language not in self.modifiers:
-            return res
-        modifier = self.modifiers[language]
-        if modifier.memory is not None:
-            return modifier.memory
-        return res
-
     @model_validator(mode='after')
     def check_first_solution_is_main_if_there_is_ac(self):
         if all(sol.outcome != Outcome.ACCEPTED for sol in self.solutions):

rbx/box/solutions.py

@@ -5,7 +5,7 @@ import dataclasses
 import pathlib
 import shutil
 from collections.abc import Iterator
-from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
+from typing import Dict, Iterable, List, Optional, Set, Tuple
 
 import rich
 import rich.live
@@ -17,7 +17,7 @@ from ordered_set import OrderedSet
 from pydantic import BaseModel
 
 from rbx import console, utils
-from rbx.box import checkers, environment, package, remote, state
+from rbx.box import checkers, code, limits_info, package, remote, state
 from rbx.box.code import (
     SanitizationLevel,
     compile_item,
@@ -34,6 +34,7 @@ from rbx.box.generators import (
     generate_output_for_testcase,
     generate_standalone,
 )
+from rbx.box.sanitizers import issue_stack
 from rbx.box.schema import (
     ExpectedOutcome,
     GeneratorCall,
@@ -92,6 +93,20 @@ class SolutionReportSkeleton(BaseModel):
     verification: VerificationLevel
     capture_pipes: bool = False
 
+    def get_solution_limits(self, solution: Solution) -> Limits:
+        lang = code.find_language_name(solution)
+        if lang is None:
+            return limits_info.get_package_limits(self.verification)
+        return self.limits[lang]
+
+    def get_solution_limits_from_disk(self, solution: Solution) -> Limits:
+        lang = code.find_language_name(solution)
+        return limits_info.get_limits(
+            language=lang,
+            profile=self.get_solution_limits(solution).profile,
+            verification=self.verification,
+        )
+
     def find_group_skeleton(self, group_name: str) -> Optional[GroupSkeleton]:
         groups = [group for group in self.groups if group.name == group_name]
         if not groups:
@@ -131,6 +146,17 @@ class RunSolutionResult:
         return self.skeleton.empty_structured_evaluation()
 
 
+class FailedSolutionIssue(issue_stack.Issue):
+    def __init__(self, solution: Solution):
+        self.solution = solution
+
+    def get_detailed_section(self) -> Tuple[str, ...]:
+        return ('solutions',)
+
+    def get_detailed_message(self) -> str:
+        return f'[item]{href(self.solution.path)}[/item] has an unexpected outcome.'
+
+
 def is_fast(solution: Solution) -> bool:
     # If solution has TLE tag, it is considered slow.
     return not solution.outcome.is_slow()
@@ -535,11 +561,13 @@ async def _generate_testcase_interactively(
         if progress:
             progress.update('Generating output for test...')
         # TODO: Add stderr path
-        await generate_output_for_testcase(
-            main_solution_digest,
-            testcase,
-            interactor_digest=interactor_digest,
-        )
+        if main_solution is not None:
+            await generate_output_for_testcase(
+                main_solution,
+                main_solution_digest,
+                testcase,
+                interactor_digest=interactor_digest,
+            )
 
     if check and testcase.outputPath is not None and not testcase.outputPath.is_file():
         # Output was not created, throw an error.
@@ -882,10 +910,10 @@ def get_evals_formatted_time(evals: List[Evaluation]) -> str:
 
 
 def get_capped_evals_formatted_time(
-    solution: Solution, evals: List[Evaluation], verification: VerificationLevel
+    limits: Limits,
+    evals: List[Evaluation],
+    verification: VerificationLevel,
 ) -> str:
-    pkg = package.find_problem_package_or_die()
-
     max_time = _get_evals_time_in_ms(evals)
     has_tle = any(eval.result.outcome.is_slow() for eval in evals)
     has_ile = any(
@@ -902,13 +930,13 @@ def get_capped_evals_formatted_time(
     if timelimits:
         tl = min(timelimits)
     if tl is None:
-        tl = pkg.timelimit_for_language(solution.language)
+        tl = limits.time
 
-        if verification.value >= VerificationLevel.FULL.value:
+        if tl is not None and verification.value >= VerificationLevel.FULL.value:
             # Using double TL for verification.
             tl = tl * 2
 
-    if has_tle and max_time >= tl or has_ile:
+    if tl is not None and has_tle and max_time >= tl or has_ile:
         return f'>{tl} ms'
     return f'{max_time} ms'
 
@@ -930,6 +958,7 @@ def get_truncated_message(message: str, max_length: int = 100) -> str:
 
 class SolutionOutcomeReport(BaseModel):
     solution: Solution
+    limits: Limits
     evals: List[Evaluation]
     ok: bool
     message: Optional[Tuple[TestcaseEntry, str]]
@@ -972,7 +1001,7 @@ class SolutionOutcomeReport(BaseModel):
 
     def get_outcome_markup(self, print_message: bool = True) -> str:
         res = self.get_verdict_markup_with_warnings()
-        res += f'\nTime: {get_capped_evals_formatted_time(self.solution, self.evals, self.verification)}'
+        res += f'\nTime: {get_capped_evals_formatted_time(self.limits, self.evals, self.verification)}'
         res += f'\nMemory: {get_evals_formatted_memory(self.evals)}'
         if print_message and self.message is not None:
             tc, msg = self.message
@@ -989,8 +1018,6 @@ def get_solution_outcome_report(
     verification: VerificationLevel = VerificationLevel.NONE,
     subset: bool = False,
 ) -> SolutionOutcomeReport:
-    pkg = package.find_problem_package_or_die()
-
     has_plain_tle = False
     all_verdicts = set()
     bad_verdicts = set()
@@ -1051,6 +1078,7 @@ def get_solution_outcome_report(
 
     evals_time = _get_evals_time_in_ms(evals)
     expected_outcome_is_tle = solution.outcome.matches_tle_and_is_incorrect()
+    limits = skeleton.get_solution_limits(solution)
     if (
         # Running verification with double TL.
         verification.value >= VerificationLevel.FULL.value
@@ -1061,7 +1089,8 @@ def get_solution_outcome_report(
         # A TLE has happened.
         and Outcome.TIME_LIMIT_EXCEEDED in matched_bad_verdicts
         # The solution runs in double TL.
-        and evals_time < pkg.timelimit_for_language(solution.language) * 2
+        and limits.time is not None
+        and evals_time < limits.time * 2
     ):
         other_verdicts = (bad_verdicts | no_tle_bad_verdicts) - {
             Outcome.TIME_LIMIT_EXCEEDED
@@ -1078,6 +1107,7 @@ def get_solution_outcome_report(
 
     return SolutionOutcomeReport(
         solution=solution,
+        limits=skeleton.get_solution_limits(solution),
         evals=evals,
         ok=not has_failed,
         message=message,
@@ -1102,11 +1132,13 @@ def _print_solution_outcome(
     report = get_solution_outcome_report(
         solution, skeleton, evals, verification, subset
     )
+    if not report.ok:
+        issue_stack.add_issue(FailedSolutionIssue(solution))
     console.print(report.get_outcome_markup(print_message))
     return report.ok
 
 
-def _consume_and_key_evaluation_items(
+def consume_and_key_evaluation_items(
     items: Iterable[EvaluationItem],
     skeleton: SolutionReportSkeleton,
 ) -> StructuredEvaluation:
@@ -1169,9 +1201,7 @@ async def _print_timing(
     console: rich.console.Console,
     skeleton: SolutionReportSkeleton,
     evaluations: StructuredEvaluation,
-    verification: VerificationLevel,
 ):
-    pkg = package.find_problem_package_or_die()
     summary = TimingSummary()
     summary_per_language = collections.defaultdict(TimingSummary)
     tls_per_language = {}
@@ -1196,8 +1226,12 @@ async def _print_timing(
         if solution_tls:
             solution_tl = min(solution_tls)
         else:
-            solution_tl = pkg.timelimit_for_language(solution.language)
-            if verification.value >= VerificationLevel.FULL.value:
+            limits = skeleton.get_solution_limits(solution)
+            if limits.time is None:
+                limits = skeleton.get_solution_limits_from_disk(solution)
+            assert limits.time is not None
+            solution_tl = limits.time
+            if limits.isDoubleTL:
                 solution_tl = solution_tl * 2
         all_tls.add(solution_tl)
         for eval in all_evals:
@@ -1306,7 +1340,8 @@ async def _render_detailed_group_table(
                 evals_per_solution[str(solution.path)].append(eval)
 
                 verdict = get_testcase_markup_verdict(eval)
-                time = get_capped_evals_formatted_time(solution, [eval], verification)
+                limits = skeleton.get_solution_limits(solution)
+                time = get_capped_evals_formatted_time(limits, [eval], verification)
                 memory = get_evals_formatted_memory([eval])
                 full_item = (f'[info]#{tc}[/info]', verdict, time, '/', memory, '')
                 if eval.result.sanitizer_warnings:
@@ -1323,8 +1358,9 @@ async def _render_detailed_group_table(
                 if not non_null_evals:
                     summary_row.append('...')
                     continue
+                limits = skeleton.get_solution_limits(solution)
                 formatted_time = get_capped_evals_formatted_time(
-                    solution, non_null_evals, verification
+                    limits, non_null_evals, verification
                 )
                 formatted_memory = get_evals_formatted_memory(non_null_evals)
                 worst_outcome = get_worst_outcome(non_null_evals)
@@ -1398,7 +1434,9 @@ async def _print_detailed_run_report(
 
     if timing:
         await _print_timing(
-            console, result.skeleton, structured_evaluations, verification=verification
+            console,
+            result.skeleton,
+            structured_evaluations,
         )
     return ok
 
@@ -1408,7 +1446,10 @@ def _print_limits(limits: Dict[str, Limits]):
         '[bold][success]Running with the following limits (per language):[/success][/bold]'
     )
     for lang, limit in limits.items():
-        console.console.print(f'[bold][status]{lang}[/status][/bold]')
+        extracted_from = ' (extracted from package)'
+        if limit.profile:
+            extracted_from = f' (extracted from profile [item]{limit.profile}[/item])'
+        console.console.print(f'[bold][status]{lang}[/status][/bold]{extracted_from}')
         time = (
             '<No time limit>' if limit.time is None else get_formatted_time(limit.time)
         )
@@ -1435,7 +1476,7 @@ async def print_run_report(
     if not skip_printing_limits:
         _print_limits(result.skeleton.limits)
 
-    structured_evaluations = _consume_and_key_evaluation_items(
+    structured_evaluations = consume_and_key_evaluation_items(
         result.items, result.skeleton
     )
     if detailed:
@@ -1455,6 +1496,7 @@ async def print_run_report(
         if single_solution:
             console.print()
         solution_evals = []
+        limits = result.skeleton.get_solution_limits(solution)
         for group in result.skeleton.groups:
             if not single_solution:
                 console.print(f'[bold][status]{group.name}[/status][/bold] ', end='')
@@ -1469,9 +1511,7 @@ async def print_run_report(
                     console.print(f'{group.name}/{i}', end='')
                     if eval.result.sanitizer_warnings:
                         console.print('[warning]*[/warning]', end='')
-                    time = get_capped_evals_formatted_time(
-                        solution, [eval], verification
-                    )
+                    time = get_capped_evals_formatted_time(limits, [eval], verification)
                     memory = get_evals_formatted_memory([eval])
                     console.print(f' ({time}, {memory})', end='')
                     checker_msg = eval.result.message
@@ -1493,7 +1533,7 @@ async def print_run_report(
             if single_solution:
                 console.print(f'  [status]{group.name}[/status]', end=' ')
             console.print(
-                f'({get_capped_evals_formatted_time(solution, group_evals, verification)}, {get_evals_formatted_memory(group_evals)})',
+                f'({get_capped_evals_formatted_time(limits, group_evals, verification)}, {get_evals_formatted_memory(group_evals)})',
                 end='',
             )
             console.print()
@@ -1513,92 +1553,9 @@ async def print_run_report(
 
     if not single_solution:
         await _print_timing(
-            console, result.skeleton, structured_evaluations, verification=verification
+            console,
+            result.skeleton,
+            structured_evaluations,
         )
 
     return ok
-
-
-def _step_up(x: Any, step: int) -> int:
-    x = int(x)
-    return (x + step - 1) // step * step
-
-
-def _step_down(x: Any, step: int) -> int:
-    x = int(x)
-    return x // step * step
-
-
-async def estimate_time_limit(
-    console: rich.console.Console,
-    result: RunSolutionResult,
-) -> Optional[int]:
-    structured_evaluations = _consume_and_key_evaluation_items(
-        result.items, result.skeleton
-    )
-
-    timing_per_solution = {}
-    timing_per_language = {}
-
-    if not result.skeleton.solutions:
-        console.print('[error]No solutions to estimate time limit from.[/error]')
-        return None
-
-    for solution in result.skeleton.solutions:
-        timings = []
-        for evals in structured_evaluations[str(solution.path)].values():
-            for ev in evals:
-                if ev is None:
-                    continue
-                ev = await ev()
-                if ev.log.time is not None:
-                    timings.append(int(ev.log.time * 1000))
-
-        if not timings:
-            console.print(
-                f'[warning]No timings for solution {href(solution.path)}.[/warning]'
-            )
-            continue
-
-        timing_per_solution[str(solution.path)] = max(timings)
-        lang = find_language_name(solution)
-        if lang not in timing_per_language:
-            timing_per_language[lang] = 0
-        timing_per_language[lang] = max(timing_per_language[lang], max(timings))
-
-    console.rule('[status]Time estimation[/status]', style='status')
-
-    fastest_time = min(timing_per_solution.values())
-    slowest_time = max(timing_per_solution.values())
-
-    console.print(f'Fastest solution: {fastest_time} ms')
-    console.print(f'Slowest solution: {slowest_time} ms')
-
-    if len(timing_per_language) > 0:
-        timing_language_list = [(t, lang) for lang, t in timing_per_language.items()]
-        fastest_language_time, fastest_language = min(timing_language_list)
-        slowest_language_time, slowest_language = max(timing_language_list)
-
-        console.print(
-            f'Fastest language: {fastest_language} ({fastest_language_time} ms)'
-        )
-        console.print(
-            f'Slowest language: {slowest_language} ({slowest_language_time} ms)'
-        )
-
-    env = environment.get_environment()
-    estimated_tl = int(
-        eval(
-            env.timing.formula,
-            {
-                'fastest': fastest_time,
-                'slowest': slowest_time,
-                'step_up': _step_up,
-                'step_down': _step_down,
-            },
-        )
-    )
-
-    console.print(f'Using formula: {env.timing.formula}')
-    console.print(f'[success]Estimated time limit:[/success] {estimated_tl} ms')
-    return estimated_tl