rbx.cp 0.13.8__py3-none-any.whl → 0.14.0__py3-none-any.whl

rbx/box/schema.py

@@ -3,13 +3,14 @@ from __future__ import annotations
 import os
 import pathlib
 import re
+import typing
 from typing import Annotated, Any, Dict, List, Optional
 
 from pydantic import AfterValidator, BaseModel, ConfigDict, Field, model_validator
 from pydantic_core import PydanticCustomError
 
 from rbx.autoenum import AutoEnum, alias
-from rbx.box.fields import NameField, Primitive, expand_vars
+from rbx.box.fields import NameField, Primitive, RecVars, Vars, expand_vars
 from rbx.box.statements.expander import expand_statements
 from rbx.box.statements.schema import Statement
 from rbx.grading.steps import Outcome
@@ -52,7 +53,7 @@ def convert_to_primitive(value: Any) -> Primitive:
 
 def expand_any_vars(vars: Dict[str, Any]) -> Dict[str, Primitive]:
     converted_vars = {key: convert_to_primitive(value) for key, value in vars.items()}
-    return expand_vars(converted_vars)
+    return expand_vars(typing.cast(RecVars, converted_vars))
 
 
 def is_unique_by_name(statements: List['Statement']) -> List['Statement']:
@@ -330,6 +331,26 @@ problems that have points.
 """,
     )
 
+    model_solution: Optional[CodeItem] = Field(
+        default=None,
+        description="""
+The solution to be used to generate outputs for this testgroup.
+
+Can only be set for the "samples" testgroup.
+""",
+    )
+
+    @model_validator(mode='after')
+    def check_model_solution_for_samples(self):
+        if self.name == 'samples':
+            return self
+        if self.model_solution is not None:
+            raise PydanticCustomError(
+                'MODEL_SOLUTION_NOT_ALLOWED',
+                'Model solution can only be set for the "samples" testgroup.',
+            )
+        return self
+
 
 class Generator(CodeItem):
     model_config = ConfigDict(extra='forbid')
@@ -419,6 +440,68 @@ class UnitTests(BaseModel):
     )
 
 
+class LimitsProfile(BaseModel):
+    model_config = ConfigDict(extra='forbid')
+
+    inheritFromPackage: bool = Field(
+        default=False,
+        description="""
+Whether to inherit limits from the package.
+""",
+    )
+
+    timeLimit: Optional[int] = Field(
+        default=None, description='Time limit of the problem, in milliseconds.'
+    )
+
+    memoryLimit: Optional[int] = Field(
+        default=None, description='Memory limit of the problem, in MB.'
+    )
+
+    outputLimit: Optional[int] = Field(
+        default=None, description='Output limit of the problem, in KB.'
+    )
+
+    modifiers: Dict[str, LimitModifiers] = Field(
+        default={},
+        description="""
+    Limit modifiers that can be specified per language.
+    """,
+    )
+
+    formula: Optional[str] = Field(
+        default=None,
+        description="""
+A formula to estimate the time limit for the problem.
+""",
+    )
+
+    def timelimit_for_language(self, language: Optional[str] = None) -> int:
+        assert self.timeLimit is not None
+        res = self.timeLimit
+        if language is not None and language in self.modifiers:
+            modifier = self.modifiers[language]
+            if modifier.time is not None:
+                res = modifier.time
+            if modifier.timeMultiplier is not None:
+                res = int(res * float(modifier.timeMultiplier))
+        if 'RBX_TIME_MULTIPLIER' in os.environ:
+            res = int(res * float(os.environ['RBX_TIME_MULTIPLIER']))
+        return res
+
+    def memorylimit_for_language(self, language: Optional[str] = None) -> int:
+        assert self.memoryLimit is not None
+        res = self.memoryLimit
+        if language is None:
+            return res
+        if language not in self.modifiers:
+            return res
+        modifier = self.modifiers[language]
+        if modifier.memory is not None:
+            return modifier.memory
+        return res
+
+
 class Package(BaseModel):
     model_config = ConfigDict(extra='forbid')
 
@@ -486,7 +569,7 @@ that is correct and used as reference -- and should have the `accepted` outcome.
     # Vars to be re-used across the package.
     #   - It will be passed as --key=value arguments to the validator.
     #   - It will be available as \VAR{key} variables in the rbx statement.
-    vars: Dict[str, Primitive] = Field(
+    vars: RecVars = Field(
         default={}, description='Variables to be re-used across the package.'
     )
 
@@ -500,32 +583,9 @@ that is correct and used as reference -- and should have the `accepted` outcome.
         return expand_statements(self.statements)
 
     @property
-    def expanded_vars(self) -> Dict[str, Primitive]:
+    def expanded_vars(self) -> Vars:
         return expand_vars(self.vars)
 
-    def timelimit_for_language(self, language: Optional[str]) -> int:
-        res = self.timeLimit
-        if language is not None and language in self.modifiers:
-            modifier = self.modifiers[language]
-            if modifier.time is not None:
-                res = modifier.time
-            if modifier.timeMultiplier is not None:
-                res = int(res * float(modifier.timeMultiplier))
-        if 'RBX_TIME_MULTIPLIER' in os.environ:
-            res = int(res * float(os.environ['RBX_TIME_MULTIPLIER']))
-        return res
-
-    def memorylimit_for_language(self, language: Optional[str]) -> int:
-        res = self.memoryLimit
-        if language is None:
-            return res
-        if language not in self.modifiers:
-            return res
-        modifier = self.modifiers[language]
-        if modifier.memory is not None:
-            return modifier.memory
-        return res
-
     @model_validator(mode='after')
     def check_first_solution_is_main_if_there_is_ac(self):
         if all(sol.outcome != Outcome.ACCEPTED for sol in self.solutions):

rbx/box/solutions.py

@@ -5,7 +5,7 @@ import dataclasses
 import pathlib
 import shutil
 from collections.abc import Iterator
-from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
+from typing import Dict, Iterable, List, Optional, Set, Tuple
 
 import rich
 import rich.live
@@ -17,7 +17,7 @@ from ordered_set import OrderedSet
 from pydantic import BaseModel
 
 from rbx import console, utils
-from rbx.box import checkers, environment, package, remote, state
+from rbx.box import checkers, code, limits_info, package, remote, state
 from rbx.box.code import (
     SanitizationLevel,
     compile_item,
@@ -34,6 +34,7 @@ from rbx.box.generators import (
     generate_output_for_testcase,
     generate_standalone,
 )
+from rbx.box.sanitizers import issue_stack
 from rbx.box.schema import (
     ExpectedOutcome,
     GeneratorCall,
@@ -92,6 +93,20 @@ class SolutionReportSkeleton(BaseModel):
     verification: VerificationLevel
     capture_pipes: bool = False
 
+    def get_solution_limits(self, solution: Solution) -> Limits:
+        lang = code.find_language_name(solution)
+        if lang is None:
+            return limits_info.get_package_limits(self.verification)
+        return self.limits[lang]
+
+    def get_solution_limits_from_disk(self, solution: Solution) -> Limits:
+        lang = code.find_language_name(solution)
+        return limits_info.get_limits(
+            language=lang,
+            profile=self.get_solution_limits(solution).profile,
+            verification=self.verification,
+        )
+
     def find_group_skeleton(self, group_name: str) -> Optional[GroupSkeleton]:
         groups = [group for group in self.groups if group.name == group_name]
         if not groups:
@@ -131,6 +146,17 @@ class RunSolutionResult:
         return self.skeleton.empty_structured_evaluation()
 
 
+class FailedSolutionIssue(issue_stack.Issue):
+    def __init__(self, solution: Solution):
+        self.solution = solution
+
+    def get_detailed_section(self) -> Tuple[str, ...]:
+        return ('solutions',)
+
+    def get_detailed_message(self) -> str:
+        return f'[item]{href(self.solution.path)}[/item] has an unexpected outcome.'
+
+
 def is_fast(solution: Solution) -> bool:
     # If solution has TLE tag, it is considered slow.
     return not solution.outcome.is_slow()
@@ -535,11 +561,13 @@ async def _generate_testcase_interactively(
         if progress:
             progress.update('Generating output for test...')
         # TODO: Add stderr path
-        await generate_output_for_testcase(
-            main_solution_digest,
-            testcase,
-            interactor_digest=interactor_digest,
-        )
+        if main_solution is not None:
+            await generate_output_for_testcase(
+                main_solution,
+                main_solution_digest,
+                testcase,
+                interactor_digest=interactor_digest,
+            )
 
     if check and testcase.outputPath is not None and not testcase.outputPath.is_file():
         # Output was not created, throw an error.
@@ -882,10 +910,10 @@ def get_evals_formatted_time(evals: List[Evaluation]) -> str:
 
 
 def get_capped_evals_formatted_time(
-    solution: Solution, evals: List[Evaluation], verification: VerificationLevel
+    limits: Limits,
+    evals: List[Evaluation],
+    verification: VerificationLevel,
 ) -> str:
-    pkg = package.find_problem_package_or_die()
-
     max_time = _get_evals_time_in_ms(evals)
     has_tle = any(eval.result.outcome.is_slow() for eval in evals)
     has_ile = any(
@@ -902,13 +930,13 @@ def get_capped_evals_formatted_time(
     if timelimits:
         tl = min(timelimits)
     if tl is None:
-        tl = pkg.timelimit_for_language(solution.language)
+        tl = limits.time
 
-        if verification.value >= VerificationLevel.FULL.value:
+        if tl is not None and verification.value >= VerificationLevel.FULL.value:
             # Using double TL for verification.
             tl = tl * 2
 
-    if has_tle and max_time >= tl or has_ile:
+    if tl is not None and has_tle and max_time >= tl or has_ile:
         return f'>{tl} ms'
     return f'{max_time} ms'
 
@@ -930,6 +958,7 @@ def get_truncated_message(message: str, max_length: int = 100) -> str:
 
 class SolutionOutcomeReport(BaseModel):
     solution: Solution
+    limits: Limits
     evals: List[Evaluation]
     ok: bool
     message: Optional[Tuple[TestcaseEntry, str]]
@@ -972,7 +1001,7 @@ class SolutionOutcomeReport(BaseModel):
 
     def get_outcome_markup(self, print_message: bool = True) -> str:
         res = self.get_verdict_markup_with_warnings()
-        res += f'\nTime: {get_capped_evals_formatted_time(self.solution, self.evals, self.verification)}'
+        res += f'\nTime: {get_capped_evals_formatted_time(self.limits, self.evals, self.verification)}'
         res += f'\nMemory: {get_evals_formatted_memory(self.evals)}'
         if print_message and self.message is not None:
             tc, msg = self.message
@@ -989,8 +1018,6 @@ def get_solution_outcome_report(
     verification: VerificationLevel = VerificationLevel.NONE,
     subset: bool = False,
 ) -> SolutionOutcomeReport:
-    pkg = package.find_problem_package_or_die()
-
     has_plain_tle = False
     all_verdicts = set()
     bad_verdicts = set()
@@ -1051,6 +1078,7 @@ def get_solution_outcome_report(
 
     evals_time = _get_evals_time_in_ms(evals)
     expected_outcome_is_tle = solution.outcome.matches_tle_and_is_incorrect()
+    limits = skeleton.get_solution_limits(solution)
     if (
         # Running verification with double TL.
         verification.value >= VerificationLevel.FULL.value
@@ -1061,7 +1089,8 @@ def get_solution_outcome_report(
         # A TLE has happened.
         and Outcome.TIME_LIMIT_EXCEEDED in matched_bad_verdicts
         # The solution runs in double TL.
-        and evals_time < pkg.timelimit_for_language(solution.language) * 2
+        and limits.time is not None
+        and evals_time < limits.time * 2
     ):
         other_verdicts = (bad_verdicts | no_tle_bad_verdicts) - {
             Outcome.TIME_LIMIT_EXCEEDED
@@ -1078,6 +1107,7 @@ def get_solution_outcome_report(
 
     return SolutionOutcomeReport(
         solution=solution,
+        limits=skeleton.get_solution_limits(solution),
         evals=evals,
         ok=not has_failed,
         message=message,
@@ -1102,11 +1132,13 @@ def _print_solution_outcome(
     report = get_solution_outcome_report(
         solution, skeleton, evals, verification, subset
     )
+    if not report.ok:
+        issue_stack.add_issue(FailedSolutionIssue(solution))
     console.print(report.get_outcome_markup(print_message))
     return report.ok
 
 
-def _consume_and_key_evaluation_items(
+def consume_and_key_evaluation_items(
     items: Iterable[EvaluationItem],
     skeleton: SolutionReportSkeleton,
 ) -> StructuredEvaluation:
@@ -1169,9 +1201,7 @@ async def _print_timing(
     console: rich.console.Console,
     skeleton: SolutionReportSkeleton,
     evaluations: StructuredEvaluation,
-    verification: VerificationLevel,
 ):
-    pkg = package.find_problem_package_or_die()
     summary = TimingSummary()
     summary_per_language = collections.defaultdict(TimingSummary)
     tls_per_language = {}
@@ -1196,8 +1226,12 @@ async def _print_timing(
         if solution_tls:
             solution_tl = min(solution_tls)
         else:
-            solution_tl = pkg.timelimit_for_language(solution.language)
-            if verification.value >= VerificationLevel.FULL.value:
+            limits = skeleton.get_solution_limits(solution)
+            if limits.time is None:
+                limits = skeleton.get_solution_limits_from_disk(solution)
+            assert limits.time is not None
+            solution_tl = limits.time
+            if limits.isDoubleTL:
                 solution_tl = solution_tl * 2
         all_tls.add(solution_tl)
         for eval in all_evals:
@@ -1306,7 +1340,8 @@ async def _render_detailed_group_table(
                 evals_per_solution[str(solution.path)].append(eval)
 
                 verdict = get_testcase_markup_verdict(eval)
-                time = get_capped_evals_formatted_time(solution, [eval], verification)
+                limits = skeleton.get_solution_limits(solution)
+                time = get_capped_evals_formatted_time(limits, [eval], verification)
                 memory = get_evals_formatted_memory([eval])
                 full_item = (f'[info]#{tc}[/info]', verdict, time, '/', memory, '')
                 if eval.result.sanitizer_warnings:
@@ -1323,8 +1358,9 @@ async def _render_detailed_group_table(
                 if not non_null_evals:
                     summary_row.append('...')
                     continue
+                limits = skeleton.get_solution_limits(solution)
                 formatted_time = get_capped_evals_formatted_time(
-                    solution, non_null_evals, verification
+                    limits, non_null_evals, verification
                 )
                 formatted_memory = get_evals_formatted_memory(non_null_evals)
                 worst_outcome = get_worst_outcome(non_null_evals)
@@ -1398,7 +1434,9 @@ async def _print_detailed_run_report(
 
     if timing:
         await _print_timing(
-            console, result.skeleton, structured_evaluations, verification=verification
+            console,
+            result.skeleton,
+            structured_evaluations,
         )
     return ok
 
@@ -1408,7 +1446,10 @@ def _print_limits(limits: Dict[str, Limits]):
         '[bold][success]Running with the following limits (per language):[/success][/bold]'
     )
     for lang, limit in limits.items():
-        console.console.print(f'[bold][status]{lang}[/status][/bold]')
+        extracted_from = ' (extracted from package)'
+        if limit.profile:
+            extracted_from = f' (extracted from profile [item]{limit.profile}[/item])'
+        console.console.print(f'[bold][status]{lang}[/status][/bold]{extracted_from}')
         time = (
             '<No time limit>' if limit.time is None else get_formatted_time(limit.time)
         )
@@ -1435,7 +1476,7 @@ async def print_run_report(
     if not skip_printing_limits:
         _print_limits(result.skeleton.limits)
 
-    structured_evaluations = _consume_and_key_evaluation_items(
+    structured_evaluations = consume_and_key_evaluation_items(
         result.items, result.skeleton
     )
     if detailed:
@@ -1455,6 +1496,7 @@ async def print_run_report(
         if single_solution:
             console.print()
         solution_evals = []
+        limits = result.skeleton.get_solution_limits(solution)
         for group in result.skeleton.groups:
             if not single_solution:
                 console.print(f'[bold][status]{group.name}[/status][/bold] ', end='')
@@ -1469,9 +1511,7 @@ async def print_run_report(
                     console.print(f'{group.name}/{i}', end='')
                     if eval.result.sanitizer_warnings:
                         console.print('[warning]*[/warning]', end='')
-                    time = get_capped_evals_formatted_time(
-                        solution, [eval], verification
-                    )
+                    time = get_capped_evals_formatted_time(limits, [eval], verification)
                     memory = get_evals_formatted_memory([eval])
                     console.print(f' ({time}, {memory})', end='')
                     checker_msg = eval.result.message
@@ -1493,7 +1533,7 @@ async def print_run_report(
             if single_solution:
                 console.print(f'  [status]{group.name}[/status]', end=' ')
             console.print(
-                f'({get_capped_evals_formatted_time(solution, group_evals, verification)}, {get_evals_formatted_memory(group_evals)})',
+                f'({get_capped_evals_formatted_time(limits, group_evals, verification)}, {get_evals_formatted_memory(group_evals)})',
                 end='',
             )
             console.print()
@@ -1513,92 +1553,9 @@ async def print_run_report(
 
     if not single_solution:
         await _print_timing(
-            console, result.skeleton, structured_evaluations, verification=verification
+            console,
+            result.skeleton,
+            structured_evaluations,
         )
 
     return ok
-
-
-def _step_up(x: Any, step: int) -> int:
-    x = int(x)
-    return (x + step - 1) // step * step
-
-
-def _step_down(x: Any, step: int) -> int:
-    x = int(x)
-    return x // step * step
-
-
-async def estimate_time_limit(
-    console: rich.console.Console,
-    result: RunSolutionResult,
-) -> Optional[int]:
-    structured_evaluations = _consume_and_key_evaluation_items(
-        result.items, result.skeleton
-    )
-
-    timing_per_solution = {}
-    timing_per_language = {}
-
-    if not result.skeleton.solutions:
-        console.print('[error]No solutions to estimate time limit from.[/error]')
-        return None
-
-    for solution in result.skeleton.solutions:
-        timings = []
-        for evals in structured_evaluations[str(solution.path)].values():
-            for ev in evals:
-                if ev is None:
-                    continue
-                ev = await ev()
-                if ev.log.time is not None:
-                    timings.append(int(ev.log.time * 1000))
-
-        if not timings:
-            console.print(
-                f'[warning]No timings for solution {href(solution.path)}.[/warning]'
-            )
-            continue
-
-        timing_per_solution[str(solution.path)] = max(timings)
-        lang = find_language_name(solution)
-        if lang not in timing_per_language:
-            timing_per_language[lang] = 0
-        timing_per_language[lang] = max(timing_per_language[lang], max(timings))
-
-    console.rule('[status]Time estimation[/status]', style='status')
-
-    fastest_time = min(timing_per_solution.values())
-    slowest_time = max(timing_per_solution.values())
-
-    console.print(f'Fastest solution: {fastest_time} ms')
-    console.print(f'Slowest solution: {slowest_time} ms')
-
-    if len(timing_per_language) > 0:
-        timing_language_list = [(t, lang) for lang, t in timing_per_language.items()]
-        fastest_language_time, fastest_language = min(timing_language_list)
-        slowest_language_time, slowest_language = max(timing_language_list)
-
-        console.print(
-            f'Fastest language: {fastest_language} ({fastest_language_time} ms)'
-        )
-        console.print(
-            f'Slowest language: {slowest_language} ({slowest_language_time} ms)'
-        )
-
-    env = environment.get_environment()
-    estimated_tl = int(
-        eval(
-            env.timing.formula,
-            {
-                'fastest': fastest_time,
-                'slowest': slowest_time,
-                'step_up': _step_up,
-                'step_down': _step_down,
-            },
-        )
-    )
-
-    console.print(f'Using formula: {env.timing.formula}')
-    console.print(f'[success]Estimated time limit:[/success] {estimated_tl} ms')
-    return estimated_tl

rbx/box/statements/build_statements.py

@@ -7,7 +7,7 @@ import syncer
 import typer
 
 from rbx import annotations, console, utils
-from rbx.box import environment, naming, package
+from rbx.box import environment, limits_info, naming, package
 from rbx.box.formatting import href
 from rbx.box.schema import Package, expand_any_vars
 from rbx.box.statements.builders import (
@@ -264,6 +264,9 @@ def build_statement_bytes(
                     root=pathlib.Path(td),
                 ),
                 item=StatementBuilderProblem(
+                    limits=limits_info.get_limits_profile(
+                        profile=limits_info.get_active_profile()
+                    ),
                     package=pkg,
                     statement=statement,
                     samples=StatementSample.from_testcases(
@@ -302,6 +305,14 @@ def build_statement(
     statement_path = (package.get_build_path() / statement.name).with_suffix(
         last_output.get_file_suffix()
     )
+    active_profile = limits_info.get_active_profile()
+    if (
+        active_profile is not None
+        and limits_info.get_saved_limits_profile(active_profile) is not None
+    ):
+        statement_path = statement_path.with_stem(
+            f'{statement_path.stem}-{active_profile}'
+        )
     statement_path.parent.mkdir(parents=True, exist_ok=True)
     statement_path.write_bytes(last_content)
     console.console.print(

rbx/box/statements/builders.py

@@ -12,7 +12,7 @@ from pydantic import BaseModel
 
 from rbx import console, utils
 from rbx.box.fields import Primitive
-from rbx.box.schema import Package, Testcase
+from rbx.box.schema import LimitsProfile, Package, Testcase
 from rbx.box.statements.latex_jinja import (
     JinjaDictWrapper,
     render_latex_template,
@@ -115,6 +115,7 @@ class ExplainedStatementSample(StatementSample):
 class StatementBuilderProblem(StatementBuilderItem):
     package: Package
     statement: Statement
+    limits: LimitsProfile
     samples: List[StatementSample] = dataclasses.field(default_factory=list)
     short_name: Optional[str] = None
 
@@ -128,8 +129,9 @@ class StatementBuilderProblem(StatementBuilderItem):
             'package': self.package,
             'statement': self.statement,
             'samples': self.samples,
-            'vars': JinjaDictWrapper(self.vars or {}, key='vars'),
+            'vars': JinjaDictWrapper.from_dict(self.vars or {}, wrapper_key='vars'),
             'title': self.statement.title or self.package.name,
+            'limits': self.limits,
         }
         if self.short_name is not None:
             kwargs['short_name'] = self.short_name
@@ -166,7 +168,7 @@ class StatementBuilderContest(StatementBuilderItem):
             'problems': [
                 problem.build_inner_jinja_kwargs() for problem in self.problems
             ],
-            'vars': JinjaDictWrapper(self.vars or {}, key='vars'),
+            'vars': JinjaDictWrapper.from_dict(self.vars or {}, wrapper_key='vars'),
         }
         return res