rbx.cp 0.6.1__py3-none-any.whl → 0.8.0__py3-none-any.whl

rbx/box/global_package.py

@@ -0,0 +1,74 @@
+import functools
+import pathlib
+import shutil
+
+from rbx.config import get_app_path
+from rbx.grading.caching import DependencyCache
+from rbx.grading.judge.cacher import FileCacher
+from rbx.grading.judge.sandbox import SandboxBase
+from rbx.grading.judge.sandboxes.stupid_sandbox import StupidSandbox
+from rbx.grading.judge.storage import FilesystemStorage, Storage
+
+CACHE_STEP_VERSION = 3
+
+
+def get_cache_fingerprint() -> str:
+    return f'{CACHE_STEP_VERSION}'
+
+
+@functools.cache
+def is_cache_valid(cache_dir: pathlib.Path) -> bool:
+    if not cache_dir.is_dir():
+        return True
+    fingerprint_file = cache_dir / 'fingerprint'
+    if not fingerprint_file.is_file():
+        return False
+    fingerprint = fingerprint_file.read_text()
+    if fingerprint.strip() != get_cache_fingerprint():
+        return False
+    return True
+
+
+@functools.cache
+def get_global_cache_dir() -> pathlib.Path:
+    cache_dir = get_app_path() / '.box'
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    fingerprint_file = cache_dir / 'fingerprint'
+    if not fingerprint_file.is_file():
+        fingerprint_file.write_text(get_cache_fingerprint())
+    return cache_dir
+
+
+def is_global_cache_valid() -> bool:
+    return is_cache_valid(get_global_cache_dir())
+
+
+@functools.cache
+def get_global_storage_dir() -> pathlib.Path:
+    storage_dir = get_global_cache_dir() / '.storage'
+    storage_dir.mkdir(parents=True, exist_ok=True)
+    return storage_dir
+
+
+@functools.cache
+def get_global_cache_storage() -> Storage:
+    return FilesystemStorage(get_global_storage_dir())
+
+
+@functools.cache
+def get_global_file_cacher() -> FileCacher:
+    return FileCacher(get_global_cache_storage())
+
+
+@functools.cache
+def get_global_dependency_cache() -> DependencyCache:
+    return DependencyCache(get_global_cache_dir(), get_global_file_cacher())
+
+
+@functools.cache
+def get_global_sandbox() -> SandboxBase:
+    return StupidSandbox(get_global_file_cacher())
+
+
+def clear_global_cache():
+    shutil.rmtree(get_global_cache_dir(), ignore_errors=True)

rbx/box/linting.py

@@ -1,26 +1,95 @@
+import difflib
 import pathlib
+from typing import Optional, Type
 
 import yamlfix
 import yamlfix.model
+from pydantic import BaseModel
 
 from rbx import console
-from rbx.box.cd import is_contest_package, is_problem_package
+from rbx.box.cd import is_contest_package, is_preset_package, is_problem_package
+from rbx.box.contest.schema import Contest
+from rbx.box.presets import get_preset_yaml
+from rbx.box.presets.schema import Preset
+from rbx.box.schema import Package
 from rbx.box.stats import find_problem_packages_from_contest
+from rbx.utils import uploaded_schema_path
 
 
-def fix_yaml(path: pathlib.Path, verbose: bool = True):
-    config = yamlfix.model.YamlfixConfig(quote_basic_values=True)
+def fix_language_server(path: pathlib.Path, model_cls: Type[BaseModel]) -> bool:
+    stream = []
+    with path.open('r') as f:
+        for line in f:
+            if line.strip().startswith('# yaml-language-server:'):
+                continue
+            stream.append(line)
+            if line.startswith('---'):
+                stream.append(
+                    f'# yaml-language-server: $schema={uploaded_schema_path(model_cls)}\n'
+                )
+    content = ''.join(stream)
+    orig_text = path.read_text()
+    path.write_text(content)
+    return orig_text != content
+
+
+def fix_yaml(
+    path: pathlib.Path,
+    verbose: bool = True,
+    print_diff: bool = False,
+    model_cls: Optional[Type[BaseModel]] = None,
+):
+    orig_text = path.read_text()
+
+    # Config to go hand-to-hand with VSCode YAML extension,
+    # which we offer first class support to. Unfortunately,
+    # YAML extension is not perfect :(
+    config = yamlfix.model.YamlfixConfig(
+        quote_basic_values=True,
+        quote_representation='"',
+        comments_min_spaces_from_content=1,
+    )
     _, changed = yamlfix.fix_files([str(path)], dry_run=False, config=config)
+
+    if model_cls is not None:
+        if fix_language_server(path, model_cls):
+            changed = True
+
     if changed and verbose:
         console.console.print(
             f'Formatting [item]{path}[/item].',
         )
 
+    if print_diff and changed:
+        unified_diff = difflib.unified_diff(
+            orig_text.splitlines(), path.read_text().splitlines()
+        )
+        console.console.print(
+            f'Diff for [item]{path}[/item].\n' + '\n'.join(unified_diff),
+        )
+
+
+def fix_package(root: pathlib.Path = pathlib.Path(), print_diff: bool = False):
+    if is_preset_package(root):
+        fix_yaml(root / 'preset.rbx.yml', model_cls=Preset, print_diff=print_diff)
+        preset = get_preset_yaml(root)
+        if preset.problem is not None:
+            fix_yaml(
+                root / preset.problem / 'problem.rbx.yml',
+                model_cls=Package,
+                print_diff=print_diff,
+            )
+        if preset.contest is not None:
+            fix_package(root / preset.contest, print_diff=print_diff)
+        return
 
-def fix_package(root: pathlib.Path = pathlib.Path()):
     if is_problem_package(root):
-        fix_yaml(root / 'problem.rbx.yml')
+        fix_yaml(root / 'problem.rbx.yml', model_cls=Package, print_diff=print_diff)
     if is_contest_package(root):
-        fix_yaml(root / 'contest.rbx.yml')
+        fix_yaml(root / 'contest.rbx.yml', model_cls=Contest, print_diff=print_diff)
         for problem in find_problem_packages_from_contest(root):
-            fix_yaml(problem / 'problem.rbx.yml')
+            fix_yaml(
+                problem / 'problem.rbx.yml',
+                model_cls=Package,
+                print_diff=print_diff,
+            )

rbx/box/package.py

@@ -11,8 +11,9 @@ import typer
 from pydantic import ValidationError
 
 from rbx import console, utils
-from rbx.box import cd
+from rbx.box import cd, global_package
 from rbx.box.environment import get_sandbox_type
+from rbx.box.global_package import get_cache_fingerprint
 from rbx.box.schema import (
     CodeItem,
     ExpectedOutcome,
@@ -34,7 +35,6 @@ YAML_NAME = 'problem.rbx.yml'
 _DEFAULT_CHECKER = 'wcmp.cpp'
 _NOOP_CHECKER = 'noop.cpp'
 TEMP_DIR = None
-CACHE_STEP_VERSION = 1
 
 
 @functools.cache
@@ -107,17 +107,13 @@ def get_ruyaml(root: pathlib.Path = pathlib.Path()) -> Tuple[ruyaml.YAML, ruyaml
     return res, res.load(problem_yaml_path.read_text())
 
 
-def _get_fingerprint() -> str:
-    return f'{CACHE_STEP_VERSION}'
-
-
 @functools.cache
 def get_problem_cache_dir(root: pathlib.Path = pathlib.Path()) -> pathlib.Path:
     cache_dir = find_problem(root) / '.box'
     cache_dir.mkdir(parents=True, exist_ok=True)
     fingerprint_file = cache_dir / 'fingerprint'
     if not fingerprint_file.is_file():
-        fingerprint_file.write_text(_get_fingerprint())
+        fingerprint_file.write_text(get_cache_fingerprint())
     return cache_dir
 
 
@@ -170,12 +166,12 @@ def get_problem_preprocessed_path(
 
 @functools.cache
 def get_cache_storage(root: pathlib.Path = pathlib.Path()) -> Storage:
-    return FilesystemStorage(get_problem_storage_dir(root))
+    return FilesystemStorage(get_problem_storage_dir(root), compress=False)
 
 
 @functools.cache
 def get_dependency_cache(root: pathlib.Path = pathlib.Path()) -> DependencyCache:
-    return DependencyCache(get_problem_cache_dir(root), get_cache_storage(root))
+    return DependencyCache(get_problem_cache_dir(root), get_file_cacher(root))
 
 
 @functools.cache
@@ -427,16 +423,7 @@ def get_merged_capture_path(root: pathlib.Path = pathlib.Path()) -> pathlib.Path
 @functools.cache
 def is_cache_valid(root: pathlib.Path = pathlib.Path()):
     cache_dir = find_problem(root) / '.box'
-    if not cache_dir.is_dir():
-        return True
-
-    fingerprint_file = cache_dir / 'fingerprint'
-    if not fingerprint_file.is_file():
-        return False
-    fingerprint = fingerprint_file.read_text()
-    if fingerprint.strip() != _get_fingerprint():
-        return False
-    return True
+    return global_package.is_cache_valid(cache_dir)
 
 
 def clear_package_cache():

rbx/box/presets/__init__.py

@@ -23,7 +23,7 @@ def _find_preset_yaml(root: pathlib.Path = pathlib.Path()) -> Optional[pathlib.P
     return None
 
 
-def _get_preset_yaml(root: pathlib.Path = pathlib.Path()) -> Preset:
+def get_preset_yaml(root: pathlib.Path = pathlib.Path()) -> Preset:
     found = _find_preset_yaml(root)
     if not found:
         console.console.print(
@@ -221,7 +221,7 @@ def _copy_updated_assets(
 def get_active_preset_or_null(root: pathlib.Path = pathlib.Path()) -> Optional[Preset]:
     local_preset = _find_local_preset(root)
     if local_preset is not None:
-        return _get_preset_yaml(local_preset)
+        return get_preset_yaml(local_preset)
     return None
 
 
@@ -319,7 +319,7 @@ def _install_preset_from_dir(
     update: bool = False,
     override_uri: Optional[str] = None,
 ):
-    preset = _get_preset_yaml(src)
+    preset = get_preset_yaml(src)
 
     if ensure_contest and preset.contest is None:
         console.console.print(
@@ -391,7 +391,7 @@ def _install_preset_from_local_dir(
     update: bool = False,
 ):
     pd = pathlib.Path(fetch_info.inner_dir)
-    preset = _get_preset_yaml(pd)
+    preset = get_preset_yaml(pd)
     console.console.print(
         f'Installing local preset [item]{preset.name}[/item] into [item]{dest}[/item]...'
     )

rbx/box/remote.py

@@ -27,6 +27,16 @@ class Expander(ABC):
         pass
 
 
+class MainExpander(Expander):
+    def expand(self, path: pathlib.Path) -> Optional[pathlib.Path]:
+        if str(path) != '@main':
+            return None
+        sol = package.get_main_solution()
+        if sol is None:
+            return None
+        return sol.path
+
+
 class BocaExpander(Expander):
     BOCA_REGEX = re.compile(r'\@boca\/(\d+)(?:\-(\d+))?')
 
@@ -69,6 +79,7 @@ class BocaExpander(Expander):
 
 
 REGISTERED_EXPANDERS: List['Expander'] = [
+    MainExpander(),
     BocaExpander(),
 ]
 
@@ -132,6 +143,9 @@ def _expand_paths(paths: List[pathlib.Path]) -> List[pathlib.Path]:
             continue
         expanded = _expand_path(path)
         if expanded is None:
+            console.console.print(
+                f'[warning]Remote solution [item]{path}[/item] could not be expanded. Skipping.[/warning]'
+            )
             continue
         res.append(expanded)
     return res
@@ -149,3 +163,8 @@ def expand_file(file: str) -> pathlib.Path:
         )
         raise typer.Exit(1)
     return res[0]
+
+
+def is_path_remote(path: pathlib.Path) -> bool:
+    remote_dir = package.get_problem_remote_dir()
+    return path.resolve().is_relative_to(remote_dir.resolve())

rbx/box/sanitizers/warning_stack.py

@@ -4,7 +4,7 @@ import shutil
 
 from rbx import console
 from rbx.box.schema import CodeItem
-from rbx.grading.judge.storage import Storage
+from rbx.grading.judge.cacher import FileCacher
 from rbx.grading.steps import GradingFileOutput
 
 
@@ -18,7 +18,7 @@ class WarningStack:
         self.warnings.add(code.path)
 
     def add_sanitizer_warning(
-        self, storage: Storage, code: CodeItem, reference: GradingFileOutput
+        self, cacher: FileCacher, code: CodeItem, reference: GradingFileOutput
     ):
         if code.path in self.sanitizer_warnings:
             return
@@ -26,7 +26,7 @@ class WarningStack:
             code.path.with_suffix(code.path.suffix + '.log')
         )
         dest_path.parent.mkdir(parents=True, exist_ok=True)
-        f = reference.get_file(storage)
+        f = reference.get_file(cacher)
         if f is None:
             return
         with dest_path.open('wb') as fout:

rbx/box/solutions.py

@@ -13,6 +13,7 @@ import rich.markup
 import rich.table
 import rich.text
 import typer
+from ordered_set import OrderedSet
 from pydantic import BaseModel
 
 from rbx import console, utils
@@ -245,7 +246,7 @@ async def convert_list_of_solution_evaluations_to_dict(
 
 
 def _get_solutions_for_skeleton(
-    tracked_solutions: Optional[Set[str]] = None,
+    tracked_solutions: Optional[Iterable[str]] = None,
     verification: VerificationLevel = VerificationLevel.NONE,
 ) -> List[Solution]:
     pkg = package.find_problem_package_or_die()
@@ -260,7 +261,7 @@ def _get_solutions_for_skeleton(
 
 
 def _get_report_skeleton(
-    tracked_solutions: Optional[Set[str]] = None,
+    tracked_solutions: Optional[Iterable[str]] = None,
     verification: VerificationLevel = VerificationLevel.NONE,
     timelimit_override: Optional[int] = None,
 ) -> SolutionReportSkeleton:
@@ -393,7 +394,7 @@ def print_best_output(output_files: List[pathlib.Path], empty_warning: bool = Fa
 
 def run_solutions(
     progress: Optional[StatusProgress] = None,
-    tracked_solutions: Optional[Set[str]] = None,
+    tracked_solutions: Optional[Iterable[str]] = None,
     verification: VerificationLevel = VerificationLevel.NONE,
     check: bool = True,
     timelimit_override: Optional[int] = None,
@@ -605,7 +606,7 @@ def _run_interactive_solutions(
 
 
 def _get_interactive_skeleton(
-    tracked_solutions: Optional[Set[str]] = None,
+    tracked_solutions: Optional[Iterable[str]] = None,
     verification: VerificationLevel = VerificationLevel.NONE,
 ) -> SolutionReportSkeleton:
     solutions = _get_solutions_for_skeleton(tracked_solutions, verification)
@@ -645,7 +646,7 @@ def _get_interactive_skeleton(
 
 async def run_and_print_interactive_solutions(
     progress: Optional[StatusProgress] = None,
-    tracked_solutions: Optional[Set[str]] = None,
+    tracked_solutions: Optional[Iterable[str]] = None,
     verification: VerificationLevel = VerificationLevel.NONE,
     generator: Optional[GeneratorCall] = None,
     testcase_entry: Optional[TestcaseEntry] = None,
@@ -739,7 +740,12 @@ def expand_solutions_with_source(sols: List[str]) -> List[Tuple[Solution, bool]]
     path_sols = remote.expand_files(sols)
 
     # Ensure sols exist.
-    path_sols = [sol for sol in path_sols if sol.is_file()]
+    for sol in path_sols:
+        if not sol.is_file():
+            console.console.print(
+                f'[error]Solution [item]{sol}[/item] could not be found.[/error]'
+            )
+            raise typer.Exit(1)
 
     seen_sols = set()
     res: List[Tuple[Solution, bool]] = []
@@ -762,7 +768,7 @@ def expand_solutions(sols: List[str]) -> List[Solution]:
 
 
 async def pick_solutions(
-    tracked_solutions: Optional[Set[str]],
+    tracked_solutions: Optional[OrderedSet[str]],
     extra_solutions: Optional[List[str]] = None,
 ) -> List[str]:
     pkg = package.find_problem_package_or_die()

rbx/box/stats.py

@@ -78,6 +78,14 @@ def print_package_stats(root: pathlib.Path = pathlib.Path()) -> int:
     return cache_size + build_size
 
 
+def print_global_stats() -> int:
+    cache_size = get_cache_size()
+    console.console.print(
+        f'[status]Global cache size[/status]: [item]{get_formatted_memory(cache_size)}[/item]'
+    )
+    return cache_size
+
+
 def print_reachable_package_stats(root: pathlib.Path = pathlib.Path()) -> None:
     contest_packages, problem_packages = find_and_group_all_reachable_packages(root)
     total_size = 0
@@ -87,6 +95,8 @@ def print_reachable_package_stats(root: pathlib.Path = pathlib.Path()) -> None:
     for pkg in problem_packages:
         total_size += print_package_stats(pkg)
         console.console.print()
+
+    total_size += print_global_stats()
     console.console.print(
         f'[status]Total size[/status]: [item]{get_formatted_memory(total_size)}[/item]'
     )

rbx/box/stresses.py

@@ -1,3 +1,5 @@
+import pathlib
+import shutil
 import time
 from shutil import rmtree
 from typing import List, Optional
@@ -8,24 +10,19 @@ import typer
 from pydantic import BaseModel
 
 from rbx import console
-from rbx.box import checkers, generators, package, validators
-from rbx.box.code import SanitizationLevel, compile_item, run_item
+from rbx.box import checkers, generators, package, tasks, validators
+from rbx.box.code import SanitizationLevel, compile_item
 from rbx.box.generators import (
     GenerationMetadata,
     expand_generator_call,
     generate_standalone,
 )
-from rbx.box.retries import Retrier
 from rbx.box.schema import CodeItem, GeneratorCall, Stress, TaskType, Testcase
 from rbx.box.solutions import compile_solutions, get_outcome_style_verdict
 from rbx.box.stressing import finder_parser
 from rbx.grading.steps import (
-    DigestOrDest,
-    DigestOrSource,
     Evaluation,
     Outcome,
-    TestcaseIO,
-    TestcaseLog,
 )
 from rbx.utils import StatusProgress
 
@@ -61,11 +58,6 @@ async def run_stress(
     sanitized: bool = False,
 ) -> StressReport:
     pkg = package.find_problem_package_or_die()
-    if pkg.type == TaskType.COMMUNICATION:
-        console.console.print(
-            '[error]Communication problems do not support stress testing.[/error]'
-        )
-        raise typer.Exit(1)
 
     if finder:
         if generator_call is None:
@@ -90,6 +82,8 @@ async def run_stress(
     call = stress.generator
     generator = package.get_generator(call.name)
 
+    if progress:
+        progress.update('Compiling generator...')
     try:
         generator_digest = compile_item(generator, sanitized=SanitizationLevel.PREFER)
     except:
@@ -109,11 +103,18 @@ async def run_stress(
     solutions_digest = compile_solutions(
         tracked_solutions=set(str(solution.path) for solution in solutions),
         sanitized=sanitized,
+        progress=progress,
     )
     if progress:
         progress.update('Compiling finders...')
     finders_digest = {str(finder.path): _compile_finder(finder) for finder in finders}
 
+    interactor_digest = None
+    if pkg.type == TaskType.COMMUNICATION:
+        interactor_digest = checkers.compile_interactor(progress=progress)
+
+    if progress:
+        progress.update('Compiling validator...')
     compiled_validator = validators.compile_main_validator()
 
     # Erase old stress directory
@@ -158,47 +159,40 @@ async def run_stress(
             else None,
         )
 
-        @async_lru.alru_cache
+        @async_lru.alru_cache(maxsize=None)
         async def run_solution_fn(
             solution: str,
-            retry_index: Optional[int] = None,
+            checker_digest: Optional[str] = None,
             input_path=input_path,
-        ) -> TestcaseLog:
+            output_path: Optional[pathlib.Path] = None,
+        ) -> Evaluation:
             index = solution_indices[solution]
             sol = solutions[index]
-            output_path = input_path.with_stem(f'{index}').with_suffix('.out')
-            stderr_path = output_path.with_suffix('.err')
-
-            run_log = await run_item(
-                sol,
-                DigestOrSource.create(solutions_digest[sol.path]),
-                stdin=DigestOrSource.create(input_path),
-                stdout=DigestOrDest.create(output_path),
-                stderr=DigestOrDest.create(stderr_path),
-                retry_index=retry_index,
-            )
-
-            return TestcaseLog(
-                **(run_log.model_dump() if run_log is not None else {}),
-                stdout_absolute_path=output_path.absolute(),
-                stderr_absolute_path=stderr_path.absolute(),
+            return await tasks.run_solution_on_testcase(
+                solutions[index],
+                compiled_digest=solutions_digest[sol.path],
+                checker_digest=checker_digest,
+                interactor_digest=interactor_digest,
+                testcase=Testcase(inputPath=input_path, outputPath=output_path),
+                output_dir=input_path.parent,
+                filestem=f'{index}',
+                is_stress=True,
             )
 
         # Get main solution output.
         expected_output_path = empty_path
         if needs_expected_output:
-            main_testcase_log = await run_solution_fn(str(solutions[0].path))
-            main_checker_result = checkers.check_with_no_output(main_testcase_log)
-            if main_checker_result.outcome != Outcome.ACCEPTED:
+            eval = await run_solution_fn(str(solutions[0].path))
+            if eval.result.outcome != Outcome.ACCEPTED:
                 console.console.print(
                     '[error]Error while generating main solution output.[/error]'
                 )
                 console.console.print(f'Input written at [item]{input_path}[/item]')
                 console.console.print(
-                    f'Output written at [item]{main_testcase_log.stdout_absolute_path}[/item]'
+                    f'Output written at [item]{eval.log.stdout_absolute_path}[/item]'
                 )
                 console.console.print(
-                    f'Stderr written at [item]{main_testcase_log.stderr_absolute_path}[/item]'
+                    f'Stderr written at [item]{eval.log.stderr_absolute_path}[/item]'
                 )
                 console.console.print()
                 console.console.print(
@@ -206,44 +200,31 @@ async def run_stress(
                     "use the two-way modifier in your finder expression (':2')."
                 )
                 raise typer.Exit(1)
-            expected_output_path = main_testcase_log.stdout_absolute_path
+            if eval.log.stdout_absolute_path is not None:
+                expected_output_path = input_path.with_suffix('.ans')
+                shutil.copyfile(eval.log.stdout_absolute_path, expected_output_path)
+            else:
+                expected_output_path = None
 
-        @async_lru.alru_cache
+        @async_lru.alru_cache(maxsize=None)
         async def run_solution_and_checker_fn(
             call: finder_parser.FinderCall,
-            input_path=input_path,
             expected_output_path=expected_output_path,
         ) -> finder_parser.FinderResult:
-            async def run_fn(retry_index: int) -> Evaluation:
+            async def run_fn() -> Evaluation:
                 solution = call.solution
                 checker = call.checker
 
-                testcase_log = await run_solution_fn(solution, retry_index=retry_index)
-                assert testcase_log.stdout_absolute_path is not None
-
-                if checker is None:
-                    checker_result = checkers.check_with_no_output(testcase_log)
-                else:
-                    checker_digest = finders_digest[checker.path]
-                    checker_result = await checkers.check(
-                        checker_digest,
-                        testcase_log,
-                        Testcase(inputPath=input_path, outputPath=expected_output_path),
-                        program_output=testcase_log.stdout_absolute_path,
-                    )
-
-                return Evaluation(
-                    result=checker_result,
-                    testcase=TestcaseIO(
-                        index=0,
-                        input=input_path,
-                        output=expected_output_path,
-                    ),
-                    log=testcase_log,
+                checker_digest = (
+                    finders_digest[checker.path] if checker is not None else None
+                )
+                return await run_solution_fn(
+                    solution,
+                    checker_digest=checker_digest,
+                    output_path=expected_output_path,
                 )
 
-            retrier = Retrier(is_stress=True)
-            eval = await retrier.repeat(run_fn)
+            eval = await run_fn()
 
             return finder_parser.FinderResult(
                 solution=call.solution,