rbx.cp 0.7.0__tar.gz → 0.9.0__tar.gz

{rbx_cp-0.7.0 → rbx_cp-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: rbx.cp
-Version: 0.7.0
+Version: 0.9.0
 Summary: 
 Author: Roberto Sales
 Requires-Python: >=3.9.1,<4.0.0
@@ -18,13 +18,16 @@ Requires-Dist: dateparser (>=1.2.1,<2.0.0)
 Requires-Dist: deepmerge (>=2.0,<3.0)
 Requires-Dist: fastapi (>=0.115.8,<0.116.0)
 Requires-Dist: filelock (>=3.14.0,<4.0.0)
+Requires-Dist: gitignore-parser (>=0.1.12,<0.2.0)
 Requires-Dist: gitpython (>=3.1.43,<4.0.0)
 Requires-Dist: jinja2 (>=3.1.4,<4.0.0)
 Requires-Dist: lark (>=1.2.2,<2.0.0)
 Requires-Dist: latexbuild (>=0.2.2,<0.3.0)
+Requires-Dist: lz4 (>=4.4.4,<5.0.0)
 Requires-Dist: mechanize (>=0.4.10,<0.5.0)
 Requires-Dist: more-itertools (>=10.5.0,<11.0.0)
 Requires-Dist: nest-asyncio (>=1.6.0,<2.0.0)
+Requires-Dist: ordered-set (>=4.1.0,<5.0.0)
 Requires-Dist: psutil (>=7.0.0,<8.0.0)
 Requires-Dist: pydantic (==2.8.2)
 Requires-Dist: pydantic-xml[lxml] (>=2.11.0,<3.0.0)

{rbx_cp-0.7.0 → rbx_cp-0.9.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "rbx.cp"
-version = "0.7.0"
+version = "0.9.0"
 description = ""
 packages = [
   {include = "rbx"}
@@ -45,6 +45,9 @@ dateparser = "^1.2.1"
 deepmerge = "^2.0"
 yamlfix = "^1.17.0"
 ruamel-yaml = "^0.18.14"
+lz4 = "^4.4.4"
+ordered-set = "^4.1.0"
+gitignore-parser = "^0.1.12"
 
 [tool.poetry.scripts]
 rbc = "rbx.main:app"
@@ -75,6 +78,8 @@ version_scheme = "pep440"
 version_provider = "poetry"
 update_changelog_on_bump = true
 major_version_zero = true
+annotated_tag = true
+
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"

{rbx_cp-0.7.0 → rbx_cp-0.9.0}/rbx/box/cd.py

@@ -5,7 +5,7 @@ from typing import List, Optional
 
 import typer
 
-from rbx import console
+from rbx import console, utils
 from rbx.box.sanitizers import warning_stack
 from rbx.utils import new_cd
 
@@ -13,7 +13,7 @@ from rbx.utils import new_cd
 def find_package(
     root: pathlib.Path = pathlib.Path(), consider_presets: bool = False
 ) -> Optional[pathlib.Path]:
-    root = root.resolve()
+    root = utils.abspath(root)
 
     def has_file():
         problem_yaml_path = root / 'problem.rbx.yml'

{rbx_cp-0.7.0 → rbx_cp-0.9.0}/rbx/box/cli.py

@@ -1,3 +1,4 @@
+import atexit
 import pathlib
 import shlex
 import shutil
@@ -9,6 +10,7 @@ import rich
 import rich.prompt
 import syncer
 import typer
+from ordered_set import OrderedSet
 
 from rbx import annotations, config, console, utils
 from rbx.box import (
@@ -18,6 +20,7 @@ from rbx.box import (
     download,
     environment,
     generators,
+    global_package,
     package,
     presets,
     setter_config,
@@ -43,6 +46,7 @@ from rbx.box.statements import build_statements
 from rbx.box.testcase_utils import TestcaseEntry
 from rbx.box.testcases import main as testcases
 from rbx.box.tooling import main as tooling
+from rbx.grading import grading_context
 
 app = typer.Typer(no_args_is_help=True, cls=annotations.AliasGroup)
 app.add_typer(
@@ -105,6 +109,15 @@ app.add_typer(
 
 @app.callback()
 def main(
+    cache: Annotated[
+        int,
+        typer.Option(
+            '-c',
+            '--cache',
+            help='Which degree of caching to use.',
+            default_factory=lambda: grading_context.CacheLevel.CACHE_ALL.value,
+        ),
+    ],
     sanitized: bool = typer.Option(
         False,
         '--sanitized',
@@ -118,12 +131,23 @@ def main(
         flag_value=False,
         help='Whether to save extra logs and outputs from interactive solutions.',
     ),
+    profile: bool = typer.Option(
+        False,
+        '--profile',
+        '-p',
+        help='Whether to profile the execution.',
+    ),
 ):
     if cd.is_problem_package() and not package.is_cache_valid():
         console.console.print(
             '[warning]Cache is incompatible with the current version of [item]rbx[/item], so it will be cleared.[/warning]'
         )
         clear()
+    if not global_package.is_global_cache_valid():
+        console.console.print(
+            '[warning]Global cache is incompatible with the current version of [item]rbx[/item], so it will be cleared.[/warning]'
+        )
+        clear(global_cache=True)
 
     state.STATE.run_through_cli = True
     state.STATE.sanitized = sanitized
@@ -134,6 +158,13 @@ def main(
         )
     state.STATE.debug_logs = capture
 
+    grading_context.cache_level_var.set(grading_context.CacheLevel(cache))
+
+    if profile:
+        from rbx.grading import profiling
+
+        atexit.register(profiling.print_summary)
+
 
 @app.command('ui', hidden=True)
 @package.within_problem
@@ -252,17 +283,17 @@ async def run(
         )
         check = False
 
-    tracked_solutions = None
+    tracked_solutions: Optional[OrderedSet[str]] = None
     if outcome is not None:
-        tracked_solutions = {
+        tracked_solutions = OrderedSet(
             str(solution.path)
             for solution in get_matching_solutions(ExpectedOutcome(outcome))
-        }
+        )
     if solutions:
-        tracked_solutions = set(solutions)
+        tracked_solutions = OrderedSet(solutions)
 
     if choice:
-        tracked_solutions = set(
+        tracked_solutions = OrderedSet(
             await pick_solutions(
                 tracked_solutions,
                 extra_solutions=solutions,
@@ -307,10 +338,10 @@ async def run(
         console.console.print(
             '[warning]Sanitizers are running, and no solutions were specified to run. Will only run [item]ACCEPTED[/item] solutions.'
         )
-        tracked_solutions = {
+        tracked_solutions = OrderedSet(
             str(solution.path)
             for solution in get_exact_matching_solutions(ExpectedOutcome.ACCEPTED)
-        }
+        )
 
     with utils.StatusProgress('Running solutions...') as s:
         solution_result = run_solutions(
@@ -343,10 +374,10 @@ async def _time_impl(check: bool, detailed: bool, runs: int = 0) -> Optional[int
     verification = VerificationLevel.ALL_SOLUTIONS.value
 
     with utils.StatusProgress('Running ACCEPTED solutions...') as s:
-        tracked_solutions = {
+        tracked_solutions = OrderedSet(
             str(solution.path)
             for solution in get_exact_matching_solutions(ExpectedOutcome.ACCEPTED)
-        }
+        )
         solution_result = run_solutions(
             progress=s,
             tracked_solutions=tracked_solutions,
@@ -495,17 +526,17 @@ async def irun(
         )
         return
 
-    tracked_solutions = None
+    tracked_solutions: Optional[OrderedSet[str]] = None
     if outcome is not None:
-        tracked_solutions = {
+        tracked_solutions = OrderedSet(
             str(solution.path)
             for solution in get_matching_solutions(ExpectedOutcome(outcome))
-        }
+        )
     if solutions:
-        tracked_solutions = set(solutions)
+        tracked_solutions = OrderedSet(solutions)
 
     if choice:
-        tracked_solutions = set(
+        tracked_solutions = OrderedSet(
             await pick_solutions(
                 tracked_solutions,
                 extra_solutions=solutions,
@@ -519,10 +550,10 @@ async def irun(
         console.console.print(
             '[warning]Sanitizers are running, and no solutions were specified to run. Will only run [item]ACCEPTED[/item] solutions.'
         )
-        tracked_solutions = {
+        tracked_solutions = OrderedSet(
             str(solution.path)
             for solution in get_exact_matching_solutions(ExpectedOutcome.ACCEPTED)
-        }
+        )
 
     with utils.StatusProgress('Running solutions...') as s:
         await run_and_print_interactive_solutions(
@@ -546,7 +577,13 @@ async def irun(
     help='Create a new problem package.',
 )
 def create(
-    name: str,
+    name: Annotated[
+        str,
+        typer.Option(
+            help='Name of the problem to create, which will be used as the name of the new folder.',
+            prompt='What should the name of the problem be?',
+        ),
+    ],
     preset: Annotated[
         Optional[str], typer.Option(help='Preset to use when creating the problem.')
     ] = None,
@@ -631,16 +668,17 @@ async def stress(
     from rbx.box import stresses
 
     with utils.StatusProgress('Running stress...') as s:
-        report = await stresses.run_stress(
-            timeout,
-            name=name,
-            generator_call=generator_args,
-            finder=finder,
-            findingsLimit=findings,
-            progress=s,
-            verbose=verbose,
-            sanitized=sanitized,
-        )
+        with grading_context.cache_level(grading_context.CacheLevel.CACHE_COMPILATION):
+            report = await stresses.run_stress(
+                timeout,
+                name=name,
+                generator_call=generator_args,
+                finder=finder,
+                findingsLimit=findings,
+                progress=s,
+                verbose=verbose,
+                sanitized=sanitized,
+            )
 
     stresses.print_stress_report(report)
 
@@ -882,7 +920,7 @@ def languages():
 
     for language in env.languages:
         console.console.print(
-            f'[item]{language.name}[/item], aka [item]{language.readable_name or language.name}[/item]:'
+            f'[item]{language.name}[/item], aka [item]{language.readableName or language.name}[/item]:'
         )
         console.console.print(language)
         console.console.print()
@@ -922,13 +960,29 @@ def fix(print_diff: bool = typer.Option(False, '--print-diff', '-p')):
     linting.fix_package(print_diff=print_diff)
 
 
+@cd.within_closest_package
+def _clear_package_cache():
+    console.console.print('Cleaning cache and build directories...')
+    shutil.rmtree('.box', ignore_errors=True)
+    shutil.rmtree('build', ignore_errors=True)
+
+
 @app.command(
     'clear, clean',
     rich_help_panel='Management',
     help='Clears cache and build directories.',
 )
-@cd.within_closest_package
-def clear():
-    console.console.print('Cleaning cache and build directories...')
-    shutil.rmtree('.box', ignore_errors=True)
-    shutil.rmtree('build', ignore_errors=True)
+def clear(global_cache: bool = typer.Option(False, '--global', '-g')):
+    cleared = False
+    if global_cache:
+        console.console.print('Cleaning global cache...')
+        global_package.clear_global_cache()
+        cleared = True
+
+    closest_package = cd.find_package()
+    if closest_package is not None:
+        _clear_package_cache()
+        cleared = True
+
+    if not cleared:
+        console.console.print('[error]No cache or build directories to clean.[/error]')

{rbx_cp-0.7.0 → rbx_cp-0.9.0}/rbx/box/code.py

@@ -10,9 +10,10 @@ from typing import List, Optional
 import rich
 import rich.text
 import typer
+from pydantic import BaseModel
 
-from rbx import console
-from rbx.box import download, package, setter_config, state
+from rbx import console, utils
+from rbx.box import download, global_package, package, setter_config, state
 from rbx.box.environment import (
     CompilationConfig,
     ExecutionConfig,
@@ -27,10 +28,11 @@ from rbx.box.environment import (
     merge_execution_configs,
 )
 from rbx.box.formatting import get_formatted_memory
+from rbx.box.remote import is_path_remote
 from rbx.box.sanitizers import warning_stack
 from rbx.box.schema import CodeItem
-from rbx.grading import steps, steps_with_caching
-from rbx.grading.judge.sandbox import SandboxBase, SandboxParams
+from rbx.grading import grading_context, profiling, steps, steps_with_caching
+from rbx.grading.judge.sandbox import SandboxParams
 from rbx.grading.steps import (
     DigestHolder,
     DigestOrDest,
@@ -59,6 +61,10 @@ class SanitizationLevel(Enum):
         return self.value >= SanitizationLevel.FORCE.value
 
 
+class CompilationMetadata(BaseModel):
+    is_sanitized: bool
+
+
 def substitute_commands(commands: List[str], sanitized: bool = False) -> List[str]:
     cfg = setter_config.get_setter_config()
     return [cfg.substitute_command(command, sanitized) for command in commands]
@@ -78,8 +84,15 @@ def find_language_name(code: CodeItem) -> str:
 def is_executable_sanitized(executable: DigestOrSource) -> bool:
     if executable.digest is None:
         return False
-    storage = package.get_cache_storage()
-    return storage.exists(f'{executable.digest.value}.san')
+    if executable.digest.value is None:
+        return False
+    cacher = package.get_file_cacher()
+    desc = cacher.get_metadata(
+        executable.digest.value, 'compilation', CompilationMetadata
+    )
+    if desc is None:
+        return False
+    return desc.is_sanitized
 
 
 def add_sanitizer_flags_to_command(command: str) -> str:
@@ -283,7 +296,7 @@ def _prepare_for_communication(
             )
 
         if capture.merged_capture is not None:
-            merged_output_path = package.get_merged_capture_path().resolve()
+            merged_output_path = utils.abspath(package.get_merged_capture_path())
             run.sandbox_params.timeit_dups['Do'].append(merged_output_path)
 
             run.artifacts.outputs.append(
@@ -402,7 +415,6 @@ def _should_precompile(commands: List[str]) -> bool:
 def _precompile_header(
     compilation_options: CompilationConfig,
     sanitized: SanitizationLevel,
-    sandbox: SandboxBase,
     sandbox_params: SandboxParams,
     artifacts: GradingArtifacts,
     input_artifact: GradingFileInput,
@@ -417,7 +429,8 @@ def _precompile_header(
     """
     assert compilation_options.commands is not None
 
-    dependency_cache = package.get_dependency_cache()
+    sandbox = global_package.get_global_sandbox()
+    dependency_cache = global_package.get_global_dependency_cache()
 
     # TODO: deduplicate code with compile_item.
     commands = get_mapped_commands(
@@ -455,41 +468,60 @@ def _precompile_header(
         GradingFileOutput(
             src=PosixPath('precompilable.h.gch'),
             digest=precompiled_digest,
-            executable=True,
         )
     )
 
-    if not steps_with_caching.compile(
-        commands,
-        params=sandbox_params,
-        artifacts=precompilation_artifacts,
-        sandbox=sandbox,
-        dependency_cache=dependency_cache,
-    ):
-        console.console.print(
-            f'[error]Failed to precompile header file: [item]{input_artifact.src}[/item][/error]'
-        )
-        raise typer.Exit(1)
+    with profiling.PushContext('code.precompile_header'):
+        if not steps_with_caching.compile(
+            commands,
+            params=sandbox_params,
+            artifacts=precompilation_artifacts,
+            sandbox=sandbox,
+            dependency_cache=dependency_cache,
+        ):
+            console.console.print(
+                f'[error]Failed to precompile header file: [item]{input_artifact.src}[/item][/error]'
+            )
+            raise typer.Exit(1)
 
-    if verbose:
-        console.console.print(
-            f'[status]Precompiled header file: [item]{input_artifact.src}[/item]'
-        )
+        if verbose:
+            console.console.print(
+                f'[status]Precompiled header file: [item]{input_artifact.src}[/item]'
+            )
 
-        if (
-            precompilation_artifacts.logs is not None
-            and precompilation_artifacts.logs.preprocess is not None
-        ):
-            for log in precompilation_artifacts.logs.preprocess:
-                console.console.print(f'[status]Command:[/status] {log.get_command()}')
-                console.console.print(f'[status]Summary:[/status] {log.get_summary()}')
+            if (
+                precompilation_artifacts.logs is not None
+                and precompilation_artifacts.logs.preprocess is not None
+            ):
+                for log in precompilation_artifacts.logs.preprocess:
+                    console.console.print(
+                        f'[status]Command:[/status] {log.get_command()}'
+                    )
+                    console.console.print(
+                        f'[status]Summary:[/status] {log.get_summary()}'
+                    )
 
     assert precompiled_digest.value is not None
 
+    digest_path = dependency_cache.cacher.path_for_symlink(precompiled_digest.value)
+    if digest_path is not None and digest_path.is_file():
+        # If storage backend supports symlinks, use it as the grading input.
+        input = DigestOrSource.create(digest_path)
+    else:
+        # Otherwise, copy the file to the local cache, transiently.
+        local_cacher = package.get_file_cacher()
+        with dependency_cache.cacher.get_file(precompiled_digest.value) as f:
+            with grading_context.cache_level(
+                grading_context.CacheLevel.CACHE_TRANSIENTLY
+            ):
+                input = DigestOrSource.create(local_cacher.put_file_from_fobj(f))
+
     return GradingFileInput(
-        digest=precompiled_digest,
+        **input.expand(),
         dest=input_artifact.dest.with_suffix('.h.gch'),
-        executable=True,
+        # Do not track fingerprint of the precompiled header file,
+        # trust the compilation step above.
+        hash=False,
     )
 
 
@@ -576,37 +608,43 @@ def compile_item(
 
     # Precompile C++ interesting header files.
     if precompile and _should_precompile(commands):
-        precompilation_inputs = []
-        for input in artifacts.inputs:
-            if (
-                input.src is not None
-                and input.src.suffix == '.h'
-                and input.dest.name in ['stdc++.h', 'jngen.h', 'testlib.h']
-            ):
-                precompilation_inputs.append(
-                    _precompile_header(
-                        compilation_options,
-                        sanitized,
-                        sandbox,
-                        sandbox_params,
-                        artifacts,
-                        input,
-                        force_warnings,
-                        verbose=False,
+        with profiling.Profiler('code.precompile'):
+            precompilation_inputs = []
+            for input in artifacts.inputs:
+                if (
+                    input.src is not None
+                    and input.src.suffix == '.h'
+                    and input.dest.name in ['stdc++.h', 'jngen.h', 'testlib.h']
+                ):
+                    precompilation_inputs.append(
+                        _precompile_header(
+                            compilation_options,
+                            sanitized,
+                            sandbox_params,
+                            artifacts,
+                            input,
+                            force_warnings,
+                            verbose=False,
+                        )
                     )
-                )
-        if precompilation_inputs:
-            artifacts.inputs.extend(precompilation_inputs)
-
-    # Compile the code.
-    if not steps_with_caching.compile(
-        commands,
-        params=sandbox_params,
-        artifacts=artifacts,
-        sandbox=sandbox,
-        dependency_cache=dependency_cache,
-    ):
-        raise typer.Exit(1)
+            if precompilation_inputs:
+                artifacts.inputs.extend(precompilation_inputs)
+
+    with profiling.Profiler('code.compile'):
+        # Compile the code.
+        # Do not cache remote solutions.
+        with grading_context.cache_level(
+            grading_context.CacheLevel.NO_CACHE,
+            when=lambda: is_path_remote(code.path),
+        ):
+            if not steps_with_caching.compile(
+                commands,
+                params=sandbox_params,
+                artifacts=artifacts,
+                sandbox=sandbox,
+                dependency_cache=dependency_cache,
+            ):
+                raise typer.Exit(1)
 
     assert compiled_digest.value is not None
 
@@ -629,13 +667,13 @@ def compile_item(
             warning_stack.get_warning_stack().add_warning(code)
 
     # Create sentinel to indicate this executable is sanitized.
-    storage = package.get_cache_storage()
+    cacher = package.get_file_cacher()
     if sanitized.should_sanitize():
-        pf = storage.create_file(f'{compiled_digest.value}.san')
-        if pf is not None:
-            storage.commit_file(pf)
-    elif storage.exists(f'{compiled_digest.value}.san'):
-        storage.delete(f'{compiled_digest.value}.san')
+        cacher.set_metadata(
+            compiled_digest.value, 'compilation', CompilationMetadata(is_sanitized=True)
+        )
+    else:
+        cacher.set_metadata(compiled_digest.value, 'compilation', None)
 
     return compiled_digest.value
 
@@ -669,14 +707,20 @@ async def run_item(
         retry_index,
     )
 
-    run_log = await steps_with_caching.run(
-        prepared.command,
-        params=prepared.sandbox_params,
-        sandbox=package.get_singleton_sandbox(),
-        artifacts=prepared.artifacts,
-        dependency_cache=dependency_cache,
-        metadata=prepared.metadata,
-    )
+    with profiling.PushContext('code.run_item'):
+        # Do not cache remote solutions.
+        with grading_context.cache_level(
+            grading_context.CacheLevel.NO_CACHE,
+            when=lambda: is_path_remote(code.path),
+        ):
+            run_log = await steps_with_caching.run(
+                prepared.command,
+                params=prepared.sandbox_params,
+                sandbox=package.get_singleton_sandbox(),
+                artifacts=prepared.artifacts,
+                dependency_cache=dependency_cache,
+                metadata=prepared.metadata,
+            )
 
     # Find sanitizer logs.
     if run_log is not None and run_log.warnings:
@@ -686,7 +730,7 @@ async def run_item(
         )
         if stderr_output is not None:
             warning_stack.get_warning_stack().add_sanitizer_warning(
-                package.get_cache_storage(), code, stderr_output
+                package.get_file_cacher(), code, stderr_output
             )
     return run_log
 
@@ -773,8 +817,13 @@ async def run_communication(
         metadata=solution_prepared.metadata,
     )
 
-    return await steps_with_caching.run_coordinated(
-        interactor_run_params,
-        solution_run_params,
-        dependency_cache=package.get_dependency_cache(),
-    )
+    # Do not cache remote solutions.
+    with grading_context.cache_level(
+        grading_context.CacheLevel.NO_CACHE,
+        when=lambda: is_path_remote(solution.code.path),
+    ):
+        return await steps_with_caching.run_coordinated(
+            interactor_run_params,
+            solution_run_params,
+            dependency_cache=package.get_dependency_cache(),
+        )

{rbx_cp-0.7.0 → rbx_cp-0.9.0}/rbx/box/contest/build_contest_statements.py

@@ -6,7 +6,7 @@ from typing import Any, Dict, List, Optional, Tuple
 
 import typer
 
-from rbx import console, testing_utils
+from rbx import console, testing_utils, utils
 from rbx.box import cd, package
 from rbx.box.contest.contest_package import get_problems
 from rbx.box.contest.schema import Contest, ContestProblem, ContestStatement
@@ -156,7 +156,7 @@ def _build_problem_statements(
     console.console.print('Building problem-level statements...')
     extracted_problems = get_problems_for_statement(contest, statement)
     res = []
-    contest_cwd_absolute = pathlib.Path().resolve()
+    contest_cwd_absolute = utils.abspath(pathlib.Path())
     contest_assets = get_relative_assets(statement.path, statement.assets)
 
     extra_vars = dict(statement.override.vars if statement.override is not None else {})

{rbx_cp-0.7.0 → rbx_cp-0.9.0}/rbx/box/contest/contest_package.py

@@ -17,7 +17,7 @@ YAML_NAME = 'contest.rbx.yml'
 
 @functools.cache
 def find_contest_yaml(root: pathlib.Path = pathlib.Path()) -> Optional[pathlib.Path]:
-    root = root.resolve()
+    root = utils.abspath(root)
     contest_yaml_path = root / YAML_NAME
     while root != pathlib.PosixPath('/') and not contest_yaml_path.is_file():
         root = root.parent