rbx.cp 0.6.1__py3-none-any.whl → 0.8.0__py3-none-any.whl

rbx/box/stressing/finder_parser.py

@@ -10,6 +10,7 @@ import typer
 from rbx import console
 from rbx.box import package
 from rbx.box.schema import CodeItem, ExpectedOutcome
+from rbx.box.solutions import expand_solutions
 from rbx.grading.steps import CheckerResult, Outcome, RunLog, TestcaseLog
 
 LARK_GRAMMAR = r"""
@@ -26,11 +27,12 @@ negation: _NOT "(" disjunction ")"
 // Expressions
 logical: eval matcher expected_outcome -> matching
        | eval equality (eval | outcome) -> equating
+       | eval -> eval_only
 
-eval: "[" solution checking? "]"
+eval: "[" solution checking? "]" | solution
 
 // Eval
-solution: _filename | WILDCARD
+solution: _solution_filename | WILDCARD
 checking: "ON"i (checking_mode? checker | ":nil")
 checking_mode: MODE ":"
 MODE: "2" | "3"
@@ -55,6 +57,7 @@ WILDCARD: "$"
 
 // File name
 _filename: FILENAME | "\"" FILENAME "\""
+_solution_filename: _filename | "@" _filename
 FILENAME: /[\/A-Za-z0-9\-_\.]/+
 
 // Names (Variables)
@@ -217,20 +220,7 @@ def get_all_solutions(tree: lark.ParseTree) -> List[str]:
 
 def get_all_solution_items(tree: lark.ParseTree) -> List[CodeItem]:
     solution_names = get_all_solutions(tree)
-    res = []
-
-    for solution_name in solution_names:
-        found_solution = package.get_solution_or_nil(solution_name)
-        if found_solution is None:
-            res.append(
-                CodeItem(
-                    path=pathlib.Path(solution_name),
-                    language=None,
-                    compilationFiles=None,
-                )
-            )
-            continue
-        res.append(found_solution)
+    res = typing.cast(List[CodeItem], expand_solutions(solution_names))
 
     main_solution = package.get_main_solution()
     if main_solution is None:
@@ -391,6 +381,11 @@ class FinderTreeRunner(lark.Transformer):
 
         return FinderOutcome(truth_value=truth_value, results=results)
 
+    def eval_only(self, eval_result: FinderResult) -> FinderOutcome:
+        return self.matching(
+            eval_result, is_positive=True, expected_outcome=ExpectedOutcome.INCORRECT
+        )
+
     def negation(self, value: FinderOutcome) -> FinderOutcome:
         return dataclasses.replace(value, truth_value=not value.truth_value)
 

rbx/box/tasks.py

@@ -5,7 +5,8 @@ from rbx.box import checkers, package, state
 from rbx.box.code import CommunicationItem, run_communication, run_item
 from rbx.box.environment import EnvironmentSandbox, ExecutionConfig, VerificationLevel
 from rbx.box.retries import Retrier, get_retrier_config
-from rbx.box.schema import Solution, Testcase
+from rbx.box.schema import CodeItem, Testcase
+from rbx.grading import profiling
 from rbx.grading.judge.sandbox import SandboxBase
 from rbx.grading.limits import Limits
 from rbx.grading.steps import (
@@ -39,7 +40,7 @@ def get_limits_for_language(
 
 
 async def run_solution_on_testcase(
-    solution: Solution,
+    solution: CodeItem,
     compiled_digest: str,
     checker_digest: Optional[str],
     testcase: Testcase,
@@ -52,6 +53,8 @@ async def run_solution_on_testcase(
     use_timelimit: bool = True,
     capture_pipes: bool = False,
     nruns: int = 0,
+    filestem: Optional[str] = None,
+    is_stress: bool = False,
 ) -> Evaluation:
     if interactor_digest is not None:
         return await _run_communication_solution_on_testcase(
@@ -68,6 +71,8 @@ async def run_solution_on_testcase(
             use_timelimit=use_timelimit,
             capture_pipes=capture_pipes,
             nruns=nruns,
+            filestem=filestem,
+            is_stress=is_stress,
         )
 
     async def run_fn(retry_index: int) -> Evaluation:
@@ -85,29 +90,32 @@ async def run_solution_on_testcase(
             assert testcase.outputPath is not None
             output_path = testcase.outputPath
         else:
-            output_path = output_dir / testcase.inputPath.with_suffix('.out').name
+            stem = filestem or testcase.inputPath.stem
+            output_path = output_dir / pathlib.PosixPath(stem).with_suffix('.out')
         error_path = output_path.with_suffix('.err')
         log_path = output_path.with_suffix('.log')
         eval_path = output_path.with_suffix('.eval')
         output_path.parent.mkdir(parents=True, exist_ok=True)
 
-        run_log = await run_item(
-            solution,
-            DigestOrSource.create(compiled_digest),
-            stdin=DigestOrSource.create(testcase.inputPath),
-            stdout=DigestOrDest.create(output_path),
-            stderr=DigestOrDest.create(error_path),
-            extra_config=extra_config,
-            retry_index=retry_index,
-        )
+        with profiling.PushContext('tasks.run_solution_on_testcase'):
+            run_log = await run_item(
+                solution,
+                DigestOrSource.create(compiled_digest),
+                stdin=DigestOrSource.create(testcase.inputPath),
+                stdout=DigestOrDest.create(output_path),
+                stderr=DigestOrDest.create(error_path),
+                extra_config=extra_config,
+                retry_index=retry_index,
+            )
 
         if checker_digest is not None:
-            checker_result = await checkers.check(
-                checker_digest,
-                run_log,
-                testcase,
-                program_output=output_path,
-            )
+            with profiling.PushContext('tasks.run_solution_on_testcase.check'):
+                checker_result = await checkers.check(
+                    checker_digest,
+                    run_log,
+                    testcase,
+                    program_output=output_path,
+                )
         else:
             checker_result = checkers.check_with_no_output(run_log)
 
@@ -134,7 +142,7 @@ async def run_solution_on_testcase(
     if not use_retries:
         return await run_fn(0)
 
-    retrier = Retrier(get_retrier_config(nruns))
+    retrier = Retrier(get_retrier_config(nruns), is_stress=is_stress)
     return await retrier.repeat(run_fn)
 
 
@@ -156,7 +164,7 @@ def _get_execution_config(
 
 
 async def _run_communication_solution_on_testcase(
-    solution: Solution,
+    solution: CodeItem,
     compiled_digest: str,
     interactor_digest: str,
     checker_digest: Optional[str],
@@ -169,6 +177,8 @@ async def _run_communication_solution_on_testcase(
     use_timelimit: bool = True,
     capture_pipes: bool = False,
     nruns: int = 0,
+    filestem: Optional[str] = None,
+    is_stress: bool = False,
 ) -> Evaluation:
     capture_pipes = capture_pipes or state.STATE.debug_logs
 
@@ -200,7 +210,8 @@ async def _run_communication_solution_on_testcase(
             assert testcase.outputPath is not None
             output_path = testcase.outputPath
         else:
-            output_path = output_dir / testcase.inputPath.with_suffix('.out').name
+            stem = filestem or testcase.inputPath.stem
+            output_path = output_dir / pathlib.PosixPath(stem).with_suffix('.out')
         solution_error_path = output_path.with_suffix('.sol.err')
         interactor_error_path = output_path.with_suffix('.int.err')
         log_path = output_path.with_suffix('.log')
@@ -294,5 +305,5 @@ async def _run_communication_solution_on_testcase(
     if not use_retries:
         return await run_fn(0)
 
-    retrier = Retrier(get_retrier_config(nruns))
+    retrier = Retrier(get_retrier_config(nruns), is_stress=is_stress)
     return await retrier.repeat(run_fn)

rbx/box/tooling/boca/scraper.py

@@ -364,7 +364,7 @@ class BocaScraper:
         return final_path
 
 
-@functools.lru_cache
+@functools.cache
 def get_boca_scraper(
     base_url: Optional[str] = None,
     username: Optional[str] = None,

rbx/grading/caching.py

@@ -3,13 +3,18 @@ import io
 import os
 import pathlib
 import shelve
+import shutil
+import tempfile
 from typing import Any, Dict, List, Optional
 
 from pydantic import BaseModel
 
 from rbx import console
+from rbx.grading import grading_context
+from rbx.grading.judge.cacher import FileCacher
 from rbx.grading.judge.digester import digest_cooperatively
-from rbx.grading.judge.storage import Storage, copyfileobj
+from rbx.grading.judge.storage import copyfileobj
+from rbx.grading.profiling import Profiler
 from rbx.grading.steps import DigestHolder, GradingArtifacts, GradingLogsHolder
 
 VERBOSE = False
@@ -89,11 +94,15 @@ def _build_digest_list(artifacts_list: List[GradingArtifacts]) -> List[DigestHol
     return digests
 
 
-def _build_fingerprint_list(artifacts_list: List[GradingArtifacts]) -> List[str]:
+def _build_fingerprint_list(
+    artifacts_list: List[GradingArtifacts], cacher: FileCacher
+) -> List[str]:
     fingerprints = []
     for artifacts in artifacts_list:
         for input in artifacts.inputs:
-            if input.src is None:
+            if input.src is None or not input.hash:
+                continue
+            if cacher.digest_from_symlink(input.src) is not None:
                 continue
             with input.src.open('rb') as f:
                 fingerprints.append(digest_cooperatively(f))
@@ -124,9 +133,10 @@ def _build_logs_list(artifacts_list: List[GradingArtifacts]) -> List[GradingLogs
 
 def _build_cache_fingerprint(
     artifacts_list: List[GradingArtifacts],
+    cacher: FileCacher,
 ) -> CacheFingerprint:
     digests = [digest.value for digest in _build_digest_list(artifacts_list)]
-    fingerprints = _build_fingerprint_list(artifacts_list)
+    fingerprints = _build_fingerprint_list(artifacts_list, cacher)
     output_fingerprints = _build_output_fingerprint_list(artifacts_list)
     logs = _build_logs_list(artifacts_list)
     return CacheFingerprint(
@@ -155,6 +165,7 @@ def _build_cache_input(
     commands: List[str],
     artifact_list: List[GradingArtifacts],
     extra_params: Dict[str, Any],
+    cacher: FileCacher,
 ) -> CacheInput:
     cloned_artifact_list = [
         artifacts.model_copy(deep=True) for artifacts in artifact_list
@@ -164,6 +175,15 @@ def _build_cache_input(
         # part of the cache key.
         artifacts.logs = None
 
+        for input in artifacts.inputs:
+            if input.src is None:
+                continue
+            inferred_digest = cacher.digest_from_symlink(input.src)
+            if inferred_digest is not None:
+                # Consume cache from digest instead of file.
+                input.digest = DigestHolder(value=inferred_digest)
+                input.src = None
+
         for output in artifacts.outputs:
             if output.hash:
                 # Cleanup dest field from hash artifacts
@@ -185,7 +205,7 @@ def _build_cache_key(input: CacheInput) -> str:
         return digest_cooperatively(fobj)
 
 
-def _copy_hashed_files(artifact_list: List[GradingArtifacts], storage: Storage):
+def _copy_hashed_files(artifact_list: List[GradingArtifacts], cacher: FileCacher):
     for artifact in artifact_list:
         for output in artifact.outputs:
             if not output.hash or output.dest is None:
@@ -194,19 +214,27 @@ def _copy_hashed_files(artifact_list: List[GradingArtifacts], storage: Storage):
             if output.optional and output.digest.value is None:
                 continue
             assert output.digest.value is not None
-            with storage.get_file(output.digest.value) as fobj:
-                with output.dest.open('wb') as f:
-                    copyfileobj(fobj, f, maxlen=output.maxlen)
+            if (
+                path_to_symlink := cacher.path_for_symlink(output.digest.value)
+            ) is not None:
+                # Use a symlink to the file in the persistent cache, if available.
+                output.dest.unlink(missing_ok=True)
+                output.dest.symlink_to(path_to_symlink)
+            else:
+                # Otherwise, copy it.
+                with cacher.get_file(output.digest.value) as fobj:
+                    with output.dest.open('wb') as f:
+                        copyfileobj(fobj, f, maxlen=output.maxlen)
             if output.executable:
                 output.dest.chmod(0o755)
 
 
-def is_artifact_ok(artifact: GradingArtifacts, storage: Storage) -> bool:
+def is_artifact_ok(artifact: GradingArtifacts, cacher: FileCacher) -> bool:
     for output in artifact.outputs:
         if output.optional or output.intermediate:
             continue
         if output.digest is not None:
-            if output.digest.value is None or not storage.exists(output.digest.value):
+            if output.digest.value is None or not cacher.exists(output.digest.value):
                 return False
             return True
         assert output.dest is not None
@@ -219,9 +247,9 @@ def is_artifact_ok(artifact: GradingArtifacts, storage: Storage) -> bool:
     return True
 
 
-def are_artifacts_ok(artifacts: List[GradingArtifacts], storage: Storage) -> bool:
+def are_artifacts_ok(artifacts: List[GradingArtifacts], cacher: FileCacher) -> bool:
     for artifact in artifacts:
-        if not is_artifact_ok(artifact, storage):
+        if not is_artifact_ok(artifact, cacher):
             return False
     return True
 
@@ -244,53 +272,70 @@ class DependencyCacheBlock:
         self._key = None
 
     def __enter__(self):
-        input = _build_cache_input(
-            commands=self.commands,
-            artifact_list=self.artifact_list,
-            extra_params=self.extra_params,
-        )
-        if VERBOSE:
-            console.console.log(f'Cache input is: {input}')
-        self._key = _build_cache_key(input)
-        if VERBOSE:
-            console.console.log(f'Cache key is: {self._key}')
-        found = self.cache.find_in_cache(
-            self.commands, self.artifact_list, self.extra_params, key=self._key
-        )
-        return found
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        if exc_type is None:
-            self.cache.store_in_cache(
+        with Profiler('enter_in_cache'):
+            if grading_context.is_no_cache():
+                return False
+            input = _build_cache_input(
+                commands=self.commands,
+                artifact_list=self.artifact_list,
+                extra_params=self.extra_params,
+                cacher=self.cache.cacher,
+            )
+            if VERBOSE:
+                console.console.log(f'Cache input is: {input}')
+            self._key = _build_cache_key(input)
+            if VERBOSE:
+                console.console.log(f'Cache key is: {self._key}')
+            found = self.cache.find_in_cache(
                 self.commands, self.artifact_list, self.extra_params, key=self._key
             )
-        if exc_type is NoCacheException:
-            return True
-        return None
+            return found
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        with Profiler('exit_in_cache'):
+            if grading_context.is_no_cache():
+                return True if exc_type is NoCacheException else None
+            if exc_type is None:
+                self.cache.store_in_cache(
+                    self.commands, self.artifact_list, self.extra_params, key=self._key
+                )
+            if exc_type is NoCacheException:
+                return True
+            return None
 
 
 class DependencyCache:
     root: pathlib.Path
-    storage: Storage
+    cacher: FileCacher
 
-    def __init__(self, root: pathlib.Path, storage: Storage):
+    def __init__(self, root: pathlib.Path, cacher: FileCacher):
         self.root = root
-        self.storage = storage
+        self.cacher = cacher
         self.db = shelve.open(self._cache_name())
+        tmp_dir = pathlib.Path(tempfile.mkdtemp())
+        self.transient_db = shelve.open(tmp_dir / '.cache_db')
         atexit.register(lambda: self.db.close())
+        atexit.register(lambda: self.transient_db.close())
+        atexit.register(lambda: shutil.rmtree(tmp_dir))
 
     def _cache_name(self) -> str:
         return str(self.root / '.cache_db')
 
+    def get_db(self) -> shelve.Shelf:
+        if grading_context.is_transient():
+            return self.transient_db
+        return self.db
+
     def _find_in_cache(self, key: str) -> Optional[CacheFingerprint]:
-        return self.db.get(key)
+        return self.get_db().get(key)
 
     def _store_in_cache(self, key: str, fingerprint: CacheFingerprint):
-        self.db[key] = fingerprint
+        self.get_db()[key] = fingerprint
 
     def _evict_from_cache(self, key: str):
-        if key in self.db:
-            del self.db[key]
+        db = self.get_db()
+        if key in db:
+            del db[key]
 
     def __call__(
         self,
@@ -309,7 +354,10 @@ class DependencyCache:
         key: Optional[str] = None,
     ) -> bool:
         input = _build_cache_input(
-            commands=commands, artifact_list=artifact_list, extra_params=extra_params
+            commands=commands,
+            artifact_list=artifact_list,
+            extra_params=extra_params,
+            cacher=self.cacher,
         )
         key = key or _build_cache_key(input)
 
@@ -317,7 +365,7 @@ class DependencyCache:
         if fingerprint is None:
             return False
 
-        reference_fingerprint = _build_cache_fingerprint(artifact_list)
+        reference_fingerprint = _build_cache_fingerprint(artifact_list, self.cacher)
 
         if not _fingerprints_match(fingerprint, reference_fingerprint):
             self._evict_from_cache(key)
@@ -334,7 +382,7 @@ class DependencyCache:
         for digest, reference_digest in zip(fingerprint.digests, reference_digests):
             reference_digest.value = digest
 
-        if not are_artifacts_ok(artifact_list, self.storage):
+        if not are_artifacts_ok(artifact_list, self.cacher):
             # Rollback digest changes.
             for old_digest_value, reference_digest in zip(
                 old_digest_values, reference_digests
@@ -344,7 +392,7 @@ class DependencyCache:
             return False
 
         # Copy hashed files to file system.
-        _copy_hashed_files(artifact_list, self.storage)
+        _copy_hashed_files(artifact_list, self.cacher)
 
         # Apply logs changes.
         for logs, reference_logs in zip(fingerprint.logs, reference_fingerprint.logs):
@@ -366,11 +414,14 @@ class DependencyCache:
         key: Optional[str] = None,
     ):
         input = _build_cache_input(
-            commands=commands, artifact_list=artifact_list, extra_params=extra_params
+            commands=commands,
+            artifact_list=artifact_list,
+            extra_params=extra_params,
+            cacher=self.cacher,
         )
         key = key or _build_cache_key(input)
 
-        if not are_artifacts_ok(artifact_list, self.storage):
+        if not are_artifacts_ok(artifact_list, self.cacher):
             return
 
-        self._store_in_cache(key, _build_cache_fingerprint(artifact_list))
+        self._store_in_cache(key, _build_cache_fingerprint(artifact_list, self.cacher))

rbx/grading/debug_context.py

@@ -0,0 +1,31 @@
+import contextvars
+import dataclasses
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class DebugContext:
+    enable: bool = False
+
+
+debug_var = contextvars.ContextVar('debug', default=DebugContext())
+
+
+def get_debug_context() -> DebugContext:
+    return debug_var.get()
+
+
+class Debug:
+    def __init__(self, *args, **kwargs):
+        self.args = args
+        self.kwargs = kwargs
+        self.token = None
+
+    def __enter__(self):
+        self.token = debug_var.set(
+            dataclasses.replace(debug_var.get(), *self.args, **self.kwargs)
+        )
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if self.token is not None:
+            debug_var.reset(self.token)

rbx/grading/grading_context.py

@@ -0,0 +1,96 @@
+import contextvars
+from enum import Enum
+from typing import Callable, Optional, Union
+
+Condition = Union[bool, Callable[[], bool]]
+
+
+class ConditionedContext:
+    def __init__(self, when: Condition = True):
+        self.when = when
+
+    def should_enter(self) -> bool:
+        if isinstance(self.when, bool):
+            return self.when
+        return self.when()
+
+
+class CacheLevel(Enum):
+    NO_CACHE = 0
+    CACHE_TRANSIENTLY = 1
+    CACHE_COMPILATION = 2
+    CACHE_ALL = 3
+
+
+cache_level_var = contextvars.ContextVar('cache_level', default=CacheLevel.CACHE_ALL)
+
+
+def is_compilation_only() -> bool:
+    return cache_level_var.get() == CacheLevel.CACHE_COMPILATION
+
+
+def is_transient() -> bool:
+    return cache_level_var.get().value <= CacheLevel.CACHE_TRANSIENTLY.value
+
+
+def is_no_cache() -> bool:
+    return cache_level_var.get().value <= CacheLevel.NO_CACHE.value
+
+
+class cache_level(ConditionedContext):
+    def __init__(self, level: CacheLevel, when: Condition = True):
+        super().__init__(when)
+        self.level = level
+        self.token = None
+
+    def __enter__(self):
+        if self.should_enter():
+            self.token = cache_level_var.set(self.level)
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.token is not None:
+            cache_level_var.reset(self.token)
+        return None
+
+
+compression_level_var = contextvars.ContextVar('compression_level', default=5)
+use_compression_var = contextvars.ContextVar('use_compression', default=False)
+
+
+def get_compression_level() -> int:
+    return compression_level_var.get()
+
+
+def should_compress() -> bool:
+    return use_compression_var.get()
+
+
+class compression(ConditionedContext):
+    def __init__(
+        self,
+        level: Optional[int] = None,
+        use_compression: Optional[bool] = None,
+        when: Condition = True,
+    ):
+        super().__init__(when)
+        self.level = level
+        self.use_compression = use_compression
+        self.level_token = None
+        self.use_compression_token = None
+
+    def __enter__(self):
+        if not self.should_enter():
+            return self
+        if self.level is not None:
+            self.level_token = compression_level_var.set(self.level)
+        if self.use_compression is not None:
+            self.use_compression_token = use_compression_var.set(self.use_compression)
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.level_token is not None:
+            compression_level_var.reset(self.level_token)
+        if self.use_compression_token is not None:
+            use_compression_var.reset(self.use_compression_token)
+        return None