inspect-ai 0.3.55__py3-none-any.whl → 0.3.57__py3-none-any.whl

inspect_ai/util/_sandbox/service.py

@@ -1,5 +1,6 @@
 import asyncio
 import json
+from logging import getLogger
 from pathlib import PurePosixPath
 from textwrap import dedent
 from typing import (
@@ -14,9 +15,12 @@ from inspect_ai.util._subprocess import ExecResult
 
 from .environment import SandboxEnvironment
 
+logger = getLogger(__name__)
+
+
 REQUESTS_DIR = "requests"
 RESPONSES_DIR = "responses"
-SERVICES_DIR = "/tmp/inspect-sandbox-services"
+SERVICES_DIR = "/var/tmp/sandbox-services"
 
 ID = "id"
 METHOD = "method"
@@ -70,7 +74,7 @@ class SandboxService:
 
     ```python
     import sys
-    sys.path.append("/tmp/inspect-sandbox-services/foo")
+    sys.path.append("/var/tmp/sandbox-services/foo")
     import foo
     ```
 
@@ -79,7 +83,7 @@ class SandboxService:
     ```python
     import importlib.util
     spec = importlib.util.spec_from_file_location(
-        "foo", "/tmp/inspect-sandbox-services/foo/foo.py"
+        "foo", "/var/tmp/sandbox-services/foo/foo.py"
     )
     foo = importlib.util.module_from_spec(spec)
     spec.loader.exec_module(foo)
@@ -150,8 +154,14 @@ class SandboxService:
                 f"Error reading request for service {self._name}: '{read_request}' ({result.stderr})"
             )
 
-        # parse request
-        request_data = json.loads(result.stdout)
+        # parse request (decode error could occur if its incomplete so bypass this)
+        try:
+            request_data = json.loads(result.stdout)
+        except json.JSONDecodeError:
+            logger.warning(
+                f"JSON decoding error reading service request: {result.stdout}"
+            )
+            return None
         if not isinstance(request_data, dict):
             raise TypeError(f"Service request is not a dict (type={request_data})")
 
@@ -275,7 +285,7 @@ class SandboxService:
             return request_id
 
         def _read_{self._name}_response(request_id: str) -> tuple[bool, Any]:
-            from json import load
+            from json import JSONDecodeError, load
             from pathlib import Path
 
             responses_dir = Path("{SERVICES_DIR}", "{self._name}", "{RESPONSES_DIR}")
@@ -283,7 +293,12 @@ class SandboxService:
             if response_path.exists():
                 # read and remove the file
                 with open(response_path, "r") as f:
-                    response = load(f)
+                    # it's possible the file is still being written so
+                    # just catch and wait for another retry if this occurs
+                    try:
+                        response = load(f)
+                    except JSONDecodeError:
+                        return False, None
                 response_path.unlink()
 
                 # raise error if we have one

inspect_ai/util/_store.py

@@ -34,18 +34,14 @@ class Store:
     inheriting from Pydantic `BaseModel`)
     """
 
-    def __init__(self) -> None:
-        self._data: dict[str, Any] = {}
+    def __init__(self, data: dict[str, Any] | None = None) -> None:
+        self._data = deepcopy(data) if data else {}
 
     @overload
-    def get(self, key: str, default: None = None) -> Any:
-        return self._data.get(key, default)
+    def get(self, key: str, default: None = None) -> Any: ...
 
     @overload
-    def get(self, key: str, default: VT) -> VT:
-        if key not in self._data.keys():
-            self._data[key] = default
-        return cast(VT, self._data.get(key, default))
+    def get(self, key: str, default: VT) -> VT: ...
 
     def get(self, key: str, default: VT | None = None) -> VT | Any:
         """Get a value from the store.
@@ -60,6 +56,9 @@ class Store:
         Returns:
            Value if is exists, otherwise default.
         """
+        if default is not None:
+            if key not in self._data.keys():
+                self._data[key] = default
         return cast(VT, self._data.get(key, default))
 
     def set(self, key: str, value: Any) -> None:

inspect_ai/util/_store_model.py

@@ -0,0 +1,110 @@
+from typing import Any, Type, TypeVar
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from ._store import Store, store
+
+
+class StoreModel(BaseModel):
+    """Store backed Pydandic BaseModel.
+
+    The model is initialised from a Store, so that Store should
+    either already satisfy the validation constraints of the model
+    OR you should provide Field(default=) annotations for all of
+    your model fields (the latter approach is recommended).
+    """
+
+    store: Store = Field(exclude=True, default_factory=store)
+
+    def model_post_init(self, __context: Any) -> None:
+        for name in self.model_fields.keys():
+            if name == "store":
+                continue
+            # if its in the store, then have our dict reflect that
+            ns_name = self._ns_name(name)
+            if ns_name in self.store:
+                self.__dict__[name] = self.store.get(ns_name)
+            # if its not in the store, then reflect dict into store
+            elif name in self.__dict__.keys():
+                self.store.set(ns_name, self.__dict__[name])
+
+    def __getattribute__(self, name: str) -> Any:
+        # sidestep dunders and pydantic fields
+        if name.startswith("__") or name.startswith("model_"):
+            return object.__getattribute__(self, name)
+        # handle model_fields (except 'store') by reading the store
+        elif name in object.__getattribute__(self, "model_fields") and name != "store":
+            store_key = self._ns_name(name)
+            if store_key in self.store:
+                return self.store.get(store_key)
+            else:
+                return object.__getattribute__(self, name)
+        # default to super
+        else:
+            return super().__getattribute__(name)
+
+    def __setattr__(self, name: str, value: Any) -> None:
+        if name in self.model_fields:
+            # validate with the new value (can throw ValidationError)
+            temp_data = self.store._data.copy()
+            temp_data[self._ns_name(name)] = value
+            self._validate_store(temp_data)
+
+            # update the store and sync the underlying __dict__
+            self.store.set(self._ns_name(name), value)
+            self.__dict__[name] = value
+        else:
+            super().__setattr__(name, value)
+
+    def model_dump(self, *args: Any, **kwargs: Any) -> dict[str, Any]:
+        self._sync_model()  # in case store was updated behind our back
+        return super().model_dump(*args, **kwargs)
+
+    def model_dump_json(self, *args: Any, **kwargs: Any) -> str:
+        self._sync_model()  # in case store was updated behind our back
+        return super().model_dump_json(*args, **kwargs)
+
+    def _sync_model(self) -> None:
+        self._validate_store()
+        for field_name in self.model_fields.keys():
+            if field_name == "store":
+                continue
+            store_value = self.store.get(self._ns_name(field_name))
+            self.__dict__[field_name] = store_value
+
+    def _validate_store(self, data: dict[str, Any] | None = None) -> None:
+        # validate store or custom dict
+        data = data if data is not None else self.store._data
+
+        # pick out keys to validate
+        validate: dict[str, Any] = {}
+        for k, v in data.items():
+            if k.startswith(f"{self.__class__.__name__}:"):
+                unprefixed = self._un_ns_name(k)
+                validate[unprefixed] = v
+
+        # perform validation
+        self.__class__.model_validate(validate)
+
+    def _ns_name(self, name: str) -> str:
+        return f"{self.__class__.__name__}:{name}"
+
+    def _un_ns_name(self, name: str) -> str:
+        return name.replace(f"{self.__class__.__name__}:", "", 1)
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+
+SMT = TypeVar("SMT", bound=StoreModel)
+
+
+def store_as(model_cls: Type[SMT]) -> SMT:
+    """Get a Pydantic model interface to the store.
+
+    Args:
+      model_cls: Pydantic model type (must derive from StoreModel)
+
+    Returns:
+      StoreModel: Instance of model_cls bound to current Store.
+    """
+    return model_cls(store=store())

inspect_ai/util/_subprocess.py

@@ -101,9 +101,9 @@ async def subprocess(
     input = input.encode() if isinstance(input, str) else input
 
     # function to run command (we may or may not run it w/ concurrency)
-    async def run_command() -> (
-        AsyncGenerator[Union[Process, ExecResult[str], ExecResult[bytes]], None]
-    ):
+    async def run_command() -> AsyncGenerator[
+        Union[Process, ExecResult[str], ExecResult[bytes]], None
+    ]:
         if isinstance(args, str):
             proc = await asyncio.create_subprocess_shell(
                 args,

inspect_ai/util/_throttle.py

@@ -0,0 +1,32 @@
+import time
+from functools import wraps
+from typing import Any, Callable
+
+
+def throttle(seconds: float) -> Callable[..., Any]:
+    """Throttle a function to ensure it is called no more than every n seconds.
+
+    Args:
+       seconds (float): Throttle time.
+
+    Returns:
+       Callable: Throttled function.
+    """
+
+    def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
+        last_called: float = 0
+        last_result: Any = None
+
+        @wraps(func)
+        def wrapped(*args: Any, **kwargs: Any) -> Any:
+            nonlocal last_called
+            nonlocal last_result
+            current_time = time.time()
+            if current_time - last_called >= seconds:
+                last_result = func(*args, **kwargs)
+                last_called = current_time
+            return last_result
+
+        return wrapped
+
+    return decorator

{inspect_ai-0.3.55.dist-info → inspect_ai-0.3.57.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: inspect_ai
-Version: 0.3.55
+Version: 0.3.57
 Summary: Framework for large language model evaluations
 Author: UK AI Safety Institute
 License: MIT License
@@ -67,7 +67,7 @@ Requires-Dist: pytest-asyncio; extra == "dev"
 Requires-Dist: pytest-cov; extra == "dev"
 Requires-Dist: pytest-dotenv; extra == "dev"
 Requires-Dist: pytest-xdist; extra == "dev"
-Requires-Dist: ruff==0.8.4; extra == "dev"
+Requires-Dist: ruff==0.9.0; extra == "dev"
 Requires-Dist: textual-dev>=0.86.2; extra == "dev"
 Requires-Dist: types-PyYAML; extra == "dev"
 Requires-Dist: types-beautifulsoup4; extra == "dev"