inspect-ai 0.3.72__py3-none-any.whl → 0.3.73__py3-none-any.whl

inspect_ai/_util/http.py

@@ -1,99 +1,3 @@
-import glob
-import json
-import os
-import posixpath
-from http import HTTPStatus
-from http.server import SimpleHTTPRequestHandler
-from io import BytesIO
-from typing import Any
-from urllib.parse import parse_qs, urlparse
-
-from .dev import is_dev_mode
-
-
-class InspectHTTPRequestHandler(SimpleHTTPRequestHandler):
-    def __init__(self, *args: Any, directory: str, **kwargs: Any) -> None:
-        # note whether we are in dev mode (i.e. developing the package)
-        self.dev_mode = is_dev_mode()
-
-        # initialize file serving directory
-        directory = os.path.abspath(directory)
-        super().__init__(*args, directory=directory, **kwargs)
-
-    def do_GET(self) -> None:
-        if self.path.startswith("/api/events"):
-            self.handle_events()
-        else:
-            super().do_GET()
-
-    def handle_events(self) -> None:
-        """Client polls for events (e.g. dev reload) ~ every 1 second."""
-        query = parse_qs(urlparse(self.path).query)
-        params = dict(zip(query.keys(), [value[0] for value in query.values()]))
-        self.send_json(json.dumps(self.events_response(params)))
-
-    def events_response(self, params: dict[str, str]) -> list[str]:
-        """Send back a 'reload' event if we have modified source files."""
-        loaded_time = params.get("loaded_time", None)
-        return (
-            ["reload"] if loaded_time and self.should_reload(int(loaded_time)) else []
-        )
-
-    def translate_path(self, path: str) -> str:
-        """Ensure that paths don't escape self.directory."""
-        translated = super().translate_path(path)
-        if not os.path.abspath(translated).startswith(self.directory):
-            return self.directory
-        else:
-            return translated
-
-    def send_json(self, json: str | bytes) -> None:
-        if isinstance(json, str):
-            json = json.encode()
-        self.send_response(HTTPStatus.OK)
-        self.send_header("Content-type", "application/json")
-        self.end_headers()
-        self.copyfile(BytesIO(json), self.wfile)  # type: ignore
-
-    def send_response(self, code: int, message: str | None = None) -> None:
-        """No client side or proxy caches."""
-        super().send_response(code, message)
-        self.send_header("Expires", "Fri, 01 Jan 1990 00:00:00 GMT")
-        self.send_header("Pragma", "no-cache")
-        self.send_header(
-            "Cache-Control", "no-cache, no-store, max-age=0, must-revalidate"
-        )
-
-    def guess_type(self, path: str | os.PathLike[str]) -> str:
-        _, ext = posixpath.splitext(path)
-        if not ext or ext == ".mjs" or ext == ".js":
-            return "application/javascript"
-        elif ext == ".md":
-            return "text/markdown"
-        else:
-            return super().guess_type(path)
-
-    def log_error(self, format: str, *args: Any) -> None:
-        if self.dev_mode:
-            super().log_error(format, *args)
-
-    def log_request(self, code: int | str = "-", size: int | str = "-") -> None:
-        """Don't log status 200 or 404 (too chatty)."""
-        if code not in [200, 404]:
-            super().log_request(code, size)
-
-    def should_reload(self, loaded_time: int) -> bool:
-        if self.dev_mode:
-            for dir in self.reload_dirs():
-                files = [
-                    os.stat(file).st_mtime
-                    for file in glob.glob(f"{dir}/**/*", recursive=True)
-                ]
-                last_modified = max(files) * 1000
-                if last_modified > loaded_time:
-                    return True
-
-        return False
-
-    def reload_dirs(self) -> list[str]:
-        return [self.directory]
+# see https://cloud.google.com/storage/docs/retry-strategy
+def is_retryable_http_status(status_code: int) -> bool:
+    return status_code in [408, 429] or (500 <= status_code < 600)

inspect_ai/_util/httpx.py

@@ -0,0 +1,60 @@
+import logging
+from typing import Callable
+
+from httpx import ConnectError, ConnectTimeout, HTTPStatusError, ReadTimeout
+from tenacity import RetryCallState
+
+from inspect_ai._util.constants import HTTP
+
+logger = logging.getLogger(__name__)
+
+
+def httpx_should_retry(ex: BaseException) -> bool:
+    """Check whether an exception raised from httpx should be retried.
+
+    Implements the strategy described here: https://cloud.google.com/storage/docs/retry-strategy
+
+    Args:
+      ex (BaseException): Exception to examine for retry behavior
+
+    Returns:
+      True if a retry should occur
+    """
+    # httpx status exception
+    if isinstance(ex, HTTPStatusError):
+        # request timeout
+        if ex.response.status_code == 408:
+            return True
+        # lock timeout
+        elif ex.response.status_code == 409:
+            return True
+        # rate limit
+        elif ex.response.status_code == 429:
+            return True
+        # internal errors
+        elif ex.response.status_code >= 500:
+            return True
+        else:
+            return False
+
+    # connection error
+    elif is_httpx_connection_error(ex):
+        return True
+
+    # don't retry
+    else:
+        return False
+
+
+def log_httpx_retry_attempt(context: str) -> Callable[[RetryCallState], None]:
+    def log_attempt(retry_state: RetryCallState) -> None:
+        logger.log(
+            HTTP,
+            f"{context} connection retry {retry_state.attempt_number} after waiting for {retry_state.idle_for}",
+        )
+
+    return log_attempt
+
+
+def is_httpx_connection_error(ex: BaseException) -> bool:
+    return isinstance(ex, ConnectTimeout | ConnectError | ConnectionError | ReadTimeout)

inspect_ai/_util/interrupt.py

@@ -1,4 +1,4 @@
-import asyncio
+import anyio
 
 from .working import check_sample_working_limit
 
@@ -9,7 +9,7 @@ def check_sample_interrupt() -> None:
     # check for user interrupt
     sample = sample_active()
     if sample and sample.interrupt_action:
-        raise asyncio.CancelledError()
+        raise anyio.get_cancelled_exc_class()
 
     # check for working_limit
     check_sample_working_limit()

inspect_ai/_util/json.py

@@ -1,13 +1,13 @@
-from typing import Any, Literal, cast
+from typing import (
+    Any,
+    Literal,
+    cast,
+)
 
 import jsonpatch
 from pydantic import BaseModel, Field, JsonValue
 from pydantic_core import to_jsonable_python
 
-JSONType = Literal["string", "integer", "number", "boolean", "array", "object", "null"]
-
-PythonType = Literal["str", "int", "float", "bool", "list", "dict", "None"]
-
 
 def jsonable_python(x: Any) -> Any:
     return to_jsonable_python(x, exclude_none=True, fallback=lambda _x: None)
@@ -23,53 +23,6 @@ def jsonable_dict(x: Any) -> dict[str, JsonValue]:
         )
 
 
-def python_type_to_json_type(python_type: str | None) -> JSONType:
-    match python_type:
-        case "str":
-            return "string"
-        case "int":
-            return "integer"
-        case "float":
-            return "number"
-        case "bool":
-            return "boolean"
-        case "list":
-            return "array"
-        case "dict":
-            return "object"
-        case "None":
-            return "null"
-        # treat 'unknown' as string as anything can be converted to string
-        case None:
-            return "string"
-        case _:
-            raise ValueError(
-                f"Unsupported type: {python_type} for Python to JSON conversion."
-            )
-
-
-def json_type_to_python_type(json_type: str) -> PythonType:
-    match json_type:
-        case "string":
-            return "str"
-        case "integer":
-            return "int"
-        case "number":
-            return "float"
-        case "boolean":
-            return "bool"
-        case "array":
-            return "list"
-        case "object":
-            return "dict"
-        case "null":
-            return "None"
-        case _:
-            raise ValueError(
-                f"Unsupported type: {json_type} for JSON to Python conversion."
-            )
-
-
 class JsonChange(BaseModel):
     """Describes a change to data using JSON Patch format."""
 

inspect_ai/_util/logger.py

@@ -1,8 +1,6 @@
 import atexit
 import os
-import re
 from logging import (
-    DEBUG,
     INFO,
     WARNING,
     FileHandler,
@@ -44,10 +42,12 @@ TRACE_FILE_NAME = "trace.log"
 
 # log handler that filters messages to stderr and the log file
 class LogHandler(RichHandler):
-    def __init__(self, levelno: int, transcript_levelno: int) -> None:
-        super().__init__(levelno, console=rich.get_console())
+    def __init__(
+        self, capture_levelno: int, display_levelno: int, transcript_levelno: int
+    ) -> None:
+        super().__init__(capture_levelno, console=rich.get_console())
         self.transcript_levelno = transcript_levelno
-        self.display_level = WARNING
+        self.display_level = display_levelno
         # log into an external file if requested via env var
         file_logger = os.environ.get("INSPECT_PY_LOGGER_FILE", None)
         self.file_logger = FileHandler(file_logger) if file_logger else None
@@ -77,23 +77,6 @@ class LogHandler(RichHandler):
 
     @override
     def emit(self, record: LogRecord) -> None:
-        # demote httpx and return notifications to log_level http
-        if (
-            record.name == "httpx"
-            or "http" in record.name
-            or "Retrying request" in record.getMessage()
-        ):
-            record.levelno = HTTP
-            record.levelname = HTTP_LOG_LEVEL
-
-        # skip httpx event loop is closed errors
-        if "Event loop is closed" in record.getMessage():
-            return
-
-        # skip google-genai AFC message
-        if "AFC is enabled with max remote calls" in record.getMessage():
-            return
-
         # write to stderr if we are at or above the threshold
         if record.levelno >= self.display_level:
             super().emit(record)
@@ -110,10 +93,9 @@ class LogHandler(RichHandler):
         if self.trace_logger and record.levelno >= self.trace_logger_level:
             self.trace_logger.emit(record)
 
-        # eval log always gets info level and higher records
-        # eval log only gets debug or http if we opt-in
-        write = record.levelno >= self.transcript_levelno
-        notify_logger_record(record, write)
+        # eval log gets transcript level or higher
+        if record.levelno >= self.transcript_levelno:
+            log_to_transcript(record)
 
     @override
     def render_message(self, record: LogRecord, message: str) -> ConsoleRenderable:
@@ -122,9 +104,7 @@ class LogHandler(RichHandler):
 
 # initialize logging -- this function can be called multiple times
 # in the lifetime of the process (the levelno will update globally)
-def init_logger(
-    log_level: str | None = None, log_level_transcript: str | None = None
-) -> None:
+def init_logger(log_level: str | None, log_level_transcript: str | None = None) -> None:
     # backwards compatibility for 'tools'
     if log_level == "sandbox" or log_level == "tools":
         log_level = "trace"
@@ -146,7 +126,7 @@ def init_logger(
     ).upper()
     validate_level("log level", log_level)
 
-    # reolve log file level
+    # reolve transcript log level
     log_level_transcript = (
         log_level_transcript
         if log_level_transcript
@@ -158,76 +138,40 @@ def init_logger(
     levelno = getLevelName(log_level)
     transcript_levelno = getLevelName(log_level_transcript)
 
+    # set capture level for our logs (we won't actually display/write all of them)
+    capture_level = min(TRACE, levelno, transcript_levelno)
+
     # init logging handler on demand
     global _logHandler
-    removed_root_handlers = False
     if not _logHandler:
-        removed_root_handlers = remove_non_pytest_root_logger_handlers()
-        _logHandler = LogHandler(min(DEBUG, levelno), transcript_levelno)
-        getLogger().addHandler(_logHandler)
-
-    # establish default capture level
-    capture_level = min(TRACE, levelno, transcript_levelno)
-
-    # see all the messages (we won't actually display/write all of them)
-    getLogger().setLevel(capture_level)
-    getLogger(PKG_NAME).setLevel(capture_level)
-    getLogger("httpx").setLevel(capture_level)
-    getLogger("botocore").setLevel(DEBUG)
-
-    if removed_root_handlers:
-        getLogger(PKG_NAME).warning(
-            "Inspect removed pre-existing root logger handlers and replaced them with its own handler."
+        _logHandler = LogHandler(
+            capture_levelno=capture_level,
+            display_levelno=levelno,
+            transcript_levelno=transcript_levelno,
         )
 
-    # set the levelno on the global handler
-    _logHandler.display_level = levelno
+        # set the log level for our package
+        getLogger(PKG_NAME).setLevel(capture_level)
+        getLogger(PKG_NAME).addHandler(_logHandler)
+        getLogger(PKG_NAME).propagate = False
 
+        # add our logger to the global handlers
+        getLogger().addHandler(_logHandler)
 
-_logHandler: LogHandler | None = None
+        # httpx currently logs all requests at the INFO level
+        # this is a bit aggressive and we already do this at
+        # our own HTTP level
+        getLogger("httpx").setLevel(WARNING)
 
 
-def remove_non_pytest_root_logger_handlers() -> bool:
-    root_logger = getLogger()
-    non_pytest_handlers = [
-        handler
-        for handler in root_logger.handlers
-        if handler.__module__ != "_pytest.logging"
-    ]
-    for handler in non_pytest_handlers:
-        root_logger.removeHandler(handler)
-    return len(non_pytest_handlers) > 0
+_logHandler: LogHandler | None = None
 
 
-def notify_logger_record(record: LogRecord, write: bool) -> None:
+def log_to_transcript(record: LogRecord) -> None:
     from inspect_ai.log._message import LoggingMessage
     from inspect_ai.log._transcript import LoggerEvent, transcript
 
-    if write:
-        transcript()._event(
-            LoggerEvent(message=LoggingMessage._from_log_record(record))
-        )
-    global _rate_limit_count
-    if (record.levelno <= INFO and re.search(r"\b429\b", record.getMessage())) or (
-        record.levelno == DEBUG
-        # See https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html#validating-retry-attempts
-        # for boto retry logic / log messages (this is tracking standard or adapative retries)
-        and "botocore.retries.standard" in record.name
-        and "Retry needed, retrying request after delay of:" in record.getMessage()
-    ):
-        _rate_limit_count = _rate_limit_count + 1
-
-
-_rate_limit_count = 0
-
-
-def init_http_rate_limit_count() -> None:
-    global _rate_limit_count
-    _rate_limit_count = 0
-
-
-def http_rate_limit_count() -> int:
-    return _rate_limit_count
+    transcript()._event(LoggerEvent(message=LoggingMessage._from_log_record(record)))
 
 
 def warn_once(logger: Logger, message: str) -> None:

inspect_ai/_util/retry.py

@@ -1,67 +1,16 @@
-import logging
-from typing import Callable
+_http_retries_count: int = 0
 
-from httpx import ConnectError, ConnectTimeout, HTTPStatusError, ReadTimeout
-from tenacity import RetryCallState
 
-from inspect_ai._util.constants import HTTP
+def report_http_retry() -> None:
+    from inspect_ai.log._samples import report_active_sample_retry
 
-logger = logging.getLogger(__name__)
+    # bump global counter
+    global _http_retries_count
+    _http_retries_count = _http_retries_count + 1
 
+    # report sample retry
+    report_active_sample_retry()
 
-def httpx_should_retry(ex: BaseException) -> bool:
-    """Check whether an exception raised from httpx should be retried.
 
-    Implements the strategy described here: https://cloud.google.com/storage/docs/retry-strategy
-
-    Args:
-      ex (BaseException): Exception to examine for retry behavior
-
-    Returns:
-      True if a retry should occur
-    """
-    # httpx status exception
-    if isinstance(ex, HTTPStatusError):
-        # request timeout
-        if ex.response.status_code == 408:
-            return True
-        # lock timeout
-        elif ex.response.status_code == 409:
-            return True
-        # rate limit
-        elif ex.response.status_code == 429:
-            return True
-        # internal errors
-        elif ex.response.status_code >= 500:
-            return True
-        else:
-            return False
-
-    # connection error
-    elif is_httpx_connection_error(ex):
-        return True
-
-    # don't retry
-    else:
-        return False
-
-
-def log_rate_limit_retry(context: str, retry_state: RetryCallState) -> None:
-    logger.log(
-        HTTP,
-        f"{context} rate limit retry {retry_state.attempt_number} after waiting for {retry_state.idle_for}",
-    )
-
-
-def log_retry_attempt(context: str) -> Callable[[RetryCallState], None]:
-    def log_attempt(retry_state: RetryCallState) -> None:
-        logger.log(
-            HTTP,
-            f"{context} connection retry {retry_state.attempt_number} after waiting for {retry_state.idle_for}",
-        )
-
-    return log_attempt
-
-
-def is_httpx_connection_error(ex: BaseException) -> bool:
-    return isinstance(ex, ConnectTimeout | ConnectError | ConnectionError | ReadTimeout)
+def http_retries_count() -> int:
+    return _http_retries_count

inspect_ai/_util/trace.py

@@ -1,4 +1,3 @@
-import asyncio
 import datetime
 import gzip
 import json
@@ -13,6 +12,7 @@ from logging import FileHandler, Logger
 from pathlib import Path
 from typing import Any, Callable, Generator, Literal, TextIO
 
+import anyio
 import jsonlines
 from pydantic import BaseModel, Field, JsonValue
 from shortuuid import uuid
@@ -83,7 +83,7 @@ def trace_action(
                 "duration": duration,
             },
         )
-    except (KeyboardInterrupt, asyncio.CancelledError):
+    except (KeyboardInterrupt, anyio.get_cancelled_exc_class()):
         duration = time.monotonic() - start_monotonic
         logger.log(
             TRACE,

inspect_ai/_view/server.py

@@ -1,24 +1,27 @@
 import asyncio
+import contextlib
 import logging
 import os
 import urllib.parse
 from logging import LogRecord, getLogger
 from pathlib import Path
-from typing import Any, Awaitable, Callable
+from typing import Any, AsyncIterator, Awaitable, Callable, Literal, cast
 
 import fsspec  # type: ignore
 from aiohttp import web
 from fsspec.asyn import AsyncFileSystem  # type: ignore
 from fsspec.core import split_protocol  # type: ignore
 from pydantic_core import to_jsonable_python
+from s3fs import S3FileSystem  # type: ignore
 
 from inspect_ai._display import display
 from inspect_ai._util.constants import DEFAULT_SERVER_HOST, DEFAULT_VIEW_PORT
-from inspect_ai._util.file import filesystem, size_in_mb
+from inspect_ai._util.file import default_fs_options, filesystem, size_in_mb
 from inspect_ai.log._file import (
     EvalLogInfo,
     eval_log_json,
-    list_eval_logs_async,
+    list_eval_logs,
+    log_files_from_ls,
     read_eval_log_async,
     read_eval_log_headers_async,
 )
@@ -297,6 +300,62 @@ def resolve_header_only(path: str, header_only: int | None) -> bool:
         return False
 
 
+async def list_eval_logs_async(
+    log_dir: str = os.environ.get("INSPECT_LOG_DIR", "./logs"),
+    formats: list[Literal["eval", "json"]] | None = None,
+    recursive: bool = True,
+    descending: bool = True,
+    fs_options: dict[str, Any] = {},
+) -> list[EvalLogInfo]:
+    """List all eval logs in a directory.
+
+    Will be async for filesystem providers that support async (e.g. s3, gcs, etc.)
+    otherwise will fallback to sync implementation.
+
+    Args:
+      log_dir (str): Log directory (defaults to INSPECT_LOG_DIR)
+      formats (Literal["eval", "json"]): Formats to list (default
+        to listing all formats)
+      recursive (bool): List log files recursively (defaults to True).
+      descending (bool): List in descending order.
+      fs_options (dict[str, Any]): Optional. Additional arguments to pass through
+          to the filesystem provider (e.g. `S3FileSystem`).
+
+    Returns:
+       List of EvalLog Info.
+    """
+    # async filesystem if we can
+    fs = filesystem(log_dir, fs_options)
+    if fs.is_async():
+        async with async_fileystem(log_dir, fs_options=fs_options) as async_fs:
+            if await async_fs._exists(log_dir):
+                # prevent caching of listings
+                async_fs.invalidate_cache(log_dir)
+                # list logs
+                if recursive:
+                    files: list[dict[str, Any]] = []
+                    async for _, _, filenames in async_fs._walk(log_dir, detail=True):
+                        files.extend(filenames.values())
+                else:
+                    files = cast(
+                        list[dict[str, Any]],
+                        await async_fs._ls(log_dir, detail=True),
+                    )
+                logs = [fs._file_info(file) for file in files]
+                # resolve to eval logs
+                return log_files_from_ls(logs, formats, descending)
+            else:
+                return []
+    else:
+        return list_eval_logs(
+            log_dir=log_dir,
+            formats=formats,
+            recursive=recursive,
+            descending=descending,
+            fs_options=fs_options,
+        )
+
+
 def filter_aiohttp_log() -> None:
     #  filter overly chatty /api/events messages
     class RequestFilter(logging.Filter):
@@ -329,3 +388,27 @@ def async_connection(log_file: str) -> AsyncFileSystem:
 
     # return async file-system
     return _async_connections.get(protocol)
+
+
+@contextlib.asynccontextmanager
+async def async_fileystem(
+    location: str, fs_options: dict[str, Any] = {}
+) -> AsyncIterator[AsyncFileSystem]:
+    # determine protocol
+    protocol, _ = split_protocol(location)
+    protocol = protocol or "file"
+
+    # build options
+    options = default_fs_options(location)
+    options.update(fs_options)
+
+    if protocol == "s3":
+        s3 = S3FileSystem(asynchronous=True, **options)
+        session = await s3.set_session()
+        try:
+            yield s3
+        finally:
+            await session.close()
+    else:
+        options.update({"asynchronous": True, "loop": asyncio.get_event_loop()})
+        yield fsspec.filesystem(protocol, **options)