inspect-ai 0.3.55__py3-none-any.whl → 0.3.57__py3-none-any.whl

inspect_ai/_util/deprecation.py

@@ -174,7 +174,7 @@ def default_deprecation_msg(
 
         _qual = getattr(obj, "__qualname__", "") or ""
         if _qual.endswith(".__init__") or _qual.endswith(".__new__"):
-            _obj = f' class ({_qual.rsplit(".", 1)[0]})'
+            _obj = f" class ({_qual.rsplit('.', 1)[0]})"
         elif _qual and _obj:
             _obj += f" ({_qual})"
 

inspect_ai/_util/format.py

@@ -26,3 +26,10 @@ def format_value(value: object, width: int) -> str:
     elif isinstance(value, list | tuple | dict):
         return pprint.pformat(value, width=width)
     return str(value)
+
+
+def format_progress_time(time: float, pad_hours: bool = True) -> str:
+    minutes, seconds = divmod(time, 60)
+    hours, minutes = divmod(minutes, 60)
+    hours_fmt = f"{hours:2.0f}" if pad_hours else f"{hours:.0f}"
+    return f"{hours_fmt}:{minutes:02.0f}:{seconds:02.0f}"

inspect_ai/_util/json.py

@@ -103,10 +103,20 @@ def json_changes(
                 paths = json_change.path.split("/")[1:]
                 replaced = before
                 for path in paths:
-                    index: Any = int(path) if path.isnumeric() else path
+                    decoded_path = decode_json_pointer_segment(path)
+                    index: Any = (
+                        int(decoded_path) if decoded_path.isnumeric() else decoded_path
+                    )
                     replaced = replaced[index]
                 json_change.replaced = replaced
             changes.append(json_change)
         return changes
     else:
         return None
+
+
+def decode_json_pointer_segment(segment: str) -> str:
+    """Decode a single JSON Pointer segment."""
+    # JSON points encode ~ and / because they are special characters
+    # this decodes these values (https://www.rfc-editor.org/rfc/rfc6901)
+    return segment.replace("~1", "/").replace("~0", "~")

inspect_ai/_util/logger.py

@@ -1,4 +1,6 @@
+import atexit
 import os
+import re
 from logging import (
     DEBUG,
     INFO,
@@ -30,7 +32,12 @@ from .constants import (
     TRACE_LOG_LEVEL,
 )
 from .error import PrerequisiteError
-from .trace import TraceFileHandler, TraceFormatter, inspect_trace_dir
+from .trace import (
+    TraceFormatter,
+    compress_trace_log,
+    inspect_trace_file,
+    rotate_trace_files,
+)
 
 TRACE_FILE_NAME = "trace.log"
 
@@ -56,19 +63,13 @@ class LogHandler(RichHandler):
         else:
             self.file_logger_level = 0
 
-        # add a trace handler
-        default_trace_file = inspect_trace_dir() / TRACE_FILE_NAME
-        have_existing_trace_file = default_trace_file.exists()
+        # add a trace file handler
+        rotate_trace_files()  # remove oldest if > 10 trace files
         env_trace_file = os.environ.get("INSPECT_TRACE_FILE", None)
-        trace_file = Path(env_trace_file) if env_trace_file else default_trace_file
-        trace_total_files = 10
-        self.trace_logger = TraceFileHandler(
-            trace_file.as_posix(),
-            backupCount=trace_total_files - 1,  # exclude the current file (10 total)
-        )
+        trace_file = Path(env_trace_file) if env_trace_file else inspect_trace_file()
+        self.trace_logger = FileHandler(trace_file)
         self.trace_logger.setFormatter(TraceFormatter())
-        if have_existing_trace_file:
-            self.trace_logger.doRollover()
+        atexit.register(compress_trace_log(self.trace_logger))
 
         # set trace level
         trace_level = os.environ.get("INSPECT_TRACE_LEVEL", TRACE_LOG_LEVEL)
@@ -182,7 +183,7 @@ def notify_logger_record(record: LogRecord, write: bool) -> None:
     if write:
         transcript()._event(LoggerEvent(message=LoggingMessage.from_log_record(record)))
     global _rate_limit_count
-    if (record.levelno <= INFO and "429" in record.getMessage()) or (
+    if (record.levelno <= INFO and re.search(r"\b429\b", record.getMessage())) or (
         record.levelno == DEBUG
         # See https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html#validating-retry-attempts
         # for boto retry logic / log messages (this is tracking standard or adapative retries)

inspect_ai/_util/throttle.py

@@ -3,7 +3,16 @@ from functools import wraps
 from typing import Any, Callable
 
 
-def throttle(seconds: int) -> Callable[..., Any]:
+def throttle(seconds: float) -> Callable[..., Any]:
+    """Throttle a function to ensure it is called no more than every n seconds.
+
+    Args:
+       seconds (float): Throttle time.
+
+    Returns:
+       Callable: Throttled function.
+    """
+
     def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
         last_called: float = 0
         last_result: Any = None

inspect_ai/_util/trace.py

@@ -8,10 +8,10 @@ import shutil
 import time
 import traceback
 from contextlib import contextmanager
-from logging import Logger
-from logging.handlers import RotatingFileHandler
+from dataclasses import dataclass
+from logging import FileHandler, Logger
 from pathlib import Path
-from typing import Any, Generator, Literal, TextIO
+from typing import Any, Callable, Generator, Literal, TextIO
 
 import jsonlines
 from pydantic import BaseModel, Field, JsonValue
@@ -25,14 +25,33 @@ def inspect_trace_dir() -> Path:
     return inspect_data_dir("traces")
 
 
+def inspect_trace_file() -> Path:
+    return inspect_trace_dir() / f"trace-{os.getpid()}.log"
+
+
 @contextmanager
 def trace_action(
     logger: Logger, action: str, message: str, *args: Any, **kwargs: Any
 ) -> Generator[None, None, None]:
+    """Trace a long running or poentially unreliable action.
+
+    Trace actions for which you want to collect data on the resolution
+    (e.g. succeeded, cancelled, failed, timed out, etc.) and duration of.
+
+    Traces are written to the `TRACE` log level (which is just below
+    `HTTP` and `INFO`). List and read trace logs with `inspect trace list`
+    and related commands (see `inspect trace --help` for details).
+
+    Args:
+       logger (Logger): Logger to use for tracing (e.g. from `getLogger(__name__)`)
+       action (str): Name of action to trace (e.g. 'Model', 'Subprocess', etc.)
+       message (str): Message describing action (can be a format string w/ args or kwargs)
+       *args (Any): Positional arguments for `message` format string.
+       **kwargs (Any): Named args for `message` format string.
+    """
     trace_id = uuid()
     start_monotonic = time.monotonic()
     start_wall = time.time()
-    pid = os.getpid()
     detail = message % args if args else message % kwargs if kwargs else message
 
     def trace_message(event: str) -> str:
@@ -47,7 +66,6 @@ def trace_action(
             "event": "enter",
             "trace_id": str(trace_id),
             "start_time": start_wall,
-            "pid": pid,
         },
     )
 
@@ -63,7 +81,6 @@ def trace_action(
                 "event": "exit",
                 "trace_id": str(trace_id),
                 "duration": duration,
-                "pid": pid,
             },
         )
     except (KeyboardInterrupt, asyncio.CancelledError):
@@ -77,7 +94,6 @@ def trace_action(
                 "event": "cancel",
                 "trace_id": str(trace_id),
                 "duration": duration,
-                "pid": pid,
             },
         )
         raise
@@ -92,7 +108,6 @@ def trace_action(
                 "event": "timeout",
                 "trace_id": str(trace_id),
                 "duration": duration,
-                "pid": pid,
             },
         )
         raise
@@ -110,7 +125,6 @@ def trace_action(
                 "error": getattr(ex, "message", str(ex)) or repr(ex),
                 "error_type": type(ex).__name__,
                 "stacktrace": traceback.format_exc(),
-                "pid": pid,
             },
         )
         raise
@@ -119,6 +133,19 @@ def trace_action(
 def trace_message(
     logger: Logger, category: str, message: str, *args: Any, **kwargs: Any
 ) -> None:
+    """Log a message using the TRACE log level.
+
+    The `TRACE` log level is just below `HTTP` and `INFO`). List and
+    read trace logs with `inspect trace list` and related commands
+    (see `inspect trace --help` for details).
+
+    Args:
+       logger (Logger): Logger to use for tracing (e.g. from `getLogger(__name__)`)
+       category (str): Category of trace message.
+       message (str): Trace message (can be a format string w/ args or kwargs)
+       *args (Any): Positional arguments for `message` format string.
+       **kwargs (Any): Named args for `message` format string.
+    """
     logger.log(TRACE, f"[{category}] {message}", *args, **kwargs)
 
 
@@ -153,7 +180,6 @@ class TraceFormatter(logging.Formatter):
                 "error",
                 "error_type",
                 "stacktrace",
-                "pid",
             ]:
                 if hasattr(record, key):
                     output[key] = getattr(record, key)
@@ -215,7 +241,22 @@ class ActionTraceRecord(TraceRecord):
     error: str | None = Field(default=None)
     error_type: str | None = Field(default=None)
     stacktrace: str | None = Field(default=None)
-    pid: int | None = Field(default=None)
+
+
+@dataclass
+class TraceFile:
+    file: Path
+    mtime: float
+
+
+def list_trace_files() -> list[TraceFile]:
+    trace_files: list[TraceFile] = [
+        TraceFile(file=f, mtime=f.lstat().st_mtime)
+        for f in inspect_trace_dir().iterdir()
+        if f.is_file()
+    ]
+    trace_files.sort(key=lambda f: f.mtime, reverse=True)
+    return trace_files
 
 
 def read_trace_file(file: Path) -> list[TraceRecord]:
@@ -237,39 +278,30 @@ def read_trace_file(file: Path) -> list[TraceRecord]:
             return read_file(f)
 
 
-class TraceFileHandler(RotatingFileHandler):
-    def __init__(
-        self,
-        filename: str,
-        mode: str = "a",
-        maxBytes: int = 0,
-        backupCount: int = 0,
-        encoding: str | None = None,
-        delay: bool = False,
-    ) -> None:
-        super().__init__(filename, mode, maxBytes, backupCount, encoding, delay)
-
-    def rotation_filename(self, default_name: str) -> str:
-        """
-        Returns the name of the rotated file.
-
-        Args:
-            default_name: The default name that would be used for rotation
-
-        Returns:
-            The modified filename with .gz extension
-        """
-        return default_name + ".gz"
-
-    def rotate(self, source: str, dest: str) -> None:
-        """
-        Compresses the source file and moves it to destination.
-
-        Args:
-            source: The source file to be compressed
-            dest: The destination path for the compressed file
-        """
-        with open(source, "rb") as f_in:
-            with gzip.open(dest, "wb") as f_out:
-                shutil.copyfileobj(f_in, f_out)
-        os.remove(source)
+def rotate_trace_files() -> None:
+    # if multiple inspect processes start up at once they
+    # will all be attempting to rotate at the same time,
+    # which can lead to FileNotFoundError -- ignore these
+    # errors if they occur
+    try:
+        rotate_files = list_trace_files()[10:]
+        for file in rotate_files:
+            file.file.unlink(missing_ok=True)
+    except FileNotFoundError:
+        pass
+
+
+def compress_trace_log(log_handler: FileHandler) -> Callable[[], None]:
+    def compress() -> None:
+        # ensure log is closed
+        log_handler.close()
+
+        # compress
+        trace_file = Path(log_handler.baseFilename)
+        if trace_file.exists():
+            with open(trace_file, "rb") as f_in:
+                with gzip.open(trace_file.with_suffix(".log.gz"), "wb") as f_out:
+                    shutil.copyfileobj(f_in, f_out)
+            trace_file.unlink()
+
+    return compress

inspect_ai/_util/transcript.py

@@ -1,4 +1,5 @@
 import html
+import re
 from typing import Any
 
 from rich.align import AlignMethod
@@ -19,13 +20,43 @@ def transcript_code_theme() -> str:
 def transcript_markdown(content: str, *, escape: bool = False) -> Markdown:
     code_theme = transcript_code_theme()
     return Markdown(
-        html.escape(content) if escape else content,
+        html_escape_markdown(content) if escape else content,
         code_theme=code_theme,
         inline_code_lexer="python",
         inline_code_theme=code_theme,
     )
 
 
+def html_escape_markdown(content: str) -> str:
+    """Escape markdown lines that aren't in a code block."""
+    codeblock_pattern = re.compile("`{3,}")
+    current_codeblock = ""
+    escaped: list[str] = []
+    lines = content.splitlines()
+    for line in lines:
+        # look for matching end of codeblock
+        if current_codeblock:
+            if current_codeblock in line:
+                current_codeblock = ""
+                escaped.append(line)
+                continue
+
+        # look for beginning of codeblock
+        match = codeblock_pattern.search(line)
+        if match:
+            current_codeblock = match[0]
+            escaped.append(line)
+            continue
+
+        # escape if we are not in a codeblock
+        if current_codeblock:
+            escaped.append(line)
+        else:
+            escaped.append(html.escape(line, quote=False))
+
+    return "\n".join(escaped)
+
+
 def set_transcript_markdown_options(markdown: Markdown) -> None:
     code_theme = transcript_code_theme()
     markdown.code_theme = code_theme
@@ -89,8 +120,10 @@ def transcript_function(function: str, arguments: dict[str, Any]) -> RenderableT
     return transcript_markdown("```python\n" + call + "\n```\n")
 
 
-LINE = Box(" ── \n" "    \n" "    \n" "    \n" "    \n" "    \n" "    \n" "    \n")
+DOUBLE_LINE = Box(" ══ \n    \n    \n    \n    \n    \n    \n    \n")
+
+LINE = Box(" ── \n    \n    \n    \n    \n    \n    \n    \n")
 
-DOTTED = Box(" ·· \n" "    \n" "    \n" "    \n" "    \n" "    \n" "    \n" "    \n")
+DOTTED = Box(" ·· \n    \n    \n    \n    \n    \n    \n    \n")
 
-NOBORDER = Box("    \n" "    \n" "    \n" "    \n" "    \n" "    \n" "    \n" "    \n")
+NOBORDER = Box("    \n    \n    \n    \n    \n    \n    \n    \n")

inspect_ai/_util/vscode.py

@@ -0,0 +1,51 @@
+import os
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, Field
+from pydantic_core import to_json
+from shortuuid import uuid
+
+from .appdirs import inspect_data_dir
+
+
+class VSCodeCommand(BaseModel):
+    command: str
+    args: list[Any] = Field(default_factory=list)
+
+
+def execute_vscode_commands(commands: VSCodeCommand | list[VSCodeCommand]) -> None:
+    # resolve to list
+    commands = commands if isinstance(commands, list) else [commands]
+
+    # ensure there is someone listening
+    command_dir = vs_code_commands_dir()
+    if command_dir is None:
+        raise NotImplementedError(
+            "Not running in VS Code session or have older version of Inspect AI extension"
+        )
+
+    command_file = command_dir / uuid()
+    with open(command_file, "w") as f:
+        f.write(to_json(commands).decode())
+
+
+def can_execute_vscode_commands() -> bool:
+    return vs_code_commands_dir() is not None
+
+
+def vs_code_commands_dir() -> Path | None:
+    workspace_id = vscode_workspace_id()
+    if workspace_id:
+        workspace_dir = inspect_data_dir(os.path.join("vscode", workspace_id))
+        if workspace_dir.exists():
+            commands_dir = workspace_dir / "commands"
+            return commands_dir if commands_dir.exists() else None
+        else:
+            return None
+    else:
+        return None
+
+
+def vscode_workspace_id() -> str | None:
+    return os.environ.get("INSPECT_WORKSPACE_ID", None)

inspect_ai/_view/notify.py

@@ -4,6 +4,7 @@ from pathlib import Path
 from urllib.parse import urlparse
 
 from inspect_ai._util.appdirs import inspect_data_dir
+from inspect_ai._util.vscode import vscode_workspace_id
 
 # lightweight tracking of when the last eval task completed
 # this enables the view client to poll for changes frequently
@@ -24,7 +25,7 @@ def view_notify_eval(location: str) -> None:
         payload = {
             "location": location,
         }
-        workspace_id = os.environ.get("INSPECT_WORKSPACE_ID")
+        workspace_id = vscode_workspace_id()
         if workspace_id:
             payload["workspace_id"] = workspace_id
 

inspect_ai/_view/www/.prettierrc.js

@@ -0,0 +1,12 @@
+// Do not remove this file even if the config is empty!
+// VSCode's "Format Document" will respect this config and use the default
+// settings, which is what we want. Without prettierrc, VSCode falls back to
+// users settings, which could be different.
+
+/**
+ * @see https://prettier.io/docs/en/configuration.html
+ * @type {import("prettier").Config}
+ */
+const config = {};
+
+export default config;

inspect_ai/_view/www/App.css

@@ -235,6 +235,10 @@ body[class^="vscode-"] .sidebar .list-group {
   --bs-list-group-active-color: var(--vscode-sideBarSectionHeader-foreground);
 }
 
+body[class^="vscode-"] div.ap-control-bar .ap-fullscreen-button {
+  display: none;
+}
+
 :root {
   --bs-navbar-padding-y: 0;
   --bs-navbar-brand-padding-y: 0;
@@ -690,7 +694,7 @@ table.table.table-sm td {
 }
 
 @keyframes moveLeftToRight {
-  from {
+from {
     margin-left: 0;
   }
   to {
@@ -724,6 +728,23 @@ pre[class*="language-"].tool-output {
   border-radius: var(--bs-border-radius) !important;  
 }
 
+/* lightbox styles */
+
+
+.lightbox-overlay .close-button,
+.lightbox-overlay .nav-button {
+  /* Hide by default */
+  opacity: 0;
+  pointer-events: none; /* so it doesn't register clicks when hidden */
+  transition: opacity 0.3s ease;
+}
+
+.lightbox-overlay:hover .close-button,
+.lightbox-overlay .nav-button {
+  /* Show on hover */
+  opacity: 1;
+  pointer-events: auto;
+}
 
 /* jsondiffpatch */