inspect-ai 0.3.81__py3-none-any.whl → 0.3.82__py3-none-any.whl

inspect_ai/_eval/task/run.py

@@ -19,7 +19,7 @@ from inspect_ai._display import (
     TaskSuccess,
     display,
 )
-from inspect_ai._display.core.display import TaskDisplay, TaskDisplayMetric
+from inspect_ai._display.core.display import TaskDisplayMetric
 from inspect_ai._util._async import tg_collect
 from inspect_ai._util.constants import (
     DEFAULT_EPOCHS,
@@ -29,6 +29,7 @@ from inspect_ai._util.constants import (
 from inspect_ai._util.datetime import iso_now
 from inspect_ai._util.error import exception_message
 from inspect_ai._util.hooks import send_telemetry
+from inspect_ai._util.json import to_json_str_safe
 from inspect_ai._util.registry import (
     is_registry_object,
     registry_log_name,
@@ -51,13 +52,17 @@ from inspect_ai.log import (
 from inspect_ai.log._condense import condense_sample
 from inspect_ai.log._file import eval_log_json_str
 from inspect_ai.log._log import EvalSampleLimit, EvalSampleReductions, eval_error
-from inspect_ai.log._samples import active_sample
+from inspect_ai.log._recorders.types import SampleSummary
+from inspect_ai.log._samples import (
+    active_sample,
+)
 from inspect_ai.log._transcript import (
     ErrorEvent,
     SampleInitEvent,
     SampleLimitEvent,
     ScoreEvent,
     StepEvent,
+    Transcript,
     transcript,
 )
 from inspect_ai.model import (
@@ -264,8 +269,13 @@ async def task_run(options: TaskRunOptions) -> EvalLog:
 
                     # track when samples complete and update progress as we go
                     progress_results: list[dict[str, SampleScore]] = []
+
+                    def update_metrics(metrics: list[TaskDisplayMetric]) -> None:
+                        td.update_metrics(metrics)
+                        logger.update_metrics(metrics)
+
                     update_metrics_display = update_metrics_display_fn(
-                        td,
+                        update_metrics,
                         display_metrics=profile.eval_config.score_display is not False,
                     )
 
@@ -423,7 +433,7 @@ async def task_run(options: TaskRunOptions) -> EvalLog:
 
 
 def update_metrics_display_fn(
-    td: TaskDisplay,
+    update_fn: Callable[[list[TaskDisplayMetric]], None],
     initial_interval: float = 0,
     min_interval: float = 0.9,
     display_metrics: bool = True,
@@ -463,7 +473,7 @@ def update_metrics_display_fn(
             )
 
             # Name, reducer, value
-            task_metrics = []
+            task_metrics: list[TaskDisplayMetric] = []
             if len(results.scores) > 0:
                 for score in results.scores:
                     for key, metric in score.metrics.items():
@@ -475,7 +485,7 @@ def update_metrics_display_fn(
                                 reducer=score.reducer,
                             )
                         )
-                td.update_metrics(task_metrics)
+                update_fn(task_metrics)
 
             # determine how long to wait before recomputing metrics
             time_end = time.perf_counter()
@@ -516,7 +526,7 @@ async def task_run_sample(
 
             # log if requested
             if logger:
-                await logger.log_sample(previous_sample, flush=False)
+                await logger.complete_sample(previous_sample, flush=False)
 
             # return score
             sample_scores = (
@@ -539,10 +549,19 @@ async def task_run_sample(
         semaphore if semaphore else contextlib.nullcontext()
     )
 
+    # validate that we have sample_id (mostly for the typechecker)
+    sample_id = sample.id
+    if sample_id is None:
+        raise ValueError("sample must have id to run")
+
     # initialise subtask and scoring context
     init_sample_model_usage()
     set_sample_state(state)
-    sample_transcript = init_subtask(SAMPLE_SUBTASK, state.store)
+    sample_transcript: Transcript = init_subtask(SAMPLE_SUBTASK, state.store)
+    if logger:
+        sample_transcript._subscribe(
+            lambda event: logger.log_sample_event(sample_id, state.epoch, event)
+        )
     if scorers:
         init_scoring_context(scorers, Target(sample.target))
 
@@ -626,6 +645,28 @@ async def task_run_sample(
                         # mark started
                         active.started = datetime.now().timestamp()
 
+                        if logger is not None:
+                            await logger.start_sample(
+                                SampleSummary(
+                                    id=sample_id,
+                                    epoch=state.epoch,
+                                    input=sample.input,
+                                    target=sample.target,
+                                )
+                            )
+
+                        # sample init event (remove file bodies as they have content or absolute paths)
+                        event_sample = sample.model_copy(
+                            update=dict(files={k: "" for k in sample.files.keys()})
+                            if sample.files
+                            else None
+                        )
+                        transcript()._event(
+                            SampleInitEvent(
+                                sample=event_sample, state=state_jsonable(state)
+                            )
+                        )
+
                         # set progress for plan then run it
                         state = await plan(state, generate)
 
@@ -824,7 +865,7 @@ async def log_sample(
     id = sample.id
     if id is None:
         raise ValueError(
-            f"Samples without IDs cannot be logged: {sample.model_dump_json()}"
+            f"Samples without IDs cannot be logged: {to_json_str_safe(sample)}"
         )
 
     # construct sample for logging
@@ -866,7 +907,7 @@ async def log_sample(
         limit=limit,
     )
 
-    await logger.log_sample(condense_sample(eval_sample, log_images), flush=True)
+    await logger.complete_sample(condense_sample(eval_sample, log_images), flush=True)
 
 
 async def resolve_dataset(

inspect_ai/_util/constants.py

@@ -25,8 +25,10 @@ ALL_LOG_LEVELS = [
 ]
 DEFAULT_LOG_LEVEL = "warning"
 DEFAULT_LOG_LEVEL_TRANSCRIPT = "info"
+DEFAULT_LOG_SHARED = 10
 ALL_LOG_FORMATS = ["eval", "json"]
 DEFAULT_LOG_FORMAT: Literal["eval", "json"] = "eval"
+JSON_LOG_FORMAT = "json"
 EVAL_LOG_FORMAT = "eval"
 DEFAULT_DISPLAY = "full"
 LOG_SCHEMA_VERSION = 2

inspect_ai/_util/file.py

@@ -13,7 +13,7 @@ from urllib.parse import urlparse
 import fsspec  # type: ignore  # type: ignore
 from fsspec.core import split_protocol  # type: ignore  # type: ignore
 from fsspec.implementations.local import make_path_posix  # type: ignore
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 from s3fs import S3FileSystem  # type: ignore
 from shortuuid import uuid
 
@@ -158,6 +158,9 @@ class FileInfo(BaseModel):
     mtime: float | None
     """File modification time (None if the file is a directory on S3)."""
 
+    etag: str | None = Field(default=None)
+    """Etag (provided by some remote filesystems)"""
+
 
 class FileSystem:
     def __init__(self, fs: Any) -> None:
@@ -178,6 +181,9 @@ class FileSystem:
     ) -> None:
         self.fs.rm(path, recursive=recursive, maxdepth=maxdepth)
 
+    def mv(self, lpath: str, rpath: str) -> None:
+        self.fs.mv(lpath, rpath)
+
     def mkdir(self, path: str, exist_ok: bool = False) -> None:
         if self.is_s3():
             # try to avoid calling create_bucket on s3 filesystems (as that requires distinct
@@ -199,6 +205,9 @@ class FileSystem:
     def info(self, path: str, **kwargs: dict[str, Any]) -> FileInfo:
         return self._file_info(self.fs.info(path, **kwargs))
 
+    def path_as_uri(self, path: str) -> str:
+        return str(self.fs.unstrip_protocol(path))
+
     def ls(
         self, path: str, recursive: bool = False, **kwargs: dict[str, Any]
     ) -> list[FileInfo]:
@@ -267,11 +276,18 @@ class FileSystem:
         else:
             file["mtime"] = None
 
+        # S3 filesystems provided an ETag
+        if "ETag" in file.keys():
+            etag: str | None = file["ETag"].strip('"')
+        else:
+            etag = None
+
         return FileInfo(
             name=file["name"],
             type=file["type"],
             size=file["size"],
             mtime=file["mtime"],
+            etag=etag,
         )
 
 

inspect_ai/_util/json.py

@@ -6,7 +6,9 @@ from typing import (
 
 import jsonpatch
 from pydantic import BaseModel, Field, JsonValue
-from pydantic_core import to_jsonable_python
+from pydantic_core import to_json, to_jsonable_python
+
+from inspect_ai.util._json import JSONType
 
 
 def jsonable_python(x: Any) -> Any:
@@ -23,6 +25,39 @@ def jsonable_dict(x: Any) -> dict[str, JsonValue]:
         )
 
 
+def to_json_safe(x: Any) -> bytes:
+    return to_json(value=x, indent=2, exclude_none=True, fallback=lambda _x: None)
+
+
+def to_json_str_safe(x: Any) -> str:
+    return to_json_safe(x).decode()
+
+
+def python_type_to_json_type(python_type: str | None) -> JSONType:
+    match python_type:
+        case "str":
+            return "string"
+        case "int":
+            return "integer"
+        case "float":
+            return "number"
+        case "bool":
+            return "boolean"
+        case "list":
+            return "array"
+        case "dict":
+            return "object"
+        case "None":
+            return "null"
+        # treat 'unknown' as string as anything can be converted to string
+        case None:
+            return "string"
+        case _:
+            raise ValueError(
+                f"Unsupported type: {python_type} for Python to JSON conversion."
+            )
+
+
 class JsonChange(BaseModel):
     """Describes a change to data using JSON Patch format."""
 

inspect_ai/_view/server.py

@@ -5,7 +5,7 @@ import os
 import urllib.parse
 from logging import LogRecord, getLogger
 from pathlib import Path
-from typing import Any, AsyncIterator, Awaitable, Callable, Literal, cast
+from typing import Any, AsyncIterator, Awaitable, Callable, Literal, TypeVar, cast
 
 import fsspec  # type: ignore
 from aiohttp import web
@@ -25,6 +25,7 @@ from inspect_ai.log._file import (
     read_eval_log_async,
     read_eval_log_headers_async,
 )
+from inspect_ai.log._recorders.buffer.buffer import sample_buffer
 
 from .notify import view_last_eval_time
 
@@ -131,6 +132,60 @@ def view_server(
         )
         return web.json_response(actions)
 
+    @routes.get("/api/pending-samples")
+    async def api_pending_samples(request: web.Request) -> web.Response:
+        # log file requested
+        file = query_param_required("log", request, str)
+
+        file = urllib.parse.unquote(file)
+        validate_log_file_request(file)
+
+        # see if there is an etag
+        client_etag = request.headers.get("If-None-Match")
+
+        # get samples and respond
+        buffer = sample_buffer(file)
+        samples = buffer.get_samples(client_etag)
+        if samples == "NotModified":
+            return web.Response(status=304)
+        elif samples is None:
+            return web.Response(status=404)
+        else:
+            return web.Response(
+                body=samples.model_dump_json(), headers={"ETag": samples.etag}
+            )
+
+    @routes.get("/api/pending-sample-data")
+    async def api_sample_events(request: web.Request) -> web.Response:
+        # log file requested
+        file = query_param_required("log", request, str)
+
+        file = urllib.parse.unquote(file)
+        validate_log_file_request(file)
+
+        # sample id information
+        id = query_param_required("id", request, str)
+        epoch = query_param_required("epoch", request, int)
+
+        # get sync info
+        after_event_id = query_param_optional("last-event-id", request, int)
+        after_attachment_id = query_param_optional("after-attachment-id", request, int)
+
+        # get samples and responsd
+        buffer = sample_buffer(file)
+        sample_data = buffer.get_sample_data(
+            id=id,
+            epoch=epoch,
+            after_event_id=after_event_id,
+            after_attachment_id=after_attachment_id,
+        )
+
+        # respond
+        if sample_data is None:
+            return web.Response(status=404)
+        else:
+            return web.Response(body=sample_data.model_dump_json())
+
     # optional auth middleware
     @web.middleware
     async def authorize(
@@ -430,3 +485,60 @@ async def async_fileystem(
     else:
         options.update({"asynchronous": True, "loop": asyncio.get_event_loop()})
         yield fsspec.filesystem(protocol, **options)
+
+
+T = TypeVar("T")  # Define type variable
+
+
+def query_param_required(
+    key: str, request: web.Request, converter: Callable[[str], T]
+) -> T:
+    """
+    Generic parameter validation function.
+
+    Args:
+        key: Parameter key to look up
+        request: aiohttp Request object
+        converter: Function to convert the string parameter to type T
+
+    Returns:
+        Converted parameter value of type T
+
+    Raises:
+        HTTPBadRequest: If parameter is missing or invalid
+    """
+    value = request.query.get(key)
+    if value is None:
+        raise web.HTTPBadRequest(text=f"Missing parameter {key}")
+
+    try:
+        return converter(value)
+    except ValueError:
+        raise web.HTTPBadRequest(text=f"Invalid value {value} for {key}")
+
+
+def query_param_optional(
+    key: str, request: web.Request, converter: Callable[[str], T]
+) -> T | None:
+    """
+    Generic parameter validation function.
+
+    Args:
+        key: Parameter key to look up
+        request: aiohttp Request object
+        converter: Function to convert the string parameter to type T
+
+    Returns:
+        Converted parameter value of type T
+
+    Raises:
+        HTTPBadRequest: If parameter is missing or invalid
+    """
+    value = request.query.get(key)
+    if value is None:
+        return None
+
+    try:
+        return converter(value)
+    except ValueError:
+        raise web.HTTPBadRequest(text=f"Invalid value {value} for {key}")

inspect_ai/_view/www/App.css

@@ -335,7 +335,7 @@ body {
   padding-top: 0;
   padding-bottom: 0;
   background-color: var(--bs-light);
-  top: 2px;
+  top: 0;
 }
 
 .navbar-title-grid {