inspect-ai 0.3.103__py3-none-any.whl → 0.3.105__py3-none-any.whl

inspect_ai/_eval/task/run.py

@@ -10,6 +10,7 @@ from pathlib import PurePath
 from typing import Callable, Literal
 
 import anyio
+from anyio.abc import TaskGroup
 from typing_extensions import Unpack
 
 from inspect_ai._display import (
@@ -223,7 +224,7 @@ async def task_run(options: TaskRunOptions) -> EvalLog:
         samples=len(samples),
         steps=len(samples) * SAMPLE_TOTAL_PROGRESS_UNITS,
         eval_config=config,
-        task_args=logger.eval.task_args,
+        task_args=logger.eval.task_args_passed,
         generate_config=generate_config,
         tags=tags,
         log_location=log_location,
@@ -306,37 +307,57 @@ async def task_run(options: TaskRunOptions) -> EvalLog:
                     task.metrics,
                 )
 
+                async def run_sample(
+                    sample: Sample, state: TaskState
+                ) -> dict[str, SampleScore] | None:
+                    result: dict[str, SampleScore] | None = None
+
+                    async def run(tg: TaskGroup) -> None:
+                        try:
+                            nonlocal result
+                            result = await task_run_sample(
+                                tg=tg,
+                                task_name=task.name,
+                                log_location=profile.log_location,
+                                sample=sample,
+                                state=state,
+                                sandbox=sandbox,
+                                max_sandboxes=config.max_sandboxes,
+                                sandbox_cleanup=sandbox_cleanup,
+                                plan=plan,
+                                scorers=scorers,
+                                generate=generate,
+                                progress=progress,
+                                logger=logger if log_samples else None,
+                                log_images=log_images,
+                                sample_source=sample_source,
+                                sample_error=sample_error_handler,
+                                sample_complete=sample_complete,
+                                fails_on_error=(
+                                    config.fail_on_error is None
+                                    or config.fail_on_error is True
+                                ),
+                                retry_on_error=config.retry_on_error or 0,
+                                error_retries=[],
+                                time_limit=config.time_limit,
+                                working_limit=config.working_limit,
+                                semaphore=sample_semaphore,
+                            )
+                        finally:
+                            tg.cancel_scope.cancel()
+
+                    async with anyio.create_task_group() as tg:
+                        tg.start_soon(run, tg)
+
+                    return result
+
                 sample_results = await tg_collect(
                     [
-                        functools.partial(
-                            task_run_sample,
-                            task_name=task.name,
-                            log_location=profile.log_location,
-                            sample=sample,
-                            state=state,
-                            sandbox=sandbox,
-                            max_sandboxes=config.max_sandboxes,
-                            sandbox_cleanup=sandbox_cleanup,
-                            plan=plan,
-                            scorers=scorers,
-                            generate=generate,
-                            progress=progress,
-                            logger=logger if log_samples else None,
-                            log_images=log_images,
-                            sample_source=sample_source,
-                            sample_error=sample_error_handler,
-                            sample_complete=sample_complete,
-                            fails_on_error=(
-                                config.fail_on_error is None
-                                or config.fail_on_error is True
-                            ),
-                            retry_on_error=config.retry_on_error or 0,
-                            error_retries=[],
-                            time_limit=config.time_limit,
-                            working_limit=config.working_limit,
-                            semaphore=sample_semaphore,
+                        functools.partial(run_sample, sample, state)
+                        for (sample, state) in zip(
+                            samples,
+                            states,
                         )
-                        for (sample, state) in zip(samples, states)
                     ]
                 )
 
@@ -492,6 +513,7 @@ def update_metrics_display_fn(
 
 async def task_run_sample(
     *,
+    tg: TaskGroup,
     task_name: str,
     log_location: str,
     sample: Sample,
@@ -611,12 +633,14 @@ async def task_run_sample(
             working_limit=working_limit,
             fails_on_error=fails_on_error or (retry_on_error > 0),
             transcript=sample_transcript,
+            tg=tg,
         ) as active,
     ):
         start_time: float | None = None
         error: EvalError | None = None
         raise_error: BaseException | None = None
         results: dict[str, SampleScore] = {}
+        limit: EvalSampleLimit | None = None
         try:
             # begin init
             init_span = span("init", type="init")
@@ -704,9 +728,17 @@ async def task_run_sample(
                         # handle the cancel exception
                         raise
 
-                except (LimitExceededError, TerminateSampleError):
+                except LimitExceededError as ex:
+                    # capture most recent state for scoring
+                    state = sample_state() or state
+                    limit = EvalSampleLimit(
+                        type=ex.type, limit=ex.limit if ex.limit is not None else -1
+                    )
+
+                except TerminateSampleError:
                     # capture most recent state for scoring
                     state = sample_state() or state
+                    limit = EvalSampleLimit(type="operator", limit=1)
 
                 except BaseException as ex:
                     error, raise_error = handle_error(ex)
@@ -815,6 +847,7 @@ async def task_run_sample(
                     state=state,
                     scores=results,
                     error=error,
+                    limit=limit,
                     error_retries=error_retries,
                     log_images=log_images,
                 )
@@ -854,6 +887,7 @@ async def task_run_sample(
             time_limit=time_limit,
             working_limit=working_limit,
             semaphore=semaphore,
+            tg=tg,
         )
 
     # no error
@@ -879,6 +913,7 @@ async def log_sample(
     state: TaskState,
     scores: dict[str, SampleScore],
     error: EvalError | None,
+    limit: EvalSampleLimit | None,
     error_retries: list[EvalError],
     log_images: bool,
 ) -> None:
@@ -894,15 +929,6 @@ async def log_sample(
     # compute total time if we can
     total_time = time.monotonic() - start_time if start_time is not None else None
 
-    # if a limit was hit, note that in the Eval Sample
-    limit = None
-    for e in transcript().events:
-        if e.event == "sample_limit":
-            limit = EvalSampleLimit(
-                type=e.type, limit=e.limit if e.limit is not None else -1
-            )
-            break
-
     eval_sample = EvalSample(
         id=id,
         epoch=state.epoch,

inspect_ai/_util/citation.py

@@ -0,0 +1,88 @@
+from typing import Annotated, Literal, TypeAlias, Union
+
+from pydantic import BaseModel, Discriminator, Field, JsonValue
+
+
+class CitationBase(BaseModel):
+    """Base class for citations."""
+
+    cited_text: str | tuple[int, int] | None = Field(
+        default=None,
+        # without helping the schema generator, this will turn into [unknown, unknown] in TypeScript
+        json_schema_extra={
+            "anyOf": [
+                {"type": "string"},
+                {
+                    "type": "array",
+                    "items": [{"type": "integer"}, {"type": "integer"}],
+                    "additionalItems": False,
+                    "minItems": 2,
+                    "maxItems": 2,
+                },
+                {"type": "null"},
+            ]
+        },
+    )
+    """
+    The cited text
+
+    This can be the text itself or a start/end range of the text content within
+    the container that is the cited text.
+    """
+
+    title: str | None = None
+    """Title of the cited resource."""
+
+    internal: dict[str, JsonValue] | None = Field(default=None)
+    """Model provider specific payload - typically used to aid transformation back to model types."""
+
+
+class ContentCitation(CitationBase):
+    """A generic content citation."""
+
+    type: Literal["content"] = Field(default="content")
+    """Type."""
+
+
+class DocumentRange(BaseModel):
+    """A range specifying a section of a document."""
+
+    type: Literal["block", "page", "char"]
+    """The type of the document section specified by the range."""
+
+    start_index: int
+    """0 based index of the start of the range."""
+
+    end_index: int
+    """0 based index of the end of the range."""
+
+
+class DocumentCitation(CitationBase):
+    """A citation that refers to a page range in a document."""
+
+    type: Literal["document"] = Field(default="document")
+    """Type."""
+
+    range: DocumentRange | None = Field(default=None)
+    """Range of the document that is cited."""
+
+
+class UrlCitation(CitationBase):
+    """A citation that refers to a URL."""
+
+    type: Literal["url"] = Field(default="url")
+    """Type."""
+
+    url: str
+    """URL of the cited resource."""
+
+
+Citation: TypeAlias = Annotated[
+    Union[
+        ContentCitation,
+        DocumentCitation,
+        UrlCitation,
+    ],
+    Discriminator("type"),
+]
+"""A citation sent to or received from a model."""

inspect_ai/_util/content.py

@@ -1,7 +1,9 @@
-from typing import Literal, Union
+from typing import Literal, Sequence, Union
 
 from pydantic import BaseModel, Field, JsonValue
 
+from inspect_ai._util.citation import Citation
+
 
 class ContentBase(BaseModel):
     internal: JsonValue | None = Field(default=None)
@@ -20,6 +22,9 @@ class ContentText(ContentBase):
     refusal: bool | None = Field(default=None)
     """Was this a refusal message?"""
 
+    citations: Sequence[Citation] | None = Field(default=None)
+    """Citations supporting the text block."""
+
 
 class ContentReasoning(ContentBase):
     """Reasoning content.
@@ -82,5 +87,22 @@ class ContentVideo(ContentBase):
     """Format of video data ('mp4', 'mpeg', or 'mov')"""
 
 
-Content = Union[ContentText, ContentReasoning, ContentImage, ContentAudio, ContentVideo]
+class ContentData(ContentBase):
+    """Model internal."""
+
+    type: Literal["data"] = Field(default="data")
+    """Type."""
+
+    data: dict[str, JsonValue]
+    """Model provider specific payload - required for internal content."""
+
+
+Content = Union[
+    ContentText,
+    ContentReasoning,
+    ContentImage,
+    ContentAudio,
+    ContentVideo,
+    ContentData,
+]
 """Content sent to or received from a model."""

inspect_ai/_util/json.py

@@ -6,7 +6,7 @@ from typing import (
 
 import jsonpatch
 from pydantic import BaseModel, Field, JsonValue
-from pydantic_core import to_json, to_jsonable_python
+from pydantic_core import PydanticSerializationError, to_json, to_jsonable_python
 
 JSONType = Literal["string", "integer", "number", "boolean", "array", "object", "null"]
 """Valid types within JSON schema."""
@@ -27,7 +27,22 @@ def jsonable_dict(x: Any) -> dict[str, JsonValue]:
 
 
 def to_json_safe(x: Any) -> bytes:
-    return to_json(value=x, indent=2, exclude_none=True, fallback=lambda _x: None)
+    def clean_utf8_json(obj: Any) -> Any:
+        if isinstance(obj, str):
+            return obj.encode("utf-8", errors="replace").decode("utf-8")
+        elif isinstance(obj, dict):
+            return {k: clean_utf8_json(v) for k, v in obj.items()}
+        elif isinstance(obj, list):
+            return [clean_utf8_json(item) for item in obj]
+        return obj
+
+    try:
+        return to_json(value=x, indent=2, exclude_none=True, fallback=lambda _x: None)
+    except PydanticSerializationError as ex:
+        if "surrogates not allowed" in str(ex):
+            cleaned = clean_utf8_json(x)
+            return to_json(cleaned)
+        raise
 
 
 def to_json_str_safe(x: Any) -> str:

inspect_ai/_util/registry.py

@@ -102,9 +102,26 @@ def registry_tag(
         *args (list[Any]): Creation arguments
         **kwargs (dict[str,Any]): Creation keyword arguments
     """
+    # bind arguments to params
+    named_params = extract_named_params(type, False, *args, **kwargs)
+
+    # set attribute
+    setattr(o, REGISTRY_INFO, info)
+    setattr(o, REGISTRY_PARAMS, named_params)
+
+
+def extract_named_params(
+    type: Callable[..., Any], apply_defaults: bool, *args: Any, **kwargs: Any
+) -> dict[str, Any]:
     # bind arguments to params
     named_params: dict[str, Any] = {}
-    bound_params = inspect.signature(type).bind(*args, **kwargs)
+
+    if apply_defaults:
+        bound_params = inspect.signature(type).bind_partial(*args, **kwargs)
+        bound_params.apply_defaults()
+    else:
+        bound_params = inspect.signature(type).bind(*args, **kwargs)
+
     for param, value in bound_params.arguments.items():
         named_params[param] = registry_value(value)
 
@@ -128,9 +145,7 @@ def registry_tag(
                 or "<unknown>"
             )
 
-    # set attribute
-    setattr(o, REGISTRY_INFO, info)
-    setattr(o, REGISTRY_PARAMS, named_params)
+    return named_params
 
 
 def registry_name(o: object, name: str) -> str:

inspect_ai/_view/schema.py

@@ -1,6 +1,5 @@
 import json
 import os
-import shutil
 import subprocess
 from pathlib import Path
 from typing import Any
@@ -19,9 +18,6 @@ def sync_view_schema() -> None:
     # export schema file
     schema_path = Path(WWW_DIR, "log-schema.json")
     types_path = Path(WWW_DIR, "src", "@types", "log.d.ts")
-    vs_code_types_path = Path(
-        WWW_DIR, "..", "..", "..", "..", "tools", "vscode", "src", "@types", "log.d.ts"
-    )
 
     with open(schema_path, "w", encoding="utf-8") as f:
         # make everything required
@@ -50,8 +46,6 @@ def sync_view_schema() -> None:
 
         subprocess.run(["yarn", "prettier:write"], cwd=types_path.parent, check=True)
 
-        shutil.copyfile(types_path, vs_code_types_path)
-
 
 def schema_to_strict(schema: dict[str, Any]) -> dict[str, Any]:
     properties = schema.get("properties", None)

inspect_ai/_view/server.py

@@ -155,6 +155,23 @@ def view_server(
                 body=samples.model_dump_json(), headers={"ETag": samples.etag}
             )
 
+    @routes.get("/api/log-message")
+    async def api_log_message(request: web.Request) -> web.Response:
+        # log file requested
+        file = query_param_required("log_file", request, str)
+
+        file = urllib.parse.unquote(file)
+        validate_log_file_request(file)
+
+        # message to log
+        message = query_param_required("message", request, str)
+
+        # log the message
+        logger.warning(f"[CLIENT MESSAGE] ({file}): {message}")
+
+        # respond
+        return web.Response(status=204)
+
     @routes.get("/api/pending-sample-data")
     async def api_sample_events(request: web.Request) -> web.Response:
         # log file requested

inspect_ai/_view/www/dist/assets/index.css

@@ -15655,18 +15655,40 @@ pre[class*="language-"] {
   background-color: var(--bs-light-border-subtle);
   margin-top: -1px;
 }
-._message_17kai_1 {
+._keyPairContainer_1ltuo_1 {
+  display: grid;
+  grid-template-columns: max-content auto;
+  column-gap: 0.5em;
+  padding-top: 4px;
+  padding-bottom: 4px;
+  border-bottom: solid 1px var(--bs-border-color);
+}
+
+._key_1ltuo_1 {
+  display: grid;
+  grid-template-columns: 1em auto;
+  cursor: pointer;
+}
+
+._pre_1ltuo_16 {
+  margin-bottom: 0;
+}
+
+._treeIcon_1ltuo_20 {
+  margin-top: -3px;
+}
+._message_1ivu3_1 {
   font-weight: 300;
   margin-left: 0;
   margin-right: 0;
   white-space: normal;
 }
 
-._systemRole_17kai_8 {
+._systemRole_1ivu3_8 {
   opacity: 0.7;
 }
 
-._messageGrid_17kai_12 {
+._messageGrid_1ivu3_12 {
   display: grid;
   grid-template-columns: max-content max-content max-content;
   column-gap: 0.3em;
@@ -15674,36 +15696,98 @@ pre[class*="language-"] {
   margin-bottom: 0.3em;
 }
 
-._messageContents_17kai_20 {
+._messageContents_1ivu3_20 {
   margin-left: 0;
   padding-bottom: 0;
 }
 
-._messageContents_17kai_20._indented_17kai_25 {
+._messageContents_1ivu3_20._indented_1ivu3_25 {
   margin-left: 0rem;
 }
 
-._copyLink_17kai_29 {
+._copyLink_1ivu3_29 {
   opacity: 0;
   padding-left: 0;
   padding-right: 2em;
 }
 
-._copyLink_17kai_29:hover {
+._copyLink_1ivu3_29:hover {
   opacity: 1;
 }
-._contentImage_61gdd_1 {
+
+._metadataLabel_1ivu3_39 {
+  padding-top: 1em;
+}
+._webSearch_1376z_1 {
+  display: grid;
+  grid-template-columns: max-content 1fr;
+  column-gap: 0.5em;
+  align-items: baseline;
+}
+
+._query_1376z_8 {
+  font-family: var(--bs-font-monospace);
+}
+._contentData_1sd1z_1 {
+  border: solid var(--bs-light-border-subtle) 1px;
+  padding: 0.5em;
+  margin-bottom: 0.5em;
+  margin-top: 0.5em;
+  margin-left: 1em;
+}
+._webSearch_1mixg_1 {
+  display: grid;
+  grid-template-columns: max-content 1fr;
+  column-gap: 0.5em;
+  align-items: baseline;
+}
+
+._query_1mixg_8 {
+  font-family: var(--bs-font-monospace);
+}
+
+._result_1mixg_12 a:hover {
+  text-decoration: underline;
+}
+
+._result_1mixg_12 a {
+  opacity: 0.8;
+  text-decoration: none;
+}
+._citations_t2k1z_1 {
+  margin-top: 1em;
+  margin-bottom: 1em;
+  display: grid;
+  grid-template-columns: max-content 1fr;
+  column-gap: 0.5em;
+}
+
+a._citationLink_t2k1z_9 {
+  display: block;
+  color: var(--bs-body);
+  text-decoration: none;
+}
+a._citationLink_t2k1z_9:hover {
+  text-decoration: underline;
+}
+._contentImage_8rgix_1 {
   max-width: 800px;
   border: solid var(--bs-border-color) 1px;
 }
 
-._reasoning_61gdd_6 {
+._reasoning_8rgix_6 {
   border: solid var(--bs-light-border-subtle) 1px;
   padding: 1em;
   margin-bottom: 0.5em;
   background-color: var(--bs-light-bg-subtle);
   border-radius: var(--bs-border-radius);
 }
+
+._data_8rgix_14 {
+  border: solid var(--bs-light-border-subtle) 1px;
+  padding: 1em;
+  margin-bottom: 0.5em;
+}
 ._toolImage_bv5nm_1 {
   max-width: 800px;
   border: solid var(--bs-border-color) 1px;
@@ -16741,28 +16825,6 @@ thead th {
 .card-body.card-no-padding {
   padding: 0;
 }
-._keyPairContainer_1ltuo_1 {
-  display: grid;
-  grid-template-columns: max-content auto;
-  column-gap: 0.5em;
-  padding-top: 4px;
-  padding-bottom: 4px;
-  border-bottom: solid 1px var(--bs-border-color);
-}
-
-._key_1ltuo_1 {
-  display: grid;
-  grid-template-columns: 1em auto;
-  cursor: pointer;
-}
-
-._pre_1ltuo_16 {
-  margin-bottom: 0;
-}
-
-._treeIcon_1ltuo_20 {
-  margin-top: -3px;
-}
 ._item_1uzhd_1 {
   margin-bottom: 0em;
 }