inspect-ai 0.3.102__py3-none-any.whl → 0.3.104__py3-none-any.whl

inspect_ai/_eval/task/log.py

@@ -71,10 +71,12 @@ class TaskLogger:
         sandbox: SandboxEnvironmentSpec | None,
         task_attribs: dict[str, Any],
         task_args: dict[str, Any],
+        task_args_passed: dict[str, Any],
         model_args: dict[str, Any],
         eval_config: EvalConfig,
         metadata: dict[str, Any] | None,
         recorder: Recorder,
+        header_only: bool,
     ) -> None:
         # determine versions
         git = git_context()
@@ -127,6 +129,7 @@ class TaskLogger:
             task_registry_name=task_registry_name,
             task_attribs=task_attribs,
             task_args=task_args,
+            task_args_passed=task_args_passed,
             solver=solver.solver if solver else None,
             tags=tags,
             solver_args=solver.args if solver else None,
@@ -153,6 +156,7 @@ class TaskLogger:
 
         # stack recorder and location
         self.recorder = recorder
+        self.header_only = header_only
 
         # number of samples logged
         self._samples_completed = 0
@@ -238,7 +242,7 @@ class TaskLogger:
     ) -> EvalLog:
         # finish and get log
         log = await self.recorder.log_finish(
-            self.eval, status, stats, results, reductions, error
+            self.eval, status, stats, results, reductions, error, self.header_only
         )
 
         # cleanup the events db

inspect_ai/_eval/task/run.py

@@ -223,7 +223,7 @@ async def task_run(options: TaskRunOptions) -> EvalLog:
         samples=len(samples),
         steps=len(samples) * SAMPLE_TOTAL_PROGRESS_UNITS,
         eval_config=config,
-        task_args=logger.eval.task_args,
+        task_args=logger.eval.task_args_passed,
         generate_config=generate_config,
         tags=tags,
         log_location=log_location,

inspect_ai/_util/citation.py

@@ -0,0 +1,88 @@
+from typing import Annotated, Literal, TypeAlias, Union
+
+from pydantic import BaseModel, Discriminator, Field, JsonValue
+
+
+class CitationBase(BaseModel):
+    """Base class for citations."""
+
+    cited_text: str | tuple[int, int] | None = Field(
+        default=None,
+        # without helping the schema generator, this will turn into [unknown, unknown] in TypeScript
+        json_schema_extra={
+            "anyOf": [
+                {"type": "string"},
+                {
+                    "type": "array",
+                    "items": [{"type": "integer"}, {"type": "integer"}],
+                    "additionalItems": False,
+                    "minItems": 2,
+                    "maxItems": 2,
+                },
+                {"type": "null"},
+            ]
+        },
+    )
+    """
+    The cited text
+
+    This can be the text itself or a start/end range of the text content within
+    the container that is the cited text.
+    """
+
+    title: str | None = None
+    """Title of the cited resource."""
+
+    internal: dict[str, JsonValue] | None = Field(default=None)
+    """Model provider specific payload - typically used to aid transformation back to model types."""
+
+
+class ContentCitation(CitationBase):
+    """A generic content citation."""
+
+    type: Literal["content"] = Field(default="content")
+    """Type."""
+
+
+class DocumentRange(BaseModel):
+    """A range specifying a section of a document."""
+
+    type: Literal["block", "page", "char"]
+    """The type of the document section specified by the range."""
+
+    start_index: int
+    """0 based index of the start of the range."""
+
+    end_index: int
+    """0 based index of the end of the range."""
+
+
+class DocumentCitation(CitationBase):
+    """A citation that refers to a page range in a document."""
+
+    type: Literal["document"] = Field(default="document")
+    """Type."""
+
+    range: DocumentRange | None = Field(default=None)
+    """Range of the document that is cited."""
+
+
+class UrlCitation(CitationBase):
+    """A citation that refers to a URL."""
+
+    type: Literal["url"] = Field(default="url")
+    """Type."""
+
+    url: str
+    """URL of the cited resource."""
+
+
+Citation: TypeAlias = Annotated[
+    Union[
+        ContentCitation,
+        DocumentCitation,
+        UrlCitation,
+    ],
+    Discriminator("type"),
+]
+"""A citation sent to or received from a model."""

inspect_ai/_util/content.py

@@ -1,7 +1,9 @@
-from typing import Literal, Union
+from typing import Literal, Sequence, Union
 
 from pydantic import BaseModel, Field, JsonValue
 
+from inspect_ai._util.citation import Citation
+
 
 class ContentBase(BaseModel):
     internal: JsonValue | None = Field(default=None)
@@ -20,6 +22,9 @@ class ContentText(ContentBase):
     refusal: bool | None = Field(default=None)
     """Was this a refusal message?"""
 
+    citations: Sequence[Citation] | None = Field(default=None)
+    """Citations supporting the text block."""
+
 
 class ContentReasoning(ContentBase):
     """Reasoning content.
@@ -82,5 +87,22 @@ class ContentVideo(ContentBase):
     """Format of video data ('mp4', 'mpeg', or 'mov')"""
 
 
-Content = Union[ContentText, ContentReasoning, ContentImage, ContentAudio, ContentVideo]
+class ContentData(ContentBase):
+    """Model internal."""
+
+    type: Literal["data"] = Field(default="data")
+    """Type."""
+
+    data: dict[str, JsonValue]
+    """Model provider specific payload - required for internal content."""
+
+
+Content = Union[
+    ContentText,
+    ContentReasoning,
+    ContentImage,
+    ContentAudio,
+    ContentVideo,
+    ContentData,
+]
 """Content sent to or received from a model."""

inspect_ai/_util/json.py

@@ -6,7 +6,7 @@ from typing import (
 
 import jsonpatch
 from pydantic import BaseModel, Field, JsonValue
-from pydantic_core import to_json, to_jsonable_python
+from pydantic_core import PydanticSerializationError, to_json, to_jsonable_python
 
 JSONType = Literal["string", "integer", "number", "boolean", "array", "object", "null"]
 """Valid types within JSON schema."""
@@ -27,7 +27,22 @@ def jsonable_dict(x: Any) -> dict[str, JsonValue]:
 
 
 def to_json_safe(x: Any) -> bytes:
-    return to_json(value=x, indent=2, exclude_none=True, fallback=lambda _x: None)
+    def clean_utf8_json(obj: Any) -> Any:
+        if isinstance(obj, str):
+            return obj.encode("utf-8", errors="replace").decode("utf-8")
+        elif isinstance(obj, dict):
+            return {k: clean_utf8_json(v) for k, v in obj.items()}
+        elif isinstance(obj, list):
+            return [clean_utf8_json(item) for item in obj]
+        return obj
+
+    try:
+        return to_json(value=x, indent=2, exclude_none=True, fallback=lambda _x: None)
+    except PydanticSerializationError as ex:
+        if "surrogates not allowed" in str(ex):
+            cleaned = clean_utf8_json(x)
+            return to_json(cleaned)
+        raise
 
 
 def to_json_str_safe(x: Any) -> str:

inspect_ai/_util/registry.py

@@ -102,9 +102,26 @@ def registry_tag(
         *args (list[Any]): Creation arguments
         **kwargs (dict[str,Any]): Creation keyword arguments
     """
+    # bind arguments to params
+    named_params = extract_named_params(type, False, *args, **kwargs)
+
+    # set attribute
+    setattr(o, REGISTRY_INFO, info)
+    setattr(o, REGISTRY_PARAMS, named_params)
+
+
+def extract_named_params(
+    type: Callable[..., Any], apply_defaults: bool, *args: Any, **kwargs: Any
+) -> dict[str, Any]:
     # bind arguments to params
     named_params: dict[str, Any] = {}
-    bound_params = inspect.signature(type).bind(*args, **kwargs)
+
+    if apply_defaults:
+        bound_params = inspect.signature(type).bind_partial(*args, **kwargs)
+        bound_params.apply_defaults()
+    else:
+        bound_params = inspect.signature(type).bind(*args, **kwargs)
+
     for param, value in bound_params.arguments.items():
         named_params[param] = registry_value(value)
 
@@ -128,9 +145,7 @@ def registry_tag(
                 or "<unknown>"
             )
 
-    # set attribute
-    setattr(o, REGISTRY_INFO, info)
-    setattr(o, REGISTRY_PARAMS, named_params)
+    return named_params
 
 
 def registry_name(o: object, name: str) -> str:

inspect_ai/_view/schema.py

@@ -1,6 +1,5 @@
 import json
 import os
-import shutil
 import subprocess
 from pathlib import Path
 from typing import Any
@@ -19,9 +18,6 @@ def sync_view_schema() -> None:
     # export schema file
     schema_path = Path(WWW_DIR, "log-schema.json")
     types_path = Path(WWW_DIR, "src", "@types", "log.d.ts")
-    vs_code_types_path = Path(
-        WWW_DIR, "..", "..", "..", "..", "tools", "vscode", "src", "@types", "log.d.ts"
-    )
 
     with open(schema_path, "w", encoding="utf-8") as f:
         # make everything required
@@ -50,8 +46,6 @@ def sync_view_schema() -> None:
 
         subprocess.run(["yarn", "prettier:write"], cwd=types_path.parent, check=True)
 
-        shutil.copyfile(types_path, vs_code_types_path)
-
 
 def schema_to_strict(schema: dict[str, Any]) -> dict[str, Any]:
     properties = schema.get("properties", None)

inspect_ai/_view/www/dist/assets/index.css

@@ -15692,18 +15692,98 @@ pre[class*="language-"] {
 ._copyLink_17kai_29:hover {
   opacity: 1;
 }
-._contentImage_61gdd_1 {
+._webSearch_1376z_1 {
+  display: grid;
+  grid-template-columns: max-content 1fr;
+  column-gap: 0.5em;
+  align-items: baseline;
+}
+
+._query_1376z_8 {
+  font-family: var(--bs-font-monospace);
+}
+._keyPairContainer_1ltuo_1 {
+  display: grid;
+  grid-template-columns: max-content auto;
+  column-gap: 0.5em;
+  padding-top: 4px;
+  padding-bottom: 4px;
+  border-bottom: solid 1px var(--bs-border-color);
+}
+
+._key_1ltuo_1 {
+  display: grid;
+  grid-template-columns: 1em auto;
+  cursor: pointer;
+}
+
+._pre_1ltuo_16 {
+  margin-bottom: 0;
+}
+
+._treeIcon_1ltuo_20 {
+  margin-top: -3px;
+}
+._contentData_1sd1z_1 {
+  border: solid var(--bs-light-border-subtle) 1px;
+  padding: 0.5em;
+  margin-bottom: 0.5em;
+  margin-top: 0.5em;
+  margin-left: 1em;
+}
+._webSearch_1mixg_1 {
+  display: grid;
+  grid-template-columns: max-content 1fr;
+  column-gap: 0.5em;
+  align-items: baseline;
+}
+
+._query_1mixg_8 {
+  font-family: var(--bs-font-monospace);
+}
+
+._result_1mixg_12 a:hover {
+  text-decoration: underline;
+}
+
+._result_1mixg_12 a {
+  opacity: 0.8;
+  text-decoration: none;
+}
+._citations_t2k1z_1 {
+  margin-top: 1em;
+  margin-bottom: 1em;
+  display: grid;
+  grid-template-columns: max-content 1fr;
+  column-gap: 0.5em;
+}
+
+a._citationLink_t2k1z_9 {
+  display: block;
+  color: var(--bs-body);
+  text-decoration: none;
+}
+a._citationLink_t2k1z_9:hover {
+  text-decoration: underline;
+}
+._contentImage_8rgix_1 {
   max-width: 800px;
   border: solid var(--bs-border-color) 1px;
 }
 
-._reasoning_61gdd_6 {
+._reasoning_8rgix_6 {
   border: solid var(--bs-light-border-subtle) 1px;
   padding: 1em;
   margin-bottom: 0.5em;
   background-color: var(--bs-light-bg-subtle);
   border-radius: var(--bs-border-radius);
 }
+
+._data_8rgix_14 {
+  border: solid var(--bs-light-border-subtle) 1px;
+  padding: 1em;
+  margin-bottom: 0.5em;
+}
 ._toolImage_bv5nm_1 {
   max-width: 800px;
   border: solid var(--bs-border-color) 1px;
@@ -16741,28 +16821,6 @@ thead th {
 .card-body.card-no-padding {
   padding: 0;
 }
-._keyPairContainer_1ltuo_1 {
-  display: grid;
-  grid-template-columns: max-content auto;
-  column-gap: 0.5em;
-  padding-top: 4px;
-  padding-bottom: 4px;
-  border-bottom: solid 1px var(--bs-border-color);
-}
-
-._key_1ltuo_1 {
-  display: grid;
-  grid-template-columns: 1em auto;
-  cursor: pointer;
-}
-
-._pre_1ltuo_16 {
-  margin-bottom: 0;
-}
-
-._treeIcon_1ltuo_20 {
-  margin-top: -3px;
-}
 ._item_1uzhd_1 {
   margin-bottom: 0em;
 }