inspect-ai 0.3.58__py3-none-any.whl → 0.3.59__py3-none-any.whl

inspect_ai/_eval/score.py

@@ -85,6 +85,7 @@ async def score_async(
             sample_id=sample.id,
             epoch=sample.epoch,
             input=sample.input,
+            target=Target(sample.target),
             choices=sample.choices,
             messages=sample.messages,
             output=sample.output,

inspect_ai/_eval/task/results.py

@@ -2,6 +2,7 @@ import fnmatch
 import re
 from collections import defaultdict
 from copy import deepcopy
+from dataclasses import dataclass, field
 from typing import Any, Tuple, cast
 
 from inspect_ai._util.registry import (
@@ -19,6 +20,8 @@ from inspect_ai.log import (
 from inspect_ai.log._log import EvalSampleReductions
 from inspect_ai.scorer import Metric, Score, Scorer
 from inspect_ai.scorer._metric import SampleScore
+from inspect_ai.scorer._metrics.accuracy import accuracy
+from inspect_ai.scorer._metrics.std import stderr
 from inspect_ai.scorer._reducer import ScoreReducer, mean_score, reducer_log_name
 from inspect_ai.scorer._scorer import (
     SCORER_METRICS,
@@ -27,6 +30,27 @@ from inspect_ai.scorer._scorer import (
 )
 
 
+@dataclass
+class ScorerInfo:
+    name: str
+    metrics: list[Metric | dict[str, list[Metric]]] | dict[str, list[Metric]]
+    params: dict[str, Any] = field(default_factory=dict)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    @staticmethod
+    def from_scorer(scorer: Scorer) -> "ScorerInfo":
+        name = registry_unqualified_name(scorer)
+        metrics = scorer_metrics(scorer)
+        metadata = deepcopy(registry_info(scorer).metadata)
+        del metadata[SCORER_METRICS]
+        params = registry_params(scorer)
+        return ScorerInfo(name=name, metrics=metrics, params=params, metadata=metadata)
+
+    @staticmethod
+    def from_name(name: str) -> "ScorerInfo":
+        return ScorerInfo(name=name, metrics=[accuracy(), stderr()])
+
+
 def eval_results(
     samples: int,
     scores: list[dict[str, SampleScore]],
@@ -38,18 +62,23 @@ def eval_results(
     results = EvalResults(total_samples=samples, completed_samples=len(scores))
     reductions = None
 
+    # extract scorers info from scorers then create scorers info for any
+    # scores not already accounted for by a scorer name
+    scorers_info = [ScorerInfo.from_scorer(scorer) for scorer in (scorers or [])]
+    scorer_names = [info.name for info in scorers_info]
+    for name in set(key for sample_scores in scores for key in sample_scores):
+        if name not in scorer_names:
+            scorers_info.append(ScorerInfo.from_name(name))
+            scorer_names.append(name)
+
     # record scorer
-    if scorers:
+    if len(scorers_info) > 0:
         result_scores: list[EvalScore] = []
         sample_reductions: list[EvalSampleReductions] = []
-        for scorer in scorers:
-            # extract non-metrics metadata
-            metadata = deepcopy(registry_info(scorer).metadata)
-            del metadata[SCORER_METRICS]
-
+        for scorer_info in scorers_info:
             # this scorer
             scorer_name = unique_scorer_name(
-                scorer, [eval_score.name for eval_score in result_scores]
+                scorer_info.name, [eval_score.name for eval_score in result_scores]
             )
 
             # scores for this scorer
@@ -75,7 +104,7 @@ def eval_results(
 
                 # Compute metrics for this scorer
                 simple_scores = cast(list[Score], reduced_scores)
-                targets = metrics if metrics is not None else scorer_metrics(scorer)
+                targets = metrics if metrics is not None else scorer_info.metrics
                 if isinstance(targets, list):
                     ## split the metrics into the simple metrics and any dictionary
                     ## metrics, to be processed independently
@@ -88,8 +117,7 @@ def eval_results(
                     result_scores.extend(
                         scorer_for_metrics(
                             scorer_name=scorer_name,
-                            scorer=scorer,
-                            metadata=metadata,
+                            scorer_info=scorer_info,
                             scores=simple_scores,
                             metrics=simple_metrics,
                             reducer_name=reducer_display_nm,
@@ -99,8 +127,7 @@ def eval_results(
                         result_scores.extend(
                             scorers_from_metric_dict(
                                 scorer_name=scorer_name,
-                                scorer=scorer,
-                                metadata=metadata,
+                                scorer_info=scorer_info,
                                 scores=simple_scores,
                                 metrics=dict_metric,
                                 reducer_name=reducer_display_nm,
@@ -116,8 +143,7 @@ def eval_results(
                     result_scores.extend(
                         scorers_from_metric_dict(
                             scorer_name=scorer_name,
-                            scorer=scorer,
-                            metadata=metadata,
+                            scorer_info=scorer_info,
                             scores=simple_scores,
                             metrics=targets,
                             reducer_name=reducer_display_nm,
@@ -156,8 +182,7 @@ def split_metrics(
 
 def scorer_for_metrics(
     scorer_name: str,
-    scorer: Scorer,
-    metadata: dict[str, Any],
+    scorer_info: ScorerInfo,
     scores: list[Score],
     metrics: list[Metric],
     reducer_name: str | None = None,
@@ -218,8 +243,10 @@ def scorer_for_metrics(
             scorer=scorer_name,
             reducer=reducer_name,
             name=scorer_name,
-            params=registry_params(scorer),
-            metadata=metadata if len(metadata.keys()) > 0 else None,
+            params=scorer_info.params,
+            metadata=scorer_info.metadata
+            if len(scorer_info.metadata.keys()) > 0
+            else None,
             metrics=list_metrics,
         )
     )
@@ -228,8 +255,7 @@ def scorer_for_metrics(
 
 def scorers_from_metric_dict(
     scorer_name: str,
-    scorer: Scorer,
-    metadata: dict[str, Any],
+    scorer_info: ScorerInfo,
     scores: list[Score],
     metrics: dict[str, list[Metric]],
     reducer_name: str | None = None,
@@ -299,8 +325,10 @@ def scorers_from_metric_dict(
                 scorer=scorer_name,
                 reducer=reducer_name,
                 name=metric_key,
-                params=registry_params(scorer),
-                metadata=metadata if len(metadata.keys()) > 0 else None,
+                params=scorer_info.params,
+                metadata=scorer_info.metadata
+                if len(scorer_info.metadata.keys()) > 0
+                else None,
                 metrics=result_metrics,
             )
         )

inspect_ai/_eval/task/run.py

@@ -27,10 +27,7 @@ from inspect_ai._util.constants import (
 from inspect_ai._util.datetime import iso_now
 from inspect_ai._util.error import exception_message
 from inspect_ai._util.hooks import send_telemetry
-from inspect_ai._util.registry import (
-    is_registry_object,
-    registry_log_name,
-)
+from inspect_ai._util.registry import is_registry_object, registry_log_name
 from inspect_ai._util.timeouts import Timeout, timeout, timeout_at
 from inspect_ai._view.notify import view_notify_eval
 from inspect_ai.dataset import Dataset, Sample
@@ -45,7 +42,11 @@ from inspect_ai.log import (
 from inspect_ai.log._condense import condense_sample
 from inspect_ai.log._file import eval_log_json_str
 from inspect_ai.log._log import EvalSampleLimit, EvalSampleReductions, eval_error
-from inspect_ai.log._samples import active_sample
+from inspect_ai.log._samples import (
+    active_sample,
+    set_active_sample_message_limit,
+    set_active_sample_token_limit,
+)
 from inspect_ai.log._transcript import (
     ErrorEvent,
     SampleInitEvent,
@@ -72,6 +73,7 @@ from inspect_ai.solver._chain import Chain, unroll
 from inspect_ai.solver._fork import set_task_generate
 from inspect_ai.solver._solver import Solver
 from inspect_ai.solver._task_state import sample_state, set_sample_state, state_jsonable
+from inspect_ai.util._limit import SampleLimitExceededError
 from inspect_ai.util._sandbox.context import sandbox_connections
 from inspect_ai.util._sandbox.environment import SandboxEnvironmentSpec
 from inspect_ai.util._subtask import init_subtask
@@ -538,6 +540,9 @@ async def task_run_sample(
     # helper to handle exceptions (will throw if we've exceeded the limit)
     def handle_error(ex: BaseException) -> EvalError:
         err = sample_error(ex)
+        py_logger.warning(
+            f"Sample error (id: {sample.id}, epoch: {state.epoch}): {exception_message(ex)})"
+        )
         transcript()._event(ErrorEvent(error=err))
         return err
 
@@ -630,6 +635,20 @@ async def task_run_sample(
                     else:
                         raise
 
+                except SampleLimitExceededError as ex:
+                    # sample limit event
+                    transcript()._event(
+                        SampleLimitEvent(
+                            type=ex.type,
+                            limit=ex.limit,
+                            message=f"Sample completed: {ex.message}",
+                        )
+                    )
+
+                    # capture most recent state for scoring
+                    state = sample_state() or state
+                    state.completed = True
+
                 except BaseException as ex:
                     error = handle_error(ex)
 
@@ -648,12 +667,16 @@ async def task_run_sample(
                         assert time_limit
                         timeout_cm = timeout(time_limit / 2)
 
+                # turn off sample limits
+                set_active_sample_token_limit(None)
+                set_active_sample_message_limit(None)
+
                 # scoring
                 try:
                     # timeout during scoring will result in an ordinary sample error
                     async with timeout_cm:
-                        if scorers and error is None:
-                            for scorer in scorers:
+                        if error is None:
+                            for scorer in scorers or []:
                                 scorer_name = unique_scorer_name(
                                     scorer, list(results.keys())
                                 )
@@ -675,6 +698,16 @@ async def task_run_sample(
                                         )
                                         results[scorer_name] = sample_score
 
+                            # add scores returned by solvers
+                            if state.scores is not None:
+                                for name, score in state.scores.items():
+                                    results[name] = SampleScore(
+                                        score=score, sample_id=state.sample_id
+                                    )
+
+                            # propagate results into scores
+                            state.scores = {k: v.score for k, v in results.items()}
+
                 except asyncio.CancelledError:
                     if active.interrupt_action:
                         transcript()._event(
@@ -819,6 +852,7 @@ async def resolve_dataset(
                 epoch=epoch,
                 model=model_name,
                 input=sample.input,
+                target=Target(sample.target),
                 choices=sample.choices,
                 messages=sample_messages(sample),
                 message_limit=message_limit,

inspect_ai/_eval/task/sandbox.py

@@ -4,11 +4,13 @@ import contextlib
 from random import random
 from typing import AsyncGenerator, Callable, NamedTuple, cast
 
+import httpx
+
 from inspect_ai._eval.task.task import Task
 from inspect_ai._eval.task.util import task_run_dir
 from inspect_ai._util.file import file, filesystem
 from inspect_ai._util.registry import registry_unqualified_name
-from inspect_ai._util.url import data_uri_to_base64, is_data_uri
+from inspect_ai._util.url import data_uri_to_base64, is_data_uri, is_http_url
 from inspect_ai.dataset import Sample
 from inspect_ai.util._concurrency import concurrency
 from inspect_ai.util._sandbox.context import (
@@ -65,12 +67,12 @@ async def sandboxenv_context(
         files: dict[str, bytes] = {}
         if sample.files:
             for path, contents in sample.files.items():
-                files[path] = read_sandboxenv_file(contents)
+                files[path] = await read_sandboxenv_file(contents)
 
         # read setup script from sample (add bash shebang if necessary)
         setup: bytes | None = None
         if sample.setup:
-            setup = read_sandboxenv_file(sample.setup)
+            setup = await read_sandboxenv_file(sample.setup)
             setup_str = setup.decode(encoding="utf-8")
             if not setup_str.strip().startswith("#!"):
                 setup_str = f"#!/usr/bin/env bash\n\n{setup_str}"
@@ -108,13 +110,16 @@ async def sandboxenv_context(
                 )
 
 
-def read_sandboxenv_file(contents: str) -> bytes:
+async def read_sandboxenv_file(contents: str) -> bytes:
     if is_data_uri(contents):
         contents_base64 = data_uri_to_base64(contents)
         file_bytes = base64.b64decode(contents_base64)
+    elif is_http_url(contents):
+        client = httpx.AsyncClient()
+        file_bytes = (await client.get(contents, follow_redirects=True)).content
     else:
         # try to read as a file (if it doesn't exist or has a path not cool w/
-        # the fileystem then we fall back to contents)
+        # the filesystem then we fall back to contents)
         try:
             fs = filesystem(contents)
             if fs.exists(contents):

inspect_ai/_util/constants.py

@@ -37,3 +37,4 @@ SAMPLE_SUBTASK = "sample"
 CONSOLE_DISPLAY_WIDTH = 120
 BASE_64_DATA_REMOVED = "<base64-data-removed>"
 SANDBOX_SETUP_TIMEOUT = 300
+NO_CONTENT = "(no content)"

inspect_ai/_util/port_names.py

@@ -0,0 +1,61 @@
+from typing import Literal
+
+
+def get_service_by_port(port: int, protocol: Literal["tcp", "udp"]) -> str | None:
+    """
+    Returns the likely service running on a given port number.
+
+    Args:
+        port (int): The port number to look up
+        protocol (str): Either 'tcp' or 'udp'
+
+    Returns:
+        str: Description of the likely service, or None if not found
+    """
+    # Common port mappings based on IANA assignments and common usage
+    port_mappings = {
+        "tcp": {
+            20: "FTP (Data)",
+            21: "FTP (Control)",
+            22: "SSH",
+            23: "Telnet",
+            25: "SMTP",
+            53: "DNS",
+            80: "HTTP",
+            110: "POP3",
+            143: "IMAP",
+            443: "HTTPS",
+            445: "Microsoft-DS (SMB)",
+            587: "SMTP (Submission)",
+            993: "IMAPS",
+            995: "POP3S",
+            1433: "Microsoft SQL Server",
+            1521: "Oracle Database",
+            3306: "MySQL",
+            3389: "RDP (Remote Desktop)",
+            5432: "PostgreSQL",
+            5900: "VNC",
+            5901: "VNC Display :1",
+            5902: "VNC Display :2",
+            6080: "noVNC",
+            8080: "HTTP Alternate",
+            8443: "HTTPS Alternate",
+            27017: "MongoDB",
+            27018: "MongoDB Shard",
+            27019: "MongoDB Config Server",
+        },
+        "udp": {
+            53: "DNS",
+            67: "DHCP Server",
+            68: "DHCP Client",
+            69: "TFTP",
+            123: "NTP",
+            161: "SNMP",
+            162: "SNMP Trap",
+            514: "Syslog",
+            1194: "OpenVPN",
+            5353: "mDNS",
+        },
+    }
+
+    return port_mappings.get(protocol, {}).get(port, None)

inspect_ai/_util/text.py

@@ -108,3 +108,26 @@ def str_to_float(s: str) -> float:
         exponent = 1  # Default exponent is 1 if no superscript is present
 
     return base**exponent
+
+
+def truncate(text: str, length: int, overflow: str = "...", pad: bool = True) -> str:
+    """
+    Truncate text to specified length with optional padding and overflow indicator.
+
+    Args:
+        text (str): Text to truncate
+        length (int): Maximum length including overflow indicator
+        overflow (str): String to indicate truncation (defaults to '...')
+        pad (bool): Whether to pad the result to full length (defaults to padding)
+
+    Returns:
+        Truncated string, padded if requested
+
+    """
+    if len(text) <= length:
+        return text + (" " * (length - len(text))) if pad else text
+
+    overflow_length = len(overflow)
+    truncated = text[: length - overflow_length] + overflow
+
+    return truncated

inspect_ai/_view/www/App.css

@@ -9,6 +9,12 @@
   --inspect-input-border: var(--bs-light-border-subtle);
   --inspect-diff-add-color: #dafbe1;
   --inspect-diff-remove-color: #ffebe9;
+  --inspect-inactive-selection-background: var(--vscode-editor-inactiveSelectionBackground, #d9d9d9);
+  --inspect-active-selection-background: var(--vscode-editor-selectionBackground, #d7d4f0);
+  --inspect-focus-border-color: #86b7fe;
+  --inspect-focus-border-shadow: 0 0 0 0.25rem rgba(var(--bs-primary-rgb), 0.25);
+  --inspect-focus-border-gray-color: #808080;
+  --inspect-focus-border-gray-shadow: 0 0 0 0.25rem rgba(48, 48, 48, 0.25);
 }
 
 body:not([class^="vscode-"]) button {
@@ -650,6 +656,30 @@ table.table.table-sm td {
   height: auto !important;
 }
 
+[data-tooltip] {
+  position: relative;
+}
+[data-tooltip]:hover::after {
+  content: attr(data-tooltip);
+  position: absolute;
+  line-height: 1.25;
+  background: var(--bs-light);
+  color: var(--bs-body-color);
+  opacity: 1;
+  padding: 4px 8px;
+  border-radius: 4px;
+  border: 1px solid var(--bs-border-color);
+  box-shadow: 0 2px 10px 0 rgba(0, 0, 0, 0.25);
+  white-space: pre-wrap;
+  width: max-content;
+  max-width: 400px;
+  z-index: 1000;
+}
+[data-tooltip][data-tooltip-position="bottom-left"]:hover::after {
+  right: 0%;
+  top: 100%;
+}
+
 /* ANSI Coloring */
 .ansi-display {
   font-family: monospace;
@@ -725,7 +755,7 @@ pre[class*="language-"].tool-output {
   background: none !important;
   border: none !important;
   box-shadow: none !important;
-  border-radius: var(--bs-border-radius) !important;  
+  border-radius: var(--bs-border-radius) !important;
 }
 
 /* lightbox styles */

inspect_ai/_view/www/dist/assets/index.css

@@ -14282,6 +14282,12 @@ pre[class*="language-"] {
   --inspect-input-border: var(--bs-light-border-subtle);
   --inspect-diff-add-color: #dafbe1;
   --inspect-diff-remove-color: #ffebe9;
+  --inspect-inactive-selection-background: var(--vscode-editor-inactiveSelectionBackground, #d9d9d9);
+  --inspect-active-selection-background: var(--vscode-editor-selectionBackground, #d7d4f0);
+  --inspect-focus-border-color: #86b7fe;
+  --inspect-focus-border-shadow: 0 0 0 0.25rem rgba(var(--bs-primary-rgb), 0.25);
+  --inspect-focus-border-gray-color: #808080;
+  --inspect-focus-border-gray-shadow: 0 0 0 0.25rem rgba(48, 48, 48, 0.25);
 }
 
 body:not([class^="vscode-"]) button {
@@ -14923,6 +14929,30 @@ table.table.table-sm td {
   height: auto !important;
 }
 
+[data-tooltip] {
+  position: relative;
+}
+[data-tooltip]:hover::after {
+  content: attr(data-tooltip);
+  position: absolute;
+  line-height: 1.25;
+  background: var(--bs-light);
+  color: var(--bs-body-color);
+  opacity: 1;
+  padding: 4px 8px;
+  border-radius: 4px;
+  border: 1px solid var(--bs-border-color);
+  box-shadow: 0 2px 10px 0 rgba(0, 0, 0, 0.25);
+  white-space: pre-wrap;
+  width: max-content;
+  max-width: 400px;
+  z-index: 1000;
+}
+[data-tooltip][data-tooltip-position="bottom-left"]:hover::after {
+  right: 0%;
+  top: 100%;
+}
+
 /* ANSI Coloring */
 .ansi-display {
   font-family: monospace;
@@ -14998,7 +15028,7 @@ pre[class*="language-"].tool-output {
   background: none !important;
   border: none !important;
   box-shadow: none !important;
-  border-radius: var(--bs-border-radius) !important;  
+  border-radius: var(--bs-border-radius) !important;
 }
 
 /* lightbox styles */