inspect-evals-utils 0.1.0__py3-none-any.whl

inspect_evals_utils/__init__.py

@@ -0,0 +1,43 @@
+"""Runtime helpers for Inspect AI evaluations: dataset loading with retries, sample ids, prompts and README generation."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+from inspect_evals_utils.cache import INSPECT_EVALS_CACHE_PATH
+from inspect_evals_utils.datasets import load_csv_dataset, load_json_dataset
+from inspect_evals_utils.ids import (
+    DuplicateIdError,
+    KnownBrokenSampleError,
+    create_stable_id,
+    drop_known_broken,
+    filter_duplicate_ids,
+    require_optional_dependency,
+)
+from inspect_evals_utils.messages import message_content_text, reasoning_to_text
+from inspect_evals_utils.prompts import Prompt
+from inspect_evals_utils.shuffle import shuffle_and_seed
+
+# The version is set once, in pyproject.toml; this reads it back from the
+# installed package metadata.
+try:
+    __version__ = version("inspect-evals-utils")
+except PackageNotFoundError:  # a source tree that was never installed
+    __version__ = "unknown"
+
+# inspect_evals_utils.huggingface is not imported here: it needs the
+# huggingface extra, and the base package must import without it.
+__all__ = [
+    "INSPECT_EVALS_CACHE_PATH",
+    "DuplicateIdError",
+    "KnownBrokenSampleError",
+    "Prompt",
+    "__version__",
+    "create_stable_id",
+    "drop_known_broken",
+    "filter_duplicate_ids",
+    "load_csv_dataset",
+    "load_json_dataset",
+    "message_content_text",
+    "reasoning_to_text",
+    "require_optional_dependency",
+    "shuffle_and_seed",
+]

inspect_evals_utils/cache.py

@@ -0,0 +1,33 @@
+"""The cache directory shared by inspect_evals and standalone evals.
+
+Set INSPECT_EVALS_CACHE_DIR to move it or point it at a pre-populated directory.
+"""
+
+import os
+from pathlib import Path
+
+from platformdirs import user_cache_dir
+
+
+def cache_path() -> Path:
+    """Resolve the cache directory, rejecting a value that depends on the cwd."""
+    configured = os.environ.get("INSPECT_EVALS_CACHE_DIR")
+    if not configured:
+        return Path(user_cache_dir("inspect_evals"))
+
+    # A leading ~ survives a .env file and a quoted shell assignment, which is
+    # how this variable is usually set, so expand it rather than treating it as
+    # a directory name.
+    path = Path(configured).expanduser()
+    if not path.is_absolute():
+        raise ValueError(
+            f"INSPECT_EVALS_CACHE_DIR must be absolute or start with ~, but is "
+            f"{configured!r}. It is read once when inspect_evals_utils is imported, so "
+            f"a relative path would point somewhere different depending on where "
+            f"the eval was started from."
+        )
+    return path
+
+
+# Read once at import, because evals derive their own module-level paths from it.
+INSPECT_EVALS_CACHE_PATH = cache_path()

inspect_evals_utils/code.py

@@ -0,0 +1,123 @@
+"""Helpers for extracting generated code and JSON from Markdown responses."""
+
+import ast
+import re
+from collections.abc import Callable
+
+_LANGUAGE_TAGS = {
+    "python": ("python", "python3", "py"),
+    "cpp": ("cpp", "c++", "cc", "cxx", "c"),
+    "java": ("java",),
+    "javascript": ("javascript", "js"),
+    "typescript": ("typescript", "ts"),
+    "csharp": ("csharp", "cs", "c#"),
+    "php": ("php",),
+    "shell": ("bash", "sh", "shell"),
+}
+
+# A block ends at a bare fence line, just before any other fence line (which a model
+# may write without closing the block first), at a fence glued to the end of the last
+# code line, or at the end of the text when the fence is never closed. The last case is
+# its own group so that an unclosed unlabelled fence, usually a stray closing fence
+# after unfenced code, can be ignored.
+_CLOSE = r"(?:^```[ \t\r]*$|(?=^```)|(?<=\S)```[ \t\r]*$|(\Z))"
+
+_FENCE = re.compile(
+    r"^```[ \t]*([^\s`]*)[^\n]*\r?\n(.*?)" + _CLOSE,
+    re.DOTALL | re.MULTILINE,
+)
+
+
+_JSON_FENCE = re.compile(
+    r"\A\s*```[ \t]*(?:json)?[ \t]*\r?\n(.*?)\s*```\s*\Z", re.DOTALL | re.IGNORECASE
+)
+
+
+def _labelled_fence(tags: tuple[str, ...]) -> re.Pattern[str]:
+    # A labelled fence may follow prose on the same line, but only when nothing
+    # follows the label, so that a fence mentioned in a sentence is not an opener.
+    tag = "|".join(re.escape(t) for t in sorted(tags, key=len, reverse=True))
+    opener = rf"(?:^```[ \t]*(?:{tag})(?=[\s`])[^\n]*|```[ \t]*(?:{tag})[ \t]*)\r?\n"
+    return re.compile(opener + "(.*?)" + _CLOSE, re.DOTALL | re.MULTILINE | re.IGNORECASE)
+
+
+def extract_code_blocks(completion: str, language: str = "python") -> list[str]:
+    """Return the language-labelled code blocks, or else the unlabelled ones.
+
+    Blocks are in the order they appear. Unlabelled blocks are returned only
+    when no block carries one of the language's labels. Labels are matched
+    case-insensitively and both LF and CRLF line endings are accepted. Fences
+    labelled for another language are not treated as unlabelled code, and empty
+    blocks are skipped. A labelled fence that is never closed runs to the end of
+    the text.
+    """
+    tags = _LANGUAGE_TAGS.get(language.lower(), (language.lower(),))
+    labelled = [
+        match.group(1)
+        for match in _labelled_fence(tags).finditer(completion)
+        if match.group(1).strip()
+    ]
+    if labelled:
+        return labelled
+    return [
+        match.group(2)
+        for match in _FENCE.finditer(completion)
+        if match.group(1) == "" and match.group(2).strip() and match.group(3) is None
+    ]
+
+
+def extract_code_block(
+    completion: str,
+    language: str = "python",
+    accept: Callable[[str], bool] | None = None,
+) -> str | None:
+    """Return the first block from `extract_code_blocks`, or None if there is none.
+
+    Language-labelled blocks are preferred over unlabelled blocks even when an
+    unlabelled block appears earlier in the response.
+
+    When `accept` is given, the first block it accepts is returned instead. If
+    it accepts none, the first block is returned as without it. Pass
+    `defines(name)` to take the block that implements the function the task
+    asks for, rather than a usage example or a draft that does not parse.
+    """
+    blocks = extract_code_blocks(completion, language)
+
+    if accept is not None:
+        for block in blocks:
+            if accept(block):
+                return block
+
+    return blocks[0] if blocks else None
+
+
+def strip_json_fence(text: str) -> str:
+    """Return the JSON inside one code fence wrapping the whole text, else the text unchanged.
+
+    The fence may be labelled `json` or unlabelled. Use it before parsing a
+    reply that should be bare JSON, such as one requested with a
+    `response_schema`: some providers accept the schema without enforcing it and
+    return the JSON in a fence. Text with prose around the fence, or more than
+    one fence, is returned unchanged, so it still fails validation.
+    """
+    match = _JSON_FENCE.match(text)
+    if match is None or re.search(r"^[ \t]*```", match.group(1), re.MULTILINE):
+        return text
+    return match.group(1)
+
+
+def defines(name: str) -> Callable[[str], bool]:
+    """Accept Python code that parses and defines `name` at the top level."""
+
+    def check(code: str) -> bool:
+        try:
+            tree = ast.parse(code)
+        except (SyntaxError, ValueError):
+            return False
+        return any(
+            isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef))
+            and node.name == name
+            for node in tree.body
+        )
+
+    return check

inspect_evals_utils/datasets.py

@@ -0,0 +1,311 @@
+import hashlib
+import logging
+import os
+from collections.abc import Callable
+from functools import wraps
+from pathlib import Path
+from typing import Any, Concatenate, ParamSpec, cast
+from urllib.parse import urlparse
+
+import backoff  # type: ignore
+from inspect_ai._util.file import (  # no public API available
+    file,
+)
+from inspect_ai.dataset import Dataset, csv_dataset, json_dataset
+
+from inspect_evals_utils.cache import INSPECT_EVALS_CACHE_PATH
+
+logger = logging.getLogger(__name__)
+
+P = ParamSpec("P")
+DatasetLoader = Callable[..., Dataset]
+
+SUPPORTED_REMOTE_URL_PROTOCOLS = {"https", "http", "s3", "gs", "gcs", "az", "abfs"}
+LOCAL_PROTOCOL = {"", "c", "d", "e", "file"}
+TRANSIENT_ERRORS = {408, 429, 500, 502, 503, 504}
+
+
+def is_remote_url(file_path: str) -> bool:
+    scheme = urlparse(file_path).scheme
+    known_remote = scheme in SUPPORTED_REMOTE_URL_PROTOCOLS
+    if known_remote:
+        return True
+    definitely_local = scheme in LOCAL_PROTOCOL
+    if definitely_local:
+        return False
+    raise ValueError(
+        f"Could not parse file path. Found {file_path}"
+        "Currently, following protocols are supported:"
+        f"{SUPPORTED_REMOTE_URL_PROTOCOLS}"
+    )
+
+
+def _get_extension(url: str) -> str:
+    """Get file extension from URL or file path."""
+    parsed_url = urlparse(url)
+    path = parsed_url.path
+    if path and "." in Path(path).name:
+        return Path(path).suffix
+
+    return ""
+
+
+def get_cached_fname(url: str, tag: str = "") -> str:
+    """Generate cache filename with optional tag prefix and URL hash.
+
+    Args:
+        url: The URL to hash
+        tag: Optional human-readable tag prefix (empty string for no tag)
+
+    Returns:
+        Filename in format: {tag}_{hash}.{ext} or {hash}.{ext}
+    """
+    prefix = f"{tag}_" if tag else ""
+    hash_part = hashlib.sha256(url.encode("utf-8")).hexdigest()[:32]
+    extension = _get_extension(url)
+    return f"{prefix}{hash_part}{extension}"
+
+
+def _get_cached_path(
+    eval_name: str,
+    url: str,
+    tag: str = "",
+) -> str:
+    """Generate fully qualified cache path for a given URL.
+
+    Args:
+        eval_name: Evaluation name (used for cache directory organization)
+        url: The remote URL
+        tag: Optional human-readable tag for cache filename (empty string for no tag)
+
+    """
+    return str(INSPECT_EVALS_CACHE_PATH / eval_name / get_cached_fname(url, tag))
+
+
+def is_transient_error(e: Exception) -> bool:
+    """Check if error is transient and should be retried.
+
+    Handles both HTTP errors from requests and network errors from fsspec.
+    """
+    # Network errors that should be retried
+    if isinstance(e, (ConnectionError, TimeoutError, OSError)):
+        return True
+
+    # Note: HTTPError import removed as we no longer use requests,
+    # but keeping this check for potential future compatibility
+    status_code = getattr(getattr(e, "response", None), "status_code", None)
+    if status_code is not None:
+        return status_code in TRANSIENT_ERRORS
+
+    return False
+
+
+@backoff.on_exception(  # type: ignore
+    backoff.expo,
+    exception=Exception,
+    giveup=lambda e: not is_transient_error(e),
+    max_tries=5,
+    jitter=backoff.full_jitter,
+)
+def _download_remote(
+    remote_url: str,
+    local_cache_path: str,
+    encoding: str = "utf-8",
+    fs_options: dict[str, Any] | None = None,
+) -> None:
+    """Downloads from remote_url and puts contents into local_cache_path.
+
+    Uses fsspec-based file() context manager to support multiple protocols
+    (HTTP, S3, GCS, Azure, etc.)
+
+    Args:
+        remote_url: Remote URL (supports http://, s3://, gs://, az://, etc.)
+        local_cache_path: Local path to cache the file
+        encoding: Text encoding for the remote file (default: utf-8). The cache
+            is always written as UTF-8.
+        fs_options: Filesystem options (e.g., S3 credentials, anonymous access)
+    """
+    Path(local_cache_path).parent.mkdir(parents=True, exist_ok=True)
+
+    fs_opts = fs_options or {}
+    with (
+        file(remote_url, "r", encoding=encoding, fs_options=fs_opts) as f_remote,
+        open(local_cache_path, "w", encoding="utf-8") as f_local,
+    ):
+        f_local.write(f_remote.read())
+
+
+def cache_dataset_loader() -> Callable[
+    [Callable[Concatenate[str, P], Dataset]], Callable[Concatenate[str, P], Dataset]
+]:
+    """Decorator factory that adds caching functionality to dataset loader functions.
+
+    The cache directory is inferred from the format_extension parameter.
+
+    Returns:
+        A decorator that wraps dataset loaders with caching logic
+    """
+
+    def decorate(
+        f: Callable[Concatenate[str, P], Dataset],
+    ) -> Callable[Concatenate[str, P], Dataset]:
+        @wraps(f)
+        def wrapper(
+            file_path: str,
+            eval_name: str,
+            *args: P.args,
+            **kwargs: P.kwargs,
+        ) -> Dataset:
+            # Extract caching-specific parameters from kwargs
+            refresh: bool = bool(kwargs.pop("refresh", False))
+            cache_tag: str = str(kwargs.pop("cache_tag", ""))
+            encoding: str = kwargs.get("encoding", "utf-8")  # type: ignore[assignment]
+            fs_options: dict[str, Any] | None = kwargs.get("fs_options")  # type: ignore[assignment]
+
+            if not is_remote_url(file_path):
+                logger.info(f"Reading local file {file_path}")
+                return f(file_path, *args, **kwargs)
+
+            # Validate that we can detect extension from remote URL
+            extension = _get_extension(file_path)
+            if not extension:
+                raise ValueError(
+                    f"Cannot detect file extension from URL: {file_path}. "
+                    "Please ensure the URL ends with .json, .jsonl, or .csv"
+                )
+
+            cached_path = _get_cached_path(
+                eval_name=eval_name,
+                url=file_path,
+                tag=cache_tag,
+            )
+
+            cache_exists = os.path.exists(cached_path)
+            cache_is_empty = cache_exists and os.path.getsize(cached_path) == 0
+            if refresh or not cache_exists or cache_is_empty:
+                logger.info(f"Downloading remote url {file_path} and writing to {cached_path}")
+                _download_remote(file_path, cached_path, encoding, fs_options)
+
+            # `encoding` describes the remote file; the cached copy is always UTF-8.
+            kwargs["encoding"] = "utf-8"
+
+            logger.info(f"Loading cached file from {cached_path}")
+            try:
+                return f(cached_path, *args, **kwargs)
+            except ValueError:
+                # This will catch UnicodeDecodeError and JSONDecodeError,
+                # as they are both ValueError subclasses.
+                # It will NOT catch _csv.Error, which is raised by csv.reader
+                # if using a strict dialect and the file is malformed.
+
+                if refresh:
+                    raise
+
+                # Parsing errors from json/csv readers should not poison future
+                # runs. Re-download and retry once; if parsing still fails,
+                # surface the second ValueError to the caller.
+                logger.warning(
+                    f"Failed to load cached dataset at {cached_path}; "
+                    "re-downloading and retrying once."
+                )
+                _download_remote(file_path, cached_path, encoding, fs_options)
+                return f(cached_path, *args, **kwargs)
+
+        return cast(Callable[Concatenate[str, P], Dataset], wrapper)
+
+    return decorate
+
+
+# Type stub functions for mypy
+def load_json_dataset(
+    file_path: str,
+    eval_name: str,
+    sample_fields: Any = None,
+    auto_id: bool = False,
+    shuffle: bool = False,
+    seed: int | None = None,
+    limit: int | bool | None = None,
+    epoch: int | None = None,
+    name: str = "",
+    fs_options: dict[str, Any] | None = None,
+    *,
+    encoding: str = "utf-8",
+    refresh: bool = False,
+    cache_tag: str | None = None,
+    **reader_kwargs: Any,
+) -> Dataset:
+    """Load a JSON or JSONL dataset with caching support for remote URLs.
+
+    Args:
+        file_path: Local path or remote URL to the JSON/JSONL file (extension auto-detected)
+        eval_name: Evaluation name (organizes cache directory)
+        sample_fields: Field specification or converter function for samples
+        auto_id: Automatically generate sample IDs
+        shuffle: Shuffle the dataset
+        seed: Random seed for shuffling
+        limit: Maximum number of samples to load
+        epoch: Epoch number for sampling
+        name: Dataset name
+        fs_options: Filesystem options for remote URLs
+        encoding: Text encoding of the source file. Remote files are cached as UTF-8.
+        refresh: Force re-download of cached remote files
+        cache_tag: Optional human-readable tag for cache filename
+        reader_kwargs: Optional JSON reader options.
+
+    Returns:
+        Dataset object
+    """
+    raise NotImplementedError  # replaced by the decorated implementation below
+
+
+def load_csv_dataset(
+    file_path: str,
+    eval_name: str,
+    sample_fields: Any = None,
+    auto_id: bool = False,
+    shuffle: bool = False,
+    seed: int | None = None,
+    limit: int | bool | None = None,
+    epoch: int | None = None,
+    name: str = "",
+    fieldnames: list[str] | None = None,
+    dialect: str = "unix",
+    delimiter: str | None = None,
+    encoding: str = "utf-8",
+    fs_options: dict[str, Any] | None = None,
+    shuffle_choices: bool | int = False,
+    *,
+    refresh: bool = False,
+    cache_tag: str | None = None,
+) -> Dataset:
+    """Load a CSV dataset with caching support for remote URLs.
+
+    Args:
+        file_path: Local path or remote URL to the CSV file (extension auto-detected)
+        eval_name: Evaluation name (organizes cache directory)
+        sample_fields: Field specification or converter function for samples
+        auto_id: Automatically generate sample IDs
+        shuffle: Shuffle the dataset
+        seed: Random seed for shuffling
+        limit: Maximum number of samples to load
+        epoch: Epoch number for sampling
+        name: Dataset name
+        fieldnames: CSV field names
+        dialect: CSV dialect
+        delimiter: CSV delimiter character
+        encoding: Text encoding of the source file. Remote files are cached as UTF-8.
+        fs_options: Filesystem options for remote URLs
+        shuffle_choices: Shuffle multiple choice options. An int is used as the
+            shuffle seed (as in inspect_ai's ``csv_dataset``); True shuffles unseeded.
+        refresh: Force re-download of cached remote files
+        cache_tag: Optional human-readable tag for cache filename
+
+    Returns:
+        Dataset object
+    """
+    raise NotImplementedError  # replaced by the decorated implementation below
+
+
+# Actual decorated implementations
+load_json_dataset = cache_dataset_loader()(json_dataset)  # type: ignore[assignment,arg-type]  # noqa: F811
+load_csv_dataset = cache_dataset_loader()(csv_dataset)  # type: ignore[assignment,arg-type]  # noqa: F811