agent-eval 0.1.1__py3-none-any.whl

agent_eval-0.1.1.dist-info/METADATA

@@ -0,0 +1,74 @@
+Metadata-Version: 2.4
+Name: agent-eval
+Version: 0.1.1
+Summary: Agent evaluation toolkit
+Project-URL: Homepage, https://github.com/allenai/agent-eval
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: click
+Requires-Dist: inspect-ai
+Requires-Dist: litellm
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: huggingface_hub
+Requires-Dist: pyarrow
+Requires-Dist: datasets
+Provides-Extra: dev
+Requires-Dist: black==24.2.0; extra == "dev"
+Requires-Dist: isort; extra == "dev"
+Requires-Dist: autoflake; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-asyncio; extra == "dev"
+Requires-Dist: mypy==1.15; extra == "dev"
+Requires-Dist: types-PyYAML; extra == "dev"
+Requires-Dist: types-setuptools; extra == "dev"
+
+# agent-eval
+
+A utility for evaluating agents on a suite of [Inspect](https://github.com/UKGovernmentBEIS/inspect_ai)-formatted evals, with the following primary benefits:
+1. Task suite specifications as config.
+2. Extracts the token usage of the agent from log files, and computes cost using `litellm`.
+3. Submits task suite results to a leaderboard, with submission metadata and easy upload to a HuggingFace repo for distribution of scores and logs.
+
+# Installation
+
+To install from pypi, use `pip install agent-eval`.
+
+# Usage
+
+## Run evaluation suite
+```shell
+agenteval eval --config-path CONFIG_PATH --split SPLIT LOG_DIR
+```
+Evaluate an agent on the supplied eval suite configuration. Results are written to `agenteval.json` in the log directory. 
+
+See [sample-config.yml](sample-config.yml) for a sample configuration file. 
+
+For aggregation in a leaderboard, each task specifies a `primary_metric` as `{scorer_name}/{metric_name}`. 
+The scoring utils will look for a corresponding stderr metric, 
+by looking for another metric with the same `scorer_name` and with a `metric_name` containing the string "stderr".
+
+## Score results 
+```shell
+agenteval score [OPTIONS] LOG_DIR
+```
+Compute scores for the results in `agenteval.json` and update the file with the computed scores.
+
+## Publish scores
+```shell
+agenteval publish [OPTIONS] LOG_DIR
+```
+Upload the scored results to HuggingFace datasets.
+
+# Administer the HuggingFace datasets
+Prior to publishing scores, two HuggingFace datasets should be set up, one for full submissions and one for results files.
+
+If you want to call `load_dataset()` on the results dataset (e.g., for populating a leaderboard), you probably want to explicitly tell HuggingFace about the schema and dataset structure (otherwise, HuggingFace may fail to propertly auto-convert to Parquet).
+This is done by updating the `configs` attribute in the YAML metadata block at the top of the `README.md` file at the root of the results dataset (the metadata block is identified by lines with just `---` above and below it).
+This attribute should contain a list of configs, each of which specifies the schema (under the `features` key) and dataset structure (under the `data_files` key).
+See [sample-config-hf-readme-metadata.yml](sample-config-hf-readme-metadata.yml) for a sample metadata block corresponding to [sample-comfig.yml](sample-config.yml) (note that the metadata references the [raw schema data](src/agenteval/dataset_features.yml), which must be copied).
+
+To facilitate initializing new configs, `agenteval publish` will automatically add this metadata if it is missing.
+
+# Development
+
+See [Development.md](Development.md) for development instructions.

agent_eval-0.1.1.dist-info/RECORD

@@ -0,0 +1,16 @@
+agenteval/__init__.py,sha256=nEm7fcpqtTId-83MMLiVDKkszvaGEeFUQ_IxrdA63bo,372
+agenteval/cli.py,sha256=zzx3aceprFcQQ2fNUGBbEZ7RLt3e_gWGoR8OPDgDXaU,13767
+agenteval/config.py,sha256=jrqyAPfC728ymfYUzwWJehzeW8Dpnsmpn5utAySz0uo,2294
+agenteval/dataset_features.yml,sha256=UVsQUxohyjD9YZGw_HmIg6RFbdrkUQRZvmQSADffPS0,1439
+agenteval/io.py,sha256=QjxS2Ta_-gWRBLWOog0sX2Q-WlXmsD5iF3dsVktETR0,1004
+agenteval/log.py,sha256=lJoGkeRpuo017vAAM77mP9BIRexpWuIE8JwIKCVNOto,1958
+agenteval/models.py,sha256=UMGas9wgLT9nM1cyaVEh5PwkUURYSb02dnJDmRXI4Vk,2671
+agenteval/schema_generator.py,sha256=S36u6SSjFbAdcDqJvuzTQkFF2PG7Yq8A0diUsB8yWJg,3037
+agenteval/score.py,sha256=Pb3PofFdW0IobvZu-tv7313u_kTX3tgGw6nmlliMTak,4803
+agenteval/summary.py,sha256=LzkC2ZDf0MNLlXzL3bugAdv3VLkzCMBP85_52fsnToA,3599
+agenteval/upload.py,sha256=B17OK_ZD7q82-MjKCI63NQozB7mksaLknIdqm5ETAAU,5827
+agent_eval-0.1.1.dist-info/METADATA,sha256=mzwemf3uIdIpFHMpkJPNE7MEtiUNA_G4a0XSlcQzjkQ,3447
+agent_eval-0.1.1.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
+agent_eval-0.1.1.dist-info/entry_points.txt,sha256=DxxfiIbldqK82WRgaMOL4BjPJcnr7JOkDYTch6xNahs,48
+agent_eval-0.1.1.dist-info/top_level.txt,sha256=k6izISLxVoNnLxZHqnS3X-0eDdUD8LsV-OoM0afYdew,10
+agent_eval-0.1.1.dist-info/RECORD,,

agent_eval-0.1.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.4.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

agent_eval-0.1.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+agenteval = agenteval.cli:cli

agent_eval-0.1.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+agenteval

agenteval/__init__.py

@@ -0,0 +1,14 @@
+from importlib.metadata import version as get_version
+
+__version__ = get_version("agent-eval")
+
+from .score import process_eval_logs
+from .summary import compute_summary_statistics
+from .upload import upload_folder_to_hf, upload_summary_to_hf
+
+__all__ = [
+    "process_eval_logs",
+    "compute_summary_statistics",
+    "upload_folder_to_hf",
+    "upload_summary_to_hf",
+]

agenteval/cli.py

@@ -0,0 +1,412 @@
+#!/usr/bin/env python3
+import json
+import os
+import subprocess
+from datetime import datetime, timezone
+from pathlib import Path
+
+import click
+
+from .config import load_suite_config
+from .models import EvalConfig, EvalResult
+from .score import process_eval_logs
+from .summary import compute_summary_statistics
+from .upload import sanitize_path_component, upload_folder_to_hf, upload_summary_to_hf
+
+EVAL_FILENAME = "agenteval.json"
+
+
+def verify_git_reproducibility(ignore_git: bool) -> None:
+    if ignore_git:
+        return
+    try:
+        # Get current commit SHA and origin
+        sha_result = subprocess.run(
+            ["git", "rev-parse", "--short", "HEAD"],
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        origin_result = subprocess.run(
+            ["git", "remote", "get-url", "origin"],
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        sha = sha_result.stdout.strip() if sha_result.returncode == 0 else None
+        origin = origin_result.stdout.strip() if origin_result.returncode == 0 else None
+
+        # Check for dirty working directory
+        git_dirty = (
+            subprocess.run(
+                ["git", "diff", "--quiet", "--exit-code"],
+                capture_output=True,
+                check=False,
+            ).returncode
+            != 0
+        )
+
+        # Warn about untracked (non-ignored) files
+        untracked_result = subprocess.run(
+            ["git", "ls-files", "--others", "--exclude-standard"],
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        untracked_files = untracked_result.stdout.strip().splitlines()
+        if untracked_files:
+            click.echo(
+                f"Warning: Untracked files present: {', '.join(untracked_files)}. "
+                "For reproducibility, please add, ignore, or remove these files."
+            )
+
+        # Abort if worktree is dirty
+        if git_dirty:
+            raise click.ClickException(
+                f"Git working directory contains uncommitted changes. "
+                f"For reproducibility, Inspect will save: origin={origin}, sha={sha}. "
+                "Please commit your changes or use --ignore-git to bypass this check (not recommended)."
+            )
+
+        # Check if commit exists on remote
+        if sha:
+            remote_exists = subprocess.run(
+                ["git", "branch", "-r", "--contains", sha],
+                capture_output=True,
+                text=True,
+                check=True,
+            ).stdout.strip()
+            if not remote_exists:
+                raise click.ClickException(
+                    f"Commit {sha} not found on remote '{origin}'. Others won't be able to "
+                    "access this code version. Please push your changes or use --ignore-git "
+                    "to bypass this check (not recommended)."
+                )
+    except (subprocess.SubprocessError, FileNotFoundError) as e:
+        if isinstance(e, click.ClickException):
+            raise
+        raise click.ClickException(
+            f"Unable to verify git status for reproducibility: {e}. "
+            "Use --ignore-git to bypass this check if git is not available."
+        )
+
+
+@click.group()
+def cli():
+    pass
+
+
+@click.command(
+    name="score",
+    help="Score a directory of evaluation logs.",
+)
+@click.argument("log_dir", type=click.Path(exists=True, file_okay=False))
+@click.option(
+    "--config-path",
+    "config_path",
+    type=str,
+    help=f"Path to a yml config file. Ignored if {EVAL_FILENAME} exists.",
+    default=None,
+)
+@click.option(
+    "--split",
+    type=str,
+    help=f"Config data split. Ignored if {EVAL_FILENAME} exists.",
+    default=None,
+)
+def score_command(
+    log_dir: str,
+    config_path: str | None,
+    split: str | None,
+):
+    # Load or create EvalResult and process logs (inlined from processor)
+    json_path = Path(log_dir) / EVAL_FILENAME
+    if json_path.exists():
+        try:
+            raw = json_path.read_text(encoding="utf-8")
+            eval_result = EvalResult.model_validate_json(raw)
+        except Exception as e:
+            raise click.ClickException(
+                f"Failed to load existing '{EVAL_FILENAME}' at {json_path}: {e}"
+            )
+        if config_path:
+            try:
+                cli_cfg = load_suite_config(config_path)
+                if cli_cfg.version != eval_result.suite_config.version:
+                    click.echo(
+                        f"Warning: CLI config version '{cli_cfg.version}' "
+                        f"does not match JSON config version '{eval_result.suite_config.version}'."
+                    )
+            except Exception as e:
+                click.echo(
+                    f"Warning: could not load CLI config '{config_path}' for comparison: {e}"
+                )
+        if split and split != eval_result.split:
+            raise click.ClickException(
+                f"Split mismatch: JSON split '{eval_result.split}' != CLI split '{split}'"
+            )
+    else:
+        if not config_path or not split:
+            raise click.ClickException(
+                "--config-path and --split must be provided when no existing result JSON"
+            )
+        suite_cfg = load_suite_config(config_path)
+        eval_result = EvalResult(suite_config=suite_cfg, split=split)
+
+    task_results, eval_specs = process_eval_logs(log_dir)
+    eval_result.eval_specs = eval_specs
+    eval_result.results = task_results
+
+    # Warn if multiple evaluation specs present
+    if eval_result.eval_specs and len(eval_result.eval_specs) > 1:
+        click.echo(
+            f"Warning: Found {len(eval_result.eval_specs)} different eval specs. "
+            "Logs may come from mixed runs."
+        )
+
+    # Warn about any missing tasks
+    missing_tasks = eval_result.find_missing_tasks()
+    if missing_tasks:
+        click.echo(f"Warning: Missing tasks in result set: {', '.join(missing_tasks)}")
+
+    # Compute and display summary statistics
+    stats = compute_summary_statistics(
+        eval_result.suite_config,
+        eval_result.split,
+        eval_result.results or [],
+    )
+    click.echo("Summary statistics:")
+    click.echo(json.dumps({k: v.model_dump() for k, v in stats.items()}, indent=2))
+
+    # Persist updated EvalResult JSON
+    eval_result.save_json(Path(log_dir) / EVAL_FILENAME)
+
+    click.echo(f"Saved results to {log_dir}/{EVAL_FILENAME}")
+    ctx = click.get_current_context()
+    click.echo(
+        f"You can now run '{ctx.parent.info_name if ctx.parent else 'cli'} publish {log_dir}' to publish the results"
+    )
+
+
+cli.add_command(score_command)
+
+
+@click.command(
+    name="publish",
+    help="Publish scored results in log_dir to Hugging Face leaderboard.",
+)
+@click.argument("log_dir", type=click.Path(exists=True, file_okay=False))
+@click.option(
+    "--submissions-repo-id",
+    type=str,
+    default=lambda: os.environ.get("SUBMISSIONS_REPO_ID", ""),
+    help="HF repo id for submissions. Defaults to SUBMISSIONS_REPO_ID env var.",
+)
+@click.option(
+    "--results-repo-id",
+    type=str,
+    default=lambda: os.environ.get("RESULTS_REPO_ID", ""),
+    help="HF repo id for result stats. Defaults to RESULTS_REPO_ID env var.",
+)
+@click.option(
+    "--username",
+    type=str,
+    default=None,
+    help="HF username/org for submission. Defaults to your HF account name.",
+)
+@click.option(
+    "--agent-name",
+    type=str,
+    required=True,
+    help="Descriptive agent name for submission.",
+)
+@click.option(
+    "--agent-description",
+    type=str,
+    default=None,
+    help="Description of the agent being submitted.",
+)
+@click.option(
+    "--agent-url",
+    type=str,
+    default=None,
+    help="URL to the agent's repository or documentation.",
+)
+def publish_command(
+    log_dir: str,
+    submissions_repo_id: str,
+    results_repo_id: str,
+    username: str | None,
+    agent_name: str,
+    agent_description: str | None,
+    agent_url: str | None,
+):
+    # Allow huggingface imports to be optional
+    from huggingface_hub import HfApi
+
+    # Derive a filesafe agent_name
+    safe_agent_name = sanitize_path_component(agent_name)
+    if safe_agent_name != agent_name:
+        click.echo(
+            f"Note: agent_name '{agent_name}' contains unsafe characters; "
+            f"using '{safe_agent_name}' for submission filenames."
+        )
+
+    # Load existing scored results from JSON
+    json_path = Path(log_dir) / EVAL_FILENAME
+    if not json_path.exists():
+        raise click.ClickException(f"No scored results found at {json_path}")
+    raw = json_path.read_text(encoding="utf-8")
+    eval_result = EvalResult.model_validate_json(raw)
+
+    # Validate eval result
+    if not eval_result.is_scored():
+        raise click.ClickException(
+            f"{EVAL_FILENAME} is not scored. Please run 'score {log_dir}' first."
+        )
+    missing_tasks = eval_result.find_missing_tasks()
+    if missing_tasks:
+        click.echo(f"Warning: Missing tasks in result set: {', '.join(missing_tasks)}")
+
+    # Determine HF user
+    hf_api = HfApi()
+    if not username:
+        try:
+            username = hf_api.whoami()["name"]
+            assert isinstance(username, str), "Invalid username type from HF API"
+            click.echo(f"Defaulting username to Hugging Face account: {username}")
+        except Exception:
+            raise click.ClickException(
+                "--username must be provided or ensure HF authentication is configured"
+            )
+
+    # Derive a filesafe username
+    safe_username = sanitize_path_component(username)
+    if safe_username != username:
+        click.echo(
+            f"Note: username '{username}' contains unsafe characters; "
+            f"using '{safe_username}' for submission filenames."
+        )
+
+    # Fill submission metadata
+    eval_result.submission.username = username
+    eval_result.submission.agent_name = agent_name
+    eval_result.submission.agent_description = agent_description
+    eval_result.submission.agent_url = agent_url
+    eval_result.submission.submit_time = datetime.now(timezone.utc)
+
+    # Validate suite config version
+    config_name = eval_result.suite_config.version
+    if not config_name:
+        raise click.ClickException("Suite config version is required for upload.")
+
+    # Build submission name
+    ts = eval_result.submission.submit_time.strftime("%Y-%m-%dT%H-%M-%S")
+    subm_name = f"{safe_username}_{safe_agent_name}_{ts}"
+
+    # Upload logs and summary
+    logs_url = upload_folder_to_hf(
+        hf_api, log_dir, submissions_repo_id, config_name, eval_result.split, subm_name
+    )
+    click.echo(f"Uploaded submission logs dir to {logs_url}")
+    eval_result.submission.logs_url = logs_url
+
+    summary_url = upload_summary_to_hf(
+        hf_api, eval_result, results_repo_id, config_name, eval_result.split, subm_name
+    )
+    click.echo(f"Uploaded results summary file to {summary_url}")
+    eval_result.submission.summary_url = summary_url
+
+    # Save updated JSON
+    eval_result.save_json(Path(log_dir) / EVAL_FILENAME)
+    click.echo(f"Updated {EVAL_FILENAME} with publication metadata.")
+
+
+cli.add_command(publish_command)
+
+
+@cli.command(
+    name="eval",
+    help="Run inspect eval-set on specified tasks with the given arguments",
+    context_settings={"ignore_unknown_options": True},
+)
+@click.option(
+    "--log-dir",
+    type=str,
+    help="Log directory. Defaults to INSPECT_LOG_DIR or auto-generated under ./logs.",
+)
+@click.option(
+    "--config-path",
+    "config_path",
+    type=str,
+    help="Path to a yml config file.",
+    required=True,
+)
+@click.option(
+    "--split",
+    type=str,
+    help="Config data split.",
+    required=True,
+)
+@click.option(
+    "--ignore-git",
+    is_flag=True,
+    help="Ignore git reproducibility checks (not recommended).",
+)
+@click.argument("args", nargs=-1, type=click.UNPROCESSED)
+def eval_command(
+    log_dir: str | None,
+    config_path: str,
+    split: str,
+    ignore_git: bool,
+    args: tuple[str],
+):
+    """Run inspect eval-set with arguments and append tasks"""
+    suite_config = load_suite_config(config_path)
+    tasks = suite_config.get_tasks(split)
+
+    # Verify git status for reproducibility
+    verify_git_reproducibility(ignore_git)
+
+    if not log_dir:
+        log_dir = os.environ.get("INSPECT_LOG_DIR")
+        if not log_dir:
+            timestamp = datetime.now().strftime("%Y-%m-%dT%H-%M-%S")
+            log_dir = os.path.join(
+                ".",
+                "logs",
+                f"{suite_config.name}_{suite_config.version}_{split}_{timestamp}",
+            )
+            click.echo(f"No log dir was manually set; using {log_dir}")
+    logd_args = ["--log-dir", log_dir]
+
+    # We use subprocess here to keep arg management simple; an alternative
+    # would be calling `inspect_ai.eval_set()` directly, which would allow for
+    # programmatic execution
+    full_command = (
+        ["inspect", "eval-set"] + list(args) + logd_args + [x.path for x in tasks]
+    )
+    click.echo(f"Running {config_path}: {' '.join(full_command)}")
+    proc = subprocess.run(full_command)
+
+    if proc.returncode != 0:
+        raise click.ClickException(
+            f"inspect eval-set failed while running {config_path}"
+        )
+
+    # Write the config portion of the results file
+    with open(os.path.join(log_dir, EVAL_FILENAME), "w", encoding="utf-8") as f:
+        unscored_eval_config = EvalConfig(suite_config=suite_config, split=split)
+        f.write(unscored_eval_config.model_dump_json(indent=2))
+
+    ctx = click.get_current_context()
+    click.echo(
+        f"You can now run '{ctx.parent.info_name if ctx.parent else 'cli'} score {log_dir}' to score the results"
+    )
+
+
+cli.add_command(eval_command)
+
+
+if __name__ == "__main__":
+    cli()

agenteval/config.py

@@ -0,0 +1,87 @@
+"""
+Configuration management for agent evaluation.
+"""
+
+import yaml
+from pydantic import BaseModel, ValidationError
+
+
+class Task(BaseModel):
+    name: str
+    """Canonical task name (used by the leaderboard)."""
+
+    path: str
+    """Path to the task definition (used by Inspect)."""
+
+    primary_metric: str
+    """Primary metric for the task, used for summary scores."""
+
+    tags: list[str] | None = None
+    """List of tags, used for computing summary scores for task groups."""
+
+
+class Split(BaseModel):
+    name: str
+    """Name of the split."""
+
+    tasks: list[Task]
+    """List of tasks associated with the split."""
+
+
+class SuiteConfig(BaseModel):
+    name: str
+    """Name of the suite."""
+
+    version: str | None = None
+    """Version of the suite, e.g. '1.0.0.dev1'."""
+
+    splits: list[Split]
+    """List of splits in the suite."""
+
+    def get_tasks(self, split_name: str) -> list[Task]:
+        """
+        Get the tasks for a specific split.
+
+        Args:
+            split_name: Name of the split to retrieve tasks from
+
+        Returns:
+            List of Task objects for the specified split
+
+        Raises:
+            ValueError: If the split is not found
+        """
+        for split in self.splits:
+            if split.name == split_name:
+                return split.tasks
+
+        available_splits = ", ".join(split.name for split in self.splits)
+        raise ValueError(
+            f"Split '{split_name}' not found. Available splits: {available_splits}"
+        )
+
+
+def load_suite_config(file_path: str) -> SuiteConfig:
+    """
+    Load the suite configuration from the specified YAML file.
+
+    Args:
+        file_path: Path to the YAML file containing the suite/tasks configuration
+
+    Returns:
+        A validated SuiteConfig object
+    """
+    try:
+        with open(file_path, "r") as f:
+            config_data = yaml.safe_load(f)
+    except FileNotFoundError:
+        raise FileNotFoundError(f"Task configuration file not found: {file_path}")
+    except yaml.YAMLError as e:
+        raise ValueError(f"Failed to parse YAML file: {e}")
+
+    try:
+        return SuiteConfig.model_validate(config_data)
+    except ValidationError as e:
+        raise ValueError(
+            f"Invalid task configuration: {e}\nPlease refer to the config spec."
+        )

agenteval/dataset_features.yml

@@ -0,0 +1,71 @@
+- name: suite_config
+  struct:
+  - name: name
+    dtype: string
+  - name: version
+    dtype: string
+  - name: splits
+    list:
+    - name: name
+      dtype: string
+    - name: tasks
+      list:
+      - name: name
+        dtype: string
+      - name: path
+        dtype: string
+      - name: primary_metric
+        dtype: string
+      - name: tags
+        sequence: string
+- name: split
+  dtype: string
+- name: results
+  list:
+  - name: task_name
+    dtype: string
+  - name: metrics
+    list:
+    - name: name
+      dtype: string
+    - name: value
+      dtype: float64
+  - name: model_usages
+    list:
+      list:
+      - name: model
+        dtype: string
+      - name: usage
+        struct:
+        - name: input_tokens
+          dtype: int64
+        - name: output_tokens
+          dtype: int64
+        - name: total_tokens
+          dtype: int64
+        - name: input_tokens_cache_write
+          dtype: int64
+        - name: input_tokens_cache_read
+          dtype: int64
+        - name: reasoning_tokens
+          dtype: int64
+  - name: model_costs
+    sequence: float64
+- name: submission
+  struct:
+  - name: submit_time
+    dtype: timestamp[us, tz=UTC]
+  - name: username
+    dtype: string
+  - name: agent_name
+    dtype: string
+  - name: agent_description
+    dtype: string
+  - name: agent_url
+    dtype: string
+  - name: logs_url
+    dtype: string
+  - name: logs_url_public
+    dtype: string
+  - name: summary_url
+    dtype: string