agent-eval 0.1.1__tar.gz

agent_eval-0.1.1/MANIFEST.in

@@ -0,0 +1 @@
+include src/agenteval/dataset_features.yml

agent_eval-0.1.1/PKG-INFO

@@ -0,0 +1,74 @@
+Metadata-Version: 2.4
+Name: agent-eval
+Version: 0.1.1
+Summary: Agent evaluation toolkit
+Project-URL: Homepage, https://github.com/allenai/agent-eval
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: click
+Requires-Dist: inspect-ai
+Requires-Dist: litellm
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: huggingface_hub
+Requires-Dist: pyarrow
+Requires-Dist: datasets
+Provides-Extra: dev
+Requires-Dist: black==24.2.0; extra == "dev"
+Requires-Dist: isort; extra == "dev"
+Requires-Dist: autoflake; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-asyncio; extra == "dev"
+Requires-Dist: mypy==1.15; extra == "dev"
+Requires-Dist: types-PyYAML; extra == "dev"
+Requires-Dist: types-setuptools; extra == "dev"
+
+# agent-eval
+
+A utility for evaluating agents on a suite of [Inspect](https://github.com/UKGovernmentBEIS/inspect_ai)-formatted evals, with the following primary benefits:
+1. Task suite specifications as config.
+2. Extracts the token usage of the agent from log files, and computes cost using `litellm`.
+3. Submits task suite results to a leaderboard, with submission metadata and easy upload to a HuggingFace repo for distribution of scores and logs.
+
+# Installation
+
+To install from pypi, use `pip install agent-eval`.
+
+# Usage
+
+## Run evaluation suite
+```shell
+agenteval eval --config-path CONFIG_PATH --split SPLIT LOG_DIR
+```
+Evaluate an agent on the supplied eval suite configuration. Results are written to `agenteval.json` in the log directory. 
+
+See [sample-config.yml](sample-config.yml) for a sample configuration file. 
+
+For aggregation in a leaderboard, each task specifies a `primary_metric` as `{scorer_name}/{metric_name}`. 
+The scoring utils will look for a corresponding stderr metric, 
+by looking for another metric with the same `scorer_name` and with a `metric_name` containing the string "stderr".
+
+## Score results 
+```shell
+agenteval score [OPTIONS] LOG_DIR
+```
+Compute scores for the results in `agenteval.json` and update the file with the computed scores.
+
+## Publish scores
+```shell
+agenteval publish [OPTIONS] LOG_DIR
+```
+Upload the scored results to HuggingFace datasets.
+
+# Administer the HuggingFace datasets
+Prior to publishing scores, two HuggingFace datasets should be set up, one for full submissions and one for results files.
+
+If you want to call `load_dataset()` on the results dataset (e.g., for populating a leaderboard), you probably want to explicitly tell HuggingFace about the schema and dataset structure (otherwise, HuggingFace may fail to propertly auto-convert to Parquet).
+This is done by updating the `configs` attribute in the YAML metadata block at the top of the `README.md` file at the root of the results dataset (the metadata block is identified by lines with just `---` above and below it).
+This attribute should contain a list of configs, each of which specifies the schema (under the `features` key) and dataset structure (under the `data_files` key).
+See [sample-config-hf-readme-metadata.yml](sample-config-hf-readme-metadata.yml) for a sample metadata block corresponding to [sample-comfig.yml](sample-config.yml) (note that the metadata references the [raw schema data](src/agenteval/dataset_features.yml), which must be copied).
+
+To facilitate initializing new configs, `agenteval publish` will automatically add this metadata if it is missing.
+
+# Development
+
+See [Development.md](Development.md) for development instructions.

agent_eval-0.1.1/README.md

@@ -0,0 +1,50 @@
+# agent-eval
+
+A utility for evaluating agents on a suite of [Inspect](https://github.com/UKGovernmentBEIS/inspect_ai)-formatted evals, with the following primary benefits:
+1. Task suite specifications as config.
+2. Extracts the token usage of the agent from log files, and computes cost using `litellm`.
+3. Submits task suite results to a leaderboard, with submission metadata and easy upload to a HuggingFace repo for distribution of scores and logs.
+
+# Installation
+
+To install from pypi, use `pip install agent-eval`.
+
+# Usage
+
+## Run evaluation suite
+```shell
+agenteval eval --config-path CONFIG_PATH --split SPLIT LOG_DIR
+```
+Evaluate an agent on the supplied eval suite configuration. Results are written to `agenteval.json` in the log directory. 
+
+See [sample-config.yml](sample-config.yml) for a sample configuration file. 
+
+For aggregation in a leaderboard, each task specifies a `primary_metric` as `{scorer_name}/{metric_name}`. 
+The scoring utils will look for a corresponding stderr metric, 
+by looking for another metric with the same `scorer_name` and with a `metric_name` containing the string "stderr".
+
+## Score results 
+```shell
+agenteval score [OPTIONS] LOG_DIR
+```
+Compute scores for the results in `agenteval.json` and update the file with the computed scores.
+
+## Publish scores
+```shell
+agenteval publish [OPTIONS] LOG_DIR
+```
+Upload the scored results to HuggingFace datasets.
+
+# Administer the HuggingFace datasets
+Prior to publishing scores, two HuggingFace datasets should be set up, one for full submissions and one for results files.
+
+If you want to call `load_dataset()` on the results dataset (e.g., for populating a leaderboard), you probably want to explicitly tell HuggingFace about the schema and dataset structure (otherwise, HuggingFace may fail to propertly auto-convert to Parquet).
+This is done by updating the `configs` attribute in the YAML metadata block at the top of the `README.md` file at the root of the results dataset (the metadata block is identified by lines with just `---` above and below it).
+This attribute should contain a list of configs, each of which specifies the schema (under the `features` key) and dataset structure (under the `data_files` key).
+See [sample-config-hf-readme-metadata.yml](sample-config-hf-readme-metadata.yml) for a sample metadata block corresponding to [sample-comfig.yml](sample-config.yml) (note that the metadata references the [raw schema data](src/agenteval/dataset_features.yml), which must be copied).
+
+To facilitate initializing new configs, `agenteval publish` will automatically add this metadata if it is missing.
+
+# Development
+
+See [Development.md](Development.md) for development instructions.

agent_eval-0.1.1/pyproject.toml

@@ -0,0 +1,51 @@
+[build-system]
+requires = ["setuptools>=61", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "agent-eval"
+version = "0.1.1"
+description = "Agent evaluation toolkit"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+  "click",
+  "inspect-ai",
+  "litellm",
+  "pydantic>=2.0.0",
+  # For leaderboard
+  "huggingface_hub",
+  "pyarrow",
+  "datasets",
+]
+
+[project.urls]
+Homepage = "https://github.com/allenai/agent-eval"
+
+[project.optional-dependencies]
+dev = [
+  "black==24.2.0",
+  "isort",
+  "autoflake",
+  "pytest",
+  "pytest-asyncio",
+  "mypy==1.15",
+  "types-PyYAML",
+  "types-setuptools"
+]
+
+[project.scripts]
+agenteval = "agenteval.cli:cli"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.package-data]
+"agenteval" = ["dataset_features.yml"]

agent_eval-0.1.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

agent_eval-0.1.1/src/agent_eval.egg-info/PKG-INFO

@@ -0,0 +1,74 @@
+Metadata-Version: 2.4
+Name: agent-eval
+Version: 0.1.1
+Summary: Agent evaluation toolkit
+Project-URL: Homepage, https://github.com/allenai/agent-eval
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: click
+Requires-Dist: inspect-ai
+Requires-Dist: litellm
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: huggingface_hub
+Requires-Dist: pyarrow
+Requires-Dist: datasets
+Provides-Extra: dev
+Requires-Dist: black==24.2.0; extra == "dev"
+Requires-Dist: isort; extra == "dev"
+Requires-Dist: autoflake; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-asyncio; extra == "dev"
+Requires-Dist: mypy==1.15; extra == "dev"
+Requires-Dist: types-PyYAML; extra == "dev"
+Requires-Dist: types-setuptools; extra == "dev"
+
+# agent-eval
+
+A utility for evaluating agents on a suite of [Inspect](https://github.com/UKGovernmentBEIS/inspect_ai)-formatted evals, with the following primary benefits:
+1. Task suite specifications as config.
+2. Extracts the token usage of the agent from log files, and computes cost using `litellm`.
+3. Submits task suite results to a leaderboard, with submission metadata and easy upload to a HuggingFace repo for distribution of scores and logs.
+
+# Installation
+
+To install from pypi, use `pip install agent-eval`.
+
+# Usage
+
+## Run evaluation suite
+```shell
+agenteval eval --config-path CONFIG_PATH --split SPLIT LOG_DIR
+```
+Evaluate an agent on the supplied eval suite configuration. Results are written to `agenteval.json` in the log directory. 
+
+See [sample-config.yml](sample-config.yml) for a sample configuration file. 
+
+For aggregation in a leaderboard, each task specifies a `primary_metric` as `{scorer_name}/{metric_name}`. 
+The scoring utils will look for a corresponding stderr metric, 
+by looking for another metric with the same `scorer_name` and with a `metric_name` containing the string "stderr".
+
+## Score results 
+```shell
+agenteval score [OPTIONS] LOG_DIR
+```
+Compute scores for the results in `agenteval.json` and update the file with the computed scores.
+
+## Publish scores
+```shell
+agenteval publish [OPTIONS] LOG_DIR
+```
+Upload the scored results to HuggingFace datasets.
+
+# Administer the HuggingFace datasets
+Prior to publishing scores, two HuggingFace datasets should be set up, one for full submissions and one for results files.
+
+If you want to call `load_dataset()` on the results dataset (e.g., for populating a leaderboard), you probably want to explicitly tell HuggingFace about the schema and dataset structure (otherwise, HuggingFace may fail to propertly auto-convert to Parquet).
+This is done by updating the `configs` attribute in the YAML metadata block at the top of the `README.md` file at the root of the results dataset (the metadata block is identified by lines with just `---` above and below it).
+This attribute should contain a list of configs, each of which specifies the schema (under the `features` key) and dataset structure (under the `data_files` key).
+See [sample-config-hf-readme-metadata.yml](sample-config-hf-readme-metadata.yml) for a sample metadata block corresponding to [sample-comfig.yml](sample-config.yml) (note that the metadata references the [raw schema data](src/agenteval/dataset_features.yml), which must be copied).
+
+To facilitate initializing new configs, `agenteval publish` will automatically add this metadata if it is missing.
+
+# Development
+
+See [Development.md](Development.md) for development instructions.

agent_eval-0.1.1/src/agent_eval.egg-info/SOURCES.txt

@@ -0,0 +1,21 @@
+MANIFEST.in
+README.md
+pyproject.toml
+src/agent_eval.egg-info/PKG-INFO
+src/agent_eval.egg-info/SOURCES.txt
+src/agent_eval.egg-info/dependency_links.txt
+src/agent_eval.egg-info/entry_points.txt
+src/agent_eval.egg-info/requires.txt
+src/agent_eval.egg-info/top_level.txt
+src/agenteval/__init__.py
+src/agenteval/cli.py
+src/agenteval/config.py
+src/agenteval/dataset_features.yml
+src/agenteval/io.py
+src/agenteval/log.py
+src/agenteval/models.py
+src/agenteval/schema_generator.py
+src/agenteval/score.py
+src/agenteval/summary.py
+src/agenteval/upload.py
+tests/test_cli.py

agent_eval-0.1.1/src/agent_eval.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

agent_eval-0.1.1/src/agent_eval.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+agenteval = agenteval.cli:cli

agent_eval-0.1.1/src/agent_eval.egg-info/requires.txt

@@ -0,0 +1,17 @@
+click
+inspect-ai
+litellm
+pydantic>=2.0.0
+huggingface_hub
+pyarrow
+datasets
+
+[dev]
+black==24.2.0
+isort
+autoflake
+pytest
+pytest-asyncio
+mypy==1.15
+types-PyYAML
+types-setuptools

agent_eval-0.1.1/src/agent_eval.egg-info/top_level.txt

@@ -0,0 +1 @@
+agenteval

agent_eval-0.1.1/src/agenteval/__init__.py

@@ -0,0 +1,14 @@
+from importlib.metadata import version as get_version
+
+__version__ = get_version("agent-eval")
+
+from .score import process_eval_logs
+from .summary import compute_summary_statistics
+from .upload import upload_folder_to_hf, upload_summary_to_hf
+
+__all__ = [
+    "process_eval_logs",
+    "compute_summary_statistics",
+    "upload_folder_to_hf",
+    "upload_summary_to_hf",
+]