agentinputlab 0.1.0a1__py3-none-any.whl

agentinputlab/benchmarks/assets.py

@@ -0,0 +1,226 @@
+"""Versioned, checksum-pinned benchmark smoke assets.
+
+The benchmark inventory describes experimental coverage.  This module
+is intentionally separate: it lists only source files that have been fetched,
+byte-verified, and assigned a deterministic preprocessing recipe.  A registry
+cell therefore does not imply that its full data is already materializable.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import urllib.parse
+from collections.abc import Mapping
+from dataclasses import dataclass
+from importlib import resources
+from types import MappingProxyType
+from typing import Any
+
+from agentinputlab.artifacts import AcquisitionPolicy, ArtifactKind, ArtifactManifest
+from agentinputlab.contracts.errors import RegistryError
+
+_SHA256 = re.compile(r"^[0-9a-f]{64}$")
+_SAFE_SEGMENT = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
+_KNOWN_STATUSES = {"verified_smoke", "license_review_required"}
+_KNOWN_REDISTRIBUTION = {"downloader_only", "not_redistributed"}
+_KNOWN_PROCESSORS = {
+    "beir_zip_v1",
+    "bfcl_v3_simple_tar_v1",
+    "bright_corpus_parquet_v1",
+    "bright_examples_parquet_v1",
+    "clariq_tsv_v1",
+    "hotpotqa_parquet_v1",
+    "musique_zip_v1",
+    "qasper_tgz_v1",
+    "tau3_banking_tar_v1",
+    "two_wiki_zip_v1",
+}
+
+
+def _string_mapping(value: Any, *, label: str) -> Mapping[str, str]:
+    if not isinstance(value, Mapping):
+        raise RegistryError(f"{label} must be a mapping")
+    converted: dict[str, str] = {}
+    for key, item in value.items():
+        if not isinstance(key, str) or not isinstance(item, str):
+            raise RegistryError(f"{label} keys and values must be strings")
+        converted[key] = item
+    return converted
+
+
+def _count_mapping(value: Any, *, label: str) -> Mapping[str, int]:
+    if not isinstance(value, Mapping):
+        raise RegistryError(f"{label} must be a mapping")
+    converted: dict[str, int] = {}
+    for key, item in value.items():
+        if not isinstance(key, str) or not isinstance(item, int) or isinstance(item, bool):
+            raise RegistryError(f"{label} keys must be strings and values must be integers")
+        if item < 0:
+            raise RegistryError(f"{label} values must be non-negative")
+        converted[key] = item
+    return converted
+
+
+@dataclass(frozen=True, slots=True)
+class BenchmarkSmokeAsset:
+    asset_id: str
+    cell_id: str
+    source_url: str
+    source_revision: str
+    sha256: str
+    size_bytes: int
+    license_id: str
+    redistribution: str
+    status: str
+    processor: str
+    expected_counts: Mapping[str, int]
+    prepared_sha256: Mapping[str, str]
+    allowed_hosts: tuple[str, ...]
+    provenance: Mapping[str, str]
+    note: str = ""
+
+    def __post_init__(self) -> None:
+        object.__setattr__(
+            self,
+            "allowed_hosts",
+            tuple(sorted({host.strip().lower() for host in self.allowed_hosts if host.strip()})),
+        )
+        object.__setattr__(self, "expected_counts", MappingProxyType(dict(self.expected_counts)))
+        object.__setattr__(self, "prepared_sha256", MappingProxyType(dict(self.prepared_sha256)))
+        object.__setattr__(self, "provenance", MappingProxyType(dict(self.provenance)))
+        parsed = urllib.parse.urlsplit(self.source_url)
+        if not _SAFE_SEGMENT.fullmatch(self.asset_id) or ".." in self.asset_id:
+            raise RegistryError("benchmark asset_id must be a flat safe identifier")
+        if not _SAFE_SEGMENT.fullmatch(self.source_revision) or ".." in self.source_revision:
+            raise RegistryError("benchmark source_revision must be a flat safe identifier")
+        if parsed.scheme != "https" or not parsed.hostname:
+            raise RegistryError(f"benchmark asset {self.asset_id!r} requires an HTTPS source")
+        if parsed.username is not None or parsed.password is not None:
+            raise RegistryError("benchmark asset URLs must not contain user information")
+        if not _SHA256.fullmatch(self.sha256):
+            raise RegistryError(f"benchmark asset {self.asset_id!r} has an invalid SHA-256")
+        if isinstance(self.size_bytes, bool) or not isinstance(self.size_bytes, int):
+            raise RegistryError(f"benchmark asset {self.asset_id!r} size must be an integer")
+        if self.size_bytes < 1:
+            raise RegistryError(f"benchmark asset {self.asset_id!r} has an invalid size")
+        if self.status not in _KNOWN_STATUSES:
+            raise RegistryError(f"benchmark asset {self.asset_id!r} has an unknown status")
+        if self.redistribution not in _KNOWN_REDISTRIBUTION:
+            raise RegistryError(
+                f"benchmark asset {self.asset_id!r} has an unknown redistribution policy"
+            )
+        if self.processor not in _KNOWN_PROCESSORS:
+            raise RegistryError(f"benchmark asset {self.asset_id!r} has an unknown processor")
+        if any(not _SHA256.fullmatch(digest) for digest in self.prepared_sha256.values()):
+            raise RegistryError(
+                f"benchmark asset {self.asset_id!r} has an invalid prepared SHA-256"
+            )
+        if not set(self.prepared_sha256) <= {
+            "audit.jsonl",
+            "inputs.jsonl",
+            "gold.jsonl",
+            "corpus.jsonl",
+        }:
+            raise RegistryError(f"benchmark asset {self.asset_id!r} has unknown prepared files")
+        if not self.allowed_hosts or parsed.hostname.lower() not in self.allowed_hosts:
+            raise RegistryError(
+                f"benchmark asset {self.asset_id!r} must allowlist its source hostname"
+            )
+        if self.license_id == "NOASSERTION" and self.status != "license_review_required":
+            raise RegistryError("NOASSERTION benchmark assets must remain license-review gated")
+
+    @classmethod
+    def from_mapping(cls, value: Mapping[str, Any]) -> BenchmarkSmokeAsset:
+        try:
+            hosts = value["allowed_hosts"]
+            if not isinstance(hosts, list) or not all(isinstance(item, str) for item in hosts):
+                raise RegistryError("benchmark asset allowed_hosts must be a string list")
+            expected_counts = _count_mapping(
+                value["expected_counts"], label="benchmark asset expected_counts"
+            )
+            provenance = _string_mapping(
+                value.get("provenance", {}), label="benchmark asset provenance"
+            )
+            prepared_sha256 = _string_mapping(
+                value.get("prepared_sha256", {}), label="benchmark asset prepared_sha256"
+            )
+            return cls(
+                asset_id=str(value["asset_id"]),
+                cell_id=str(value["cell_id"]),
+                source_url=str(value["source_url"]),
+                source_revision=str(value["source_revision"]),
+                sha256=str(value["sha256"]),
+                size_bytes=int(value["size_bytes"]),
+                license_id=str(value["license_id"]),
+                redistribution=str(value["redistribution"]),
+                status=str(value["status"]),
+                processor=str(value["processor"]),
+                expected_counts=expected_counts,
+                prepared_sha256=prepared_sha256,
+                allowed_hosts=tuple(sorted({item.lower() for item in hosts})),
+                provenance=provenance,
+                note=str(value.get("note", "")),
+            )
+        except (KeyError, TypeError, ValueError) as exc:
+            raise RegistryError(f"invalid benchmark smoke asset: {value!r}") from exc
+
+    def artifact_manifest(self) -> ArtifactManifest:
+        """Project the source lock into the shared artifact-store contract."""
+
+        return ArtifactManifest(
+            artifact_id=self.asset_id,
+            kind=ArtifactKind.DATASET,
+            revision=self.source_revision,
+            sha256=self.sha256,
+            acquisition=AcquisitionPolicy.DOWNLOAD,
+            license_id=self.license_id,
+            source_uri=self.source_url,
+            size_bytes=self.size_bytes,
+            lineage={"benchmark_cell": self.cell_id, "processor": self.processor},
+            metadata={
+                "redistribution": self.redistribution,
+                "status": self.status,
+                "provenance": dict(self.provenance),
+                "prepared_sha256": dict(self.prepared_sha256),
+            },
+        )
+
+
+class BenchmarkSmokeAssetRegistry:
+    def __init__(self, assets: tuple[BenchmarkSmokeAsset, ...]) -> None:
+        if not assets:
+            raise RegistryError("benchmark smoke asset registry must not be empty")
+        if len({asset.asset_id for asset in assets}) != len(assets):
+            raise RegistryError("benchmark smoke asset IDs must be unique")
+        self._assets = assets
+        self._by_id = {asset.asset_id: asset for asset in assets}
+
+    @classmethod
+    def default(cls) -> BenchmarkSmokeAssetRegistry:
+        resource = resources.files("agentinputlab.benchmarks").joinpath("data", "smoke_assets.json")
+        payload = json.loads(resource.read_text(encoding="utf-8"))
+        if payload.get("schema_version") != 1:
+            raise RegistryError("unsupported benchmark smoke asset registry snapshot")
+        raw_assets = payload.get("assets")
+        if not isinstance(raw_assets, list) or payload.get("asset_count") != len(raw_assets):
+            raise RegistryError("benchmark smoke asset count does not match its payload")
+        return cls(tuple(BenchmarkSmokeAsset.from_mapping(item) for item in raw_assets))
+
+    def get(self, asset_id: str) -> BenchmarkSmokeAsset:
+        try:
+            return self._by_id[asset_id]
+        except KeyError as exc:
+            raise RegistryError(f"unknown benchmark smoke asset: {asset_id}") from exc
+
+    def list(
+        self,
+        *,
+        include_license_review: bool = False,
+    ) -> tuple[BenchmarkSmokeAsset, ...]:
+        if include_license_review:
+            return self._assets
+        return tuple(asset for asset in self._assets if asset.status == "verified_smoke")
+
+    def __len__(self) -> int:
+        return len(self._assets)

agentinputlab/benchmarks/base.py

@@ -0,0 +1,68 @@
+"""Benchmark adapter contracts that physically isolate evaluator gold."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from typing import Any, Protocol, runtime_checkable
+
+from agentinputlab.contracts.serialization import content_hash
+from agentinputlab.contracts.state import (
+    ContextBlock,
+    ExecutionBudget,
+    FrozenAgentState,
+    Message,
+)
+
+
+@dataclass(frozen=True, slots=True)
+class BenchmarkExample:
+    example_id: str
+    query: str
+    instruction: str = ""
+    visible_context: tuple[ContextBlock, ...] = ()
+    visible_messages: tuple[Message, ...] = ()
+    visible_metadata: Mapping[str, Any] = field(default_factory=dict)
+    evaluator_gold: Mapping[str, Any] = field(default_factory=dict, repr=False)
+    visible_tool_names: tuple[str, ...] = ()
+    optimizer_visible_id: str | None = None
+
+    def visible_state(self, *, budget: ExecutionBudget | None = None) -> FrozenAgentState:
+        """Create a state with no reference to evaluator-only fields."""
+
+        return FrozenAgentState(
+            task_id=self.optimizer_visible_id or self.example_id,
+            query=self.query,
+            instruction=self.instruction,
+            context=self.visible_context,
+            messages=self.visible_messages,
+            tool_names=self.visible_tool_names,
+            metadata=dict(self.visible_metadata),
+            budget=budget or ExecutionBudget(),
+        )
+
+
+def opaque_checkpoint_state_id(
+    benchmark_id: str,
+    checkpoint_id: str,
+    replay_identity: str,
+) -> str:
+    """Derive a stable predictor-visible ID without exposing protected labels."""
+
+    return "ailcp_" + content_hash(
+        {
+            "benchmark_id": benchmark_id,
+            "checkpoint_id": checkpoint_id,
+            "replay_identity": replay_identity,
+        }
+    )
+
+
+@runtime_checkable
+class BenchmarkAdapter(Protocol):
+    benchmark_id: str
+    pairing_mode: str
+
+    def examples(self, split: str) -> tuple[BenchmarkExample, ...]: ...
+
+    def score(self, example: BenchmarkExample, prediction: Any) -> Mapping[str, float]: ...

agentinputlab/benchmarks/beir_archive.py

@@ -0,0 +1,235 @@
+"""Checksum-pinned BEIR archives with streamed corpus access and held-out qrels.
+
+No dataset is bundled. This loader preserves the complete corpus and official
+test judgments, including queries whose IDs overlap corpus IDs. CQADupstack
+uses explicit forum namespaces and never merges equal IDs from different forums.
+"""
+
+from __future__ import annotations
+
+import csv
+import io
+import json
+import stat
+import zipfile
+from pathlib import Path, PurePosixPath
+from types import MappingProxyType
+
+from agentinputlab.artifacts import sha256_file
+from agentinputlab.backends import Document
+from agentinputlab.contracts import ArtifactError, ValidationError
+
+from .base import BenchmarkExample
+from .retrieval import PreparedRetrievalBenchmarkAdapter
+
+# Raw source hashes independently match the upstream BEIR download table's MD5.
+BEIR_ARCHIVE_LOCKS = {
+    "beir.fiqa_2018": {
+        "dataset": "fiqa",
+        "size_bytes": 17948027,
+        "sha256": "32c7df99ed21252fdfb2cf3f5673502a8d245ee0c44c4a133570d92ce2b3ad02",
+        "upstream_md5": "17918ed23cd04fb15047f73e6c3bd9d9",
+    },
+    "beir.quora": {
+        "dataset": "quora",
+        "size_bytes": 15853968,
+        "sha256": "56aacd9dcc4d9c093b63f175afdda0e21cbc8442ecf6c717d09de7b358d77531",
+        "upstream_md5": "18fb154900ba42a600f84b839c173167",
+    },
+    "beir.dbpedia_entity": {
+        "dataset": "dbpedia-entity",
+        "size_bytes": 639285131,
+        "sha256": "e4c13080b5399a19e02b04c908c485f8997ac9e265a06ef5f9f28c3f1134c472",
+        "upstream_md5": "c2a39eb420a3164af735795df012ac2c",
+    },
+    "beir.fever": {
+        "dataset": "fever",
+        "size_bytes": 1236988269,
+        "sha256": "58b3faee0c152158b27970380f778ea56e4e34482034404226c10e11eadc7b72",
+        "upstream_md5": "5a818580227bfb4b35bb6fa46d9b6c03",
+    },
+}
+_FORUMS = (
+    "android",
+    "english",
+    "gaming",
+    "gis",
+    "mathematica",
+    "physics",
+    "programmers",
+    "stats",
+    "tex",
+    "unix",
+    "webmasters",
+    "wordpress",
+)
+
+
+class BEIRArchiveAdapter(PreparedRetrievalBenchmarkAdapter):
+    """Official test cases, native IR measures, and a streaming complete corpus."""
+
+    def __init__(
+        self, *, benchmark_id: str, source_archive: str | Path, license_acknowledged: bool = False
+    ):
+        if not license_acknowledged:
+            raise ValidationError(
+                "mounted BEIR dataset requires license_acknowledged=True; data licenses are separate from code"
+            )
+        if benchmark_id not in BEIR_ARCHIVE_LOCKS:
+            raise ValidationError("BEIR archive cell has no verified source lock")
+        self.lock = dict(BEIR_ARCHIVE_LOCKS[benchmark_id])
+        self.path = Path(source_archive)
+        if not self.path.is_file() or self.path.stat().st_size != self.lock["size_bytes"]:
+            raise ArtifactError("BEIR archive size differs")
+        sha = sha256_file(self.path)
+        if sha != self.lock["sha256"]:
+            raise ArtifactError("BEIR archive checksum differs")
+        self.benchmark_id = benchmark_id
+        self.split = "test"
+        self._ignore_identical_ids = True
+        with zipfile.ZipFile(self.path) as archive:
+            self.groups = self._validate(archive)
+            examples = []
+            for group, prefix in self.groups:
+                qrels = {}
+                with archive.open(prefix + "qrels/test.tsv") as raw:
+                    reader = csv.DictReader(io.TextIOWrapper(raw, encoding="utf-8"), delimiter="\t")
+                    if reader.fieldnames != ["query-id", "corpus-id", "score"]:
+                        raise ArtifactError("BEIR qrels schema differs")
+                    for row in reader:
+                        qid = self._id(group, row["query-id"])
+                        did = self._id(group, row["corpus-id"])
+                        score = int(row["score"])
+                        if score < 0 or did in qrels.setdefault(qid, {}):
+                            raise ArtifactError("BEIR invalid or duplicate judgment")
+                        qrels[qid][did] = score
+                queries = {}
+                for row in self._rows(archive, prefix + "queries.jsonl"):
+                    qid = self._id(group, row["_id"])
+                    if qid in queries:
+                        raise ArtifactError("BEIR duplicate query ID")
+                    if not isinstance(row["text"], str) or not row["text"].strip():
+                        raise ArtifactError("BEIR query text is empty")
+                    queries[qid] = row["text"]
+                if set(qrels) - set(queries):
+                    raise ArtifactError("BEIR qrels refer to absent queries")
+                for qid in sorted(qrels):
+                    if not any(s > 0 for s in qrels[qid].values()):
+                        raise ArtifactError("BEIR test query lacks positive judgment")
+                    examples.append(
+                        BenchmarkExample(
+                            example_id=qid,
+                            query=queries[qid],
+                            instruction="Retrieve the most relevant "
+                            + (
+                                "duplicate questions."
+                                if self.lock["dataset"] in ("quora", "cqadupstack")
+                                else "documents."
+                            ),
+                            visible_metadata=MappingProxyType(
+                                {
+                                    "benchmark_cell": benchmark_id,
+                                    "source_split": "test",
+                                    "forum": group,
+                                }
+                            ),
+                            evaluator_gold=MappingProxyType(
+                                {"relevance": MappingProxyType(qrels[qid])}
+                            ),
+                        )
+                    )
+        self._examples = tuple(examples)
+        self._examples_by_id = {e.example_id: e for e in examples}
+
+    def _validate(self, archive):
+        infos = archive.infolist()
+        names = set()
+        total = 0
+        if len(infos) > 256:
+            raise ArtifactError("BEIR archive has too many members")
+        for info in infos:
+            path = PurePosixPath(info.filename)
+            if (
+                path.is_absolute()
+                or ".." in path.parts
+                or "\\" in info.filename
+                or info.filename in names
+                or stat.S_ISLNK(info.external_attr >> 16)
+            ):
+                raise ArtifactError("BEIR archive contains unsafe/duplicate members")
+            names.add(info.filename)
+            total += info.file_size
+            if info.file_size > 12 * 1024**3 or info.file_size / max(info.compress_size, 1) > 400:
+                raise ArtifactError("BEIR archive member is oversized")
+        if total > 24 * 1024**3:
+            raise ArtifactError("BEIR archive is oversized")
+        dataset = self.lock["dataset"]
+        if dataset == "cqadupstack":
+            groups = tuple((f, "cqadupstack/" + f + "/") for f in _FORUMS)
+        else:
+            groups = (("", dataset + "/"),)
+        for _, prefix in groups:
+            if (
+                not {prefix + "corpus.jsonl", prefix + "queries.jsonl", prefix + "qrels/test.tsv"}
+                <= names
+            ):
+                raise ArtifactError("BEIR required corpus/query/test members missing")
+        return groups
+
+    @staticmethod
+    def _id(group, value):
+        if not isinstance(value, str) or not value:
+            raise ArtifactError("BEIR source ID must be nonempty text")
+        return group + "::" + value if group else value
+
+    @staticmethod
+    def _rows(archive, name):
+        with archive.open(name) as raw:
+            for line in io.TextIOWrapper(raw, encoding="utf-8"):
+                row = json.loads(line)
+                if not isinstance(row, dict):
+                    raise ArtifactError("BEIR JSONL record must be object")
+                yield row
+
+    def iter_documents(self, *, forum: str | None = None):
+        """Yield all source documents, without loading full corpora into RAM."""
+        if forum is not None and forum not in {g for g, _ in self.groups}:
+            raise ValidationError("unknown BEIR forum")
+        with zipfile.ZipFile(self.path) as archive:
+            for group, prefix in self.groups:
+                if forum is not None and group != forum:
+                    continue
+                for row in self._rows(archive, prefix + "corpus.jsonl"):
+                    text = row["text"]
+                    title = row.get("title", "")
+                    if not isinstance(text, str) or not isinstance(title, str):
+                        raise ArtifactError("BEIR corpus title/text must be strings")
+                    yield Document(
+                        doc_id=self._id(group, row["_id"]),
+                        text=text,
+                        title=title,
+                        corpus_revision=self.lock["sha256"],
+                        metadata={"forum": group},
+                    )
+
+    def fingerprint(self):
+        return {
+            "benchmark_id": self.benchmark_id,
+            "source_archive_sha256": self.lock["sha256"],
+            "source_size_bytes": self.lock["size_bytes"],
+            "upstream_md5": self.lock["upstream_md5"],
+            "source_url": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/"
+            + self.lock["dataset"]
+            + ".zip",
+            "split": "test",
+            "test_queries": len(self._examples),
+            "ignore_identical_ids": True,
+            "corpus_scope": "complete_source",
+            "scorer": "ir_measures:nDCG@10,R@100",
+            "cqadupstack_query_corpus": "same_forum_only",
+            "aggregate_leaderboard_claimed": False,
+            "licenses_bundled": False,
+        }
+
+
+__all__ = ["BEIR_ARCHIVE_LOCKS", "BEIRArchiveAdapter"]