repeatability-evals 0.1.0__tar.gz

repeatability_evals-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,17 @@
+name: CI
+on:
+  push: { branches: [main] }
+  pull_request:
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with: { python-version: "${{ matrix.python-version }}" }
+      - run: pip install -e ".[dev]"
+      - run: ruff check .
+      - run: pytest -q

repeatability_evals-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,16 @@
+name: Publish to PyPI
+on:
+  release:
+    types: [published]
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write   # PyPI trusted publishing; no API token needed
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with: { python-version: "3.12" }
+      - run: pip install build && python -m build
+      - uses: pypa/gh-action-pypi-publish@release/v1

repeatability_evals-0.1.0/.gitignore

@@ -0,0 +1,6 @@
+.venv/
+__pycache__/
+*.egg-info/
+.pytest_cache/
+.ruff_cache/
+dist/

repeatability_evals-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Krishna Gogineni
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

repeatability_evals-0.1.0/PKG-INFO

@@ -0,0 +1,147 @@
+Metadata-Version: 2.5
+Name: repeatability-evals
+Version: 0.1.0
+Summary: Consistency evals for non-deterministic systems: LLM generation, agentic retrieval, agent workflows.
+Project-URL: Homepage, https://github.com/krishna-gogineni-765/repeatability-evals
+Project-URL: Issues, https://github.com/krishna-gogineni-765/repeatability-evals/issues
+Author: Krishna Gogineni
+License: MIT
+License-File: LICENSE
+Keywords: agents,consistency,evaluation,llm,llm-as-judge,non-determinism,rag,reliability,repeatability,retrieval
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == 'dev'
+Requires-Dist: ruff>=0.5; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# repeatability-evals
+
+**Consistency evals for non-deterministic systems: LLM generation, agentic retrieval, agent workflows.**
+
+Composable tools for measuring **how consistent an LLM system is with itself**: the same
+input run several times, or an input and its paraphrases, and how much the outputs agree.
+
+Most eval libraries score one run for quality. A system can average 90% on quality and still
+give a customer a different answer every time they ask. This package measures that spread.
+
+It is deliberately **unaware of what you are comparing**. It has no notion of "answers",
+"retrieval" or "agents": it compares *items*. You choose the items and the similarity:
+
+| You have N of ...                    | Use a similarity such as                         |
+|--------------------------------------|--------------------------------------------------|
+| final answers (free text)            | `LLMJudge`, `EmbeddingCosine`, `RougeL`          |
+| retrieved chunk IDs                  | `JaccardSimilarity`                              |
+| retrieved chunk text                 | `NgramJaccard` (robust to re-chunking)           |
+| ranked result lists                  | `RankBiasedOverlap`                              |
+| agent tool-call traces               | `EditSimilarity` / `ExactMatch` on a trace string|
+| anything else                        | subclass `Similarity`, or pass a plain function  |
+
+Consistency is **not** correctness: a system that is consistently wrong scores 1.0. Pair this
+with an accuracy metric.
+
+## Install
+
+```bash
+pip install repeatability-evals   # pure standard library, no required dependencies
+```
+
+## Quickstart
+
+```python
+from repeatability_evals import (
+    PairwiseConsistency, JaccardSimilarity, NgramJaccard, LLMJudge, evaluate_groups,
+)
+
+# 1. Retrieval: did the 5 runs fetch the same chunks?
+ids = PairwiseConsistency(JaccardSimilarity())
+print(ids.evaluate([["a", "b", "c"], ["a", "b", "d"], ["a", "b", "c"]]).score)
+
+# 2. Same evidence, different chunk boundaries?
+text = PairwiseConsistency(NgramJaccard(n=3))
+print(text.evaluate([["backup jobs run nightly"], ["backup jobs", "run nightly"]]).score)
+
+# 3. Answers: an LLM judge scores semantic equivalence. Bring any `str -> str` function.
+def complete(prompt: str) -> str: ...            # your model call
+answers = PairwiseConsistency(LLMJudge(complete), agreement_threshold=0.75)
+r = answers.evaluate(["Retention is 30 days.", "30 days.", "Retention is 90 days."],
+                     context={"query": "How long are backups retained?"})
+print(r.score, r.details["agreement_rate"], [p.reason for p in r.pairs])
+
+# 4. A whole dataset: one group per question, with a bootstrap confidence interval.
+report = evaluate_groups({"q1": runs_q1, "q2": runs_q2}, ids)
+print(report.score, report.ci, report.worst(3))
+```
+
+## Design
+
+Three small pieces, each replaceable:
+
+* **`Similarity`**: compare two items, return a score in `[0, 1]` (optionally with a raw value
+  and a rationale). Built in: set overlap (`Jaccard`, `Dice`, `OverlapCoefficient`), text
+  (`ExactMatch`, `NgramJaccard`, `RougeL`, `EditSimilarity`), ranking (`RankBiasedOverlap`),
+  embeddings (`EmbeddingCosine`, bring your own embed function), and `RubricJudge` and `LLMJudge`.
+* **`PairSampler`**: which pairs of the N items to compare (see below).
+* **`ConsistencyMetric`**: combines them. `PairwiseConsistency` averages similarity over
+  sampled pairs; `ClusterConsistency` groups items into "same answer" clusters and reports the
+  largest cluster's share plus the entropy of the partition; `TotalAgreement` is the
+  all-N-identical rate.
+
+Everything has an async twin (`aevaluate`, `aevaluate_groups`) with bounded concurrency, which
+matters when the similarity is an LLM call.
+
+### Sampling instead of all pairs
+
+All pairs cost N(N-1)/2 comparisons: 10 for 5 runs, 45 for 10. The default `CycleSampler`
+compares a random cycle through the runs, so N pairs, and every run is compared at least twice.
+It is seeded and reproducible. `AllPairs()` gives the exact value; `RandomPairs(k)` and
+`AnchorPairs()` (every run against one reference) are also available. `benchmarks/sampling_error.py`
+measures the error this introduces on synthetic data so you can see the trade-off for your N.
+
+### The LLM judges
+
+`RubricJudge` is the recommended judge for free-text outputs. One call scores five dimensions
+of agreement (1-5 each): `core_facts`, `specifics` (names, numbers, dates), `supporting_detail`,
+`stance` (answer vs hedge vs refuse) and `format`. The headline score is their weighted mean
+(core facts count most), and `PairwiseConsistency` reports the mean per dimension in
+`details["dimensions"]`, so you can see *what* varies between runs: a system whose facts are
+stable but whose formatting drifts looks very different from one that changes its answer.
+Pass your own `Dimension`s to add, remove or reweight (e.g. for agent traces).
+
+```python
+from repeatability_evals import RubricJudge, PairwiseConsistency
+m = PairwiseConsistency(RubricJudge(complete))
+r = m.evaluate(runs, context={"query": question})
+print(r.score, r.details["dimensions"])   # {'core_facts': 0.95, 'format': 0.4, ...}
+```
+
+`LLMJudge` is the single-score version (one 1-5 semantic-equivalence rating). Both take any completion function, so it works with any provider or a local model. The
+default rubrics (1-5, normalised to `[0, 1]`) are in `repeatability_evals.prompts`. `symmetric=True`
+judges both orderings to cancel position bias at twice the cost. **Validate any judge against
+human ratings on your own data before relying on it**; agreement depends on the domain and the
+judge model.
+
+## Writing your own similarity
+
+```python
+from repeatability_evals import Similarity, PairwiseConsistency
+
+class SameFirstSentence(Similarity):
+    name = "same_first_sentence"
+    def compute(self, a, b, *, context=None):
+        return float(a.split(".")[0] == b.split(".")[0])
+
+PairwiseConsistency(SameFirstSentence())
+```
+
+or just `PairwiseConsistency(lambda a, b: ...)`.
+
+## Status
+
+Early (0.1). The API may change before 1.0.
+
+## License
+
+MIT

repeatability_evals-0.1.0/README.md

@@ -0,0 +1,128 @@
+# repeatability-evals
+
+**Consistency evals for non-deterministic systems: LLM generation, agentic retrieval, agent workflows.**
+
+Composable tools for measuring **how consistent an LLM system is with itself**: the same
+input run several times, or an input and its paraphrases, and how much the outputs agree.
+
+Most eval libraries score one run for quality. A system can average 90% on quality and still
+give a customer a different answer every time they ask. This package measures that spread.
+
+It is deliberately **unaware of what you are comparing**. It has no notion of "answers",
+"retrieval" or "agents": it compares *items*. You choose the items and the similarity:
+
+| You have N of ...                    | Use a similarity such as                         |
+|--------------------------------------|--------------------------------------------------|
+| final answers (free text)            | `LLMJudge`, `EmbeddingCosine`, `RougeL`          |
+| retrieved chunk IDs                  | `JaccardSimilarity`                              |
+| retrieved chunk text                 | `NgramJaccard` (robust to re-chunking)           |
+| ranked result lists                  | `RankBiasedOverlap`                              |
+| agent tool-call traces               | `EditSimilarity` / `ExactMatch` on a trace string|
+| anything else                        | subclass `Similarity`, or pass a plain function  |
+
+Consistency is **not** correctness: a system that is consistently wrong scores 1.0. Pair this
+with an accuracy metric.
+
+## Install
+
+```bash
+pip install repeatability-evals   # pure standard library, no required dependencies
+```
+
+## Quickstart
+
+```python
+from repeatability_evals import (
+    PairwiseConsistency, JaccardSimilarity, NgramJaccard, LLMJudge, evaluate_groups,
+)
+
+# 1. Retrieval: did the 5 runs fetch the same chunks?
+ids = PairwiseConsistency(JaccardSimilarity())
+print(ids.evaluate([["a", "b", "c"], ["a", "b", "d"], ["a", "b", "c"]]).score)
+
+# 2. Same evidence, different chunk boundaries?
+text = PairwiseConsistency(NgramJaccard(n=3))
+print(text.evaluate([["backup jobs run nightly"], ["backup jobs", "run nightly"]]).score)
+
+# 3. Answers: an LLM judge scores semantic equivalence. Bring any `str -> str` function.
+def complete(prompt: str) -> str: ...            # your model call
+answers = PairwiseConsistency(LLMJudge(complete), agreement_threshold=0.75)
+r = answers.evaluate(["Retention is 30 days.", "30 days.", "Retention is 90 days."],
+                     context={"query": "How long are backups retained?"})
+print(r.score, r.details["agreement_rate"], [p.reason for p in r.pairs])
+
+# 4. A whole dataset: one group per question, with a bootstrap confidence interval.
+report = evaluate_groups({"q1": runs_q1, "q2": runs_q2}, ids)
+print(report.score, report.ci, report.worst(3))
+```
+
+## Design
+
+Three small pieces, each replaceable:
+
+* **`Similarity`**: compare two items, return a score in `[0, 1]` (optionally with a raw value
+  and a rationale). Built in: set overlap (`Jaccard`, `Dice`, `OverlapCoefficient`), text
+  (`ExactMatch`, `NgramJaccard`, `RougeL`, `EditSimilarity`), ranking (`RankBiasedOverlap`),
+  embeddings (`EmbeddingCosine`, bring your own embed function), and `RubricJudge` and `LLMJudge`.
+* **`PairSampler`**: which pairs of the N items to compare (see below).
+* **`ConsistencyMetric`**: combines them. `PairwiseConsistency` averages similarity over
+  sampled pairs; `ClusterConsistency` groups items into "same answer" clusters and reports the
+  largest cluster's share plus the entropy of the partition; `TotalAgreement` is the
+  all-N-identical rate.
+
+Everything has an async twin (`aevaluate`, `aevaluate_groups`) with bounded concurrency, which
+matters when the similarity is an LLM call.
+
+### Sampling instead of all pairs
+
+All pairs cost N(N-1)/2 comparisons: 10 for 5 runs, 45 for 10. The default `CycleSampler`
+compares a random cycle through the runs, so N pairs, and every run is compared at least twice.
+It is seeded and reproducible. `AllPairs()` gives the exact value; `RandomPairs(k)` and
+`AnchorPairs()` (every run against one reference) are also available. `benchmarks/sampling_error.py`
+measures the error this introduces on synthetic data so you can see the trade-off for your N.
+
+### The LLM judges
+
+`RubricJudge` is the recommended judge for free-text outputs. One call scores five dimensions
+of agreement (1-5 each): `core_facts`, `specifics` (names, numbers, dates), `supporting_detail`,
+`stance` (answer vs hedge vs refuse) and `format`. The headline score is their weighted mean
+(core facts count most), and `PairwiseConsistency` reports the mean per dimension in
+`details["dimensions"]`, so you can see *what* varies between runs: a system whose facts are
+stable but whose formatting drifts looks very different from one that changes its answer.
+Pass your own `Dimension`s to add, remove or reweight (e.g. for agent traces).
+
+```python
+from repeatability_evals import RubricJudge, PairwiseConsistency
+m = PairwiseConsistency(RubricJudge(complete))
+r = m.evaluate(runs, context={"query": question})
+print(r.score, r.details["dimensions"])   # {'core_facts': 0.95, 'format': 0.4, ...}
+```
+
+`LLMJudge` is the single-score version (one 1-5 semantic-equivalence rating). Both take any completion function, so it works with any provider or a local model. The
+default rubrics (1-5, normalised to `[0, 1]`) are in `repeatability_evals.prompts`. `symmetric=True`
+judges both orderings to cancel position bias at twice the cost. **Validate any judge against
+human ratings on your own data before relying on it**; agreement depends on the domain and the
+judge model.
+
+## Writing your own similarity
+
+```python
+from repeatability_evals import Similarity, PairwiseConsistency
+
+class SameFirstSentence(Similarity):
+    name = "same_first_sentence"
+    def compute(self, a, b, *, context=None):
+        return float(a.split(".")[0] == b.split(".")[0])
+
+PairwiseConsistency(SameFirstSentence())
+```
+
+or just `PairwiseConsistency(lambda a, b: ...)`.
+
+## Status
+
+Early (0.1). The API may change before 1.0.
+
+## License
+
+MIT

repeatability_evals-0.1.0/benchmarks/sampling_error.py

@@ -0,0 +1,56 @@
+"""Estimate how much error pair subsampling adds versus scoring all pairs.
+
+Synthetic by default; pass --runs-file (JSON: {question: [answer, ...]}) to use real runs
+with ExactMatch-style token overlap (NgramJaccard) as a cheap stand-in for a judge.
+"""
+import argparse
+import itertools
+import json
+import random
+import statistics
+
+from repeatability_evals import CycleSampler, NgramJaccard
+
+
+def synthetic(n_q, n_runs, rng):
+    out = []
+    for _ in range(n_q):
+        p = rng.choice([0.0, 0.1, 0.3, 0.6])  # per-question divergence
+        base = [rng.random() for _ in range(n_runs)]
+        out.append([0.0 if rng.random() > p else b for b in base])
+    return out
+
+
+def pair_score_synth(a, b):
+    return 1.0 - min(1.0, abs(a - b) * 3)
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--runs", type=int, default=5)
+    ap.add_argument("--questions", type=int, default=300)
+    ap.add_argument("--runs-file")
+    ap.add_argument("--seed", type=int, default=0)
+    a = ap.parse_args()
+    rng = random.Random(a.seed)
+    if a.runs_file:
+        groups = list(json.load(open(a.runs_file)).values())
+        sim = NgramJaccard(2)
+        score = lambda x, y: sim(x, y)  # noqa: E731
+    else:
+        groups = synthetic(a.questions, a.runs, rng)
+        score = pair_score_synth
+    for budget in (None, 7, 8):
+        errs = []
+        for g in groups:
+            n = len(g)
+            full = statistics.mean(score(g[i], g[j]) for i, j in itertools.combinations(range(n), 2))
+            ps = CycleSampler(budget=budget, seed=rng.randrange(10**6)).sample(n)
+            sub = statistics.mean(score(g[i], g[j]) for i, j in ps)
+            errs.append(sub - full)
+        print(f"budget={budget or 'N'}: mean_abs_err={statistics.mean(map(abs, errs)):.3f} "
+              f"bias={statistics.mean(errs):+.4f} max_abs_err={max(map(abs, errs)):.3f}")
+
+
+if __name__ == "__main__":
+    main()

repeatability_evals-0.1.0/pyproject.toml

@@ -0,0 +1,40 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "repeatability-evals"
+version = "0.1.0"
+description = "Consistency evals for non-deterministic systems: LLM generation, agentic retrieval, agent workflows."
+readme = "README.md"
+license = { text = "MIT" }
+requires-python = ">=3.9"
+authors = [{ name = "Krishna Gogineni" }]
+keywords = ["llm", "evaluation", "consistency", "repeatability", "non-determinism", "rag", "retrieval", "agents", "reliability", "llm-as-judge"]
+classifiers = [
+  "Programming Language :: Python :: 3",
+  "License :: OSI Approved :: MIT License",
+  "Typing :: Typed",
+]
+dependencies = []  # core is pure standard library
+
+[project.optional-dependencies]
+dev = ["pytest>=7", "ruff>=0.5"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/repeatability_evals"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.ruff]
+line-length = 110
+src = ["src", "tests"]
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "B", "UP"]
+ignore = ["UP006", "UP007", "UP035", "UP045"]  # keep typing.Optional/List for py3.9 readability
+
+[project.urls]
+Homepage = "https://github.com/krishna-gogineni-765/repeatability-evals"
+Issues = "https://github.com/krishna-gogineni-765/repeatability-evals/issues"

repeatability_evals-0.1.0/src/repeatability_evals/__init__.py

@@ -0,0 +1,38 @@
+"""repeatability-evals: consistency evals for non-deterministic LLM systems."""
+from .dataset import DatasetResult, aevaluate_groups, evaluate_groups
+from .metrics import ClusterConsistency, ConsistencyMetric, PairwiseConsistency, TotalAgreement
+from .prompts import DEFAULT_DIMENSIONS, SEMANTIC_EQUIVALENCE_PROMPT, Dimension, build_rubric_prompt
+from .sampling import AllPairs, AnchorPairs, CycleSampler, PairSampler, RandomPairs, cycle_pairs
+from .similarity import (
+    DiceSimilarity,
+    EditSimilarity,
+    EmbeddingCosine,
+    ExactMatch,
+    FunctionSimilarity,
+    JaccardSimilarity,
+    JudgeParseError,
+    LLMJudge,
+    NgramJaccard,
+    OverlapCoefficient,
+    RankBiasedOverlap,
+    RougeL,
+    RubricJudge,
+    Similarity,
+    as_similarity,
+)
+from .stats import bootstrap_ci
+from .types import ConsistencyResult, PairScore, SimilarityResult
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "ConsistencyMetric", "PairwiseConsistency", "ClusterConsistency", "TotalAgreement",
+    "evaluate_groups", "aevaluate_groups", "DatasetResult", "bootstrap_ci",
+    "Similarity", "FunctionSimilarity", "as_similarity",
+    "JaccardSimilarity", "DiceSimilarity", "OverlapCoefficient",
+    "ExactMatch", "NgramJaccard", "RougeL", "EditSimilarity",
+    "RankBiasedOverlap", "EmbeddingCosine", "LLMJudge", "RubricJudge", "JudgeParseError",
+    "Dimension", "DEFAULT_DIMENSIONS", "build_rubric_prompt",
+    "PairSampler", "AllPairs", "CycleSampler", "RandomPairs", "AnchorPairs", "cycle_pairs",
+    "ConsistencyResult", "PairScore", "SimilarityResult", "SEMANTIC_EQUIVALENCE_PROMPT",
+]

repeatability_evals-0.1.0/src/repeatability_evals/dataset.py

@@ -0,0 +1,80 @@
+"""Run a metric over many groups (one group = the N runs of one input) and summarise."""
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass, field
+from typing import Any, Dict, Hashable, List, Mapping, Optional, Sequence, Tuple
+
+from .metrics import ConsistencyMetric
+from .stats import bootstrap_ci, mean
+from .types import ConsistencyResult
+
+
+@dataclass
+class DatasetResult:
+    metric: str
+    score: float  # mean of per-group scores
+    n_groups: int
+    ci: Optional[Tuple[float, float]] = None  # bootstrap CI of ``score``
+    per_group: Dict[Hashable, ConsistencyResult] = field(default_factory=dict)
+
+    def worst(self, k: int = 5) -> List[Tuple[Hashable, ConsistencyResult]]:
+        """The ``k`` least consistent groups: where to look first when debugging."""
+        return sorted(self.per_group.items(), key=lambda kv: kv[1].score)[:k]
+
+    def to_records(self) -> List[Dict[str, Any]]:
+        return [
+            {"group": key, "score": r.score, "min_score": r.min_score,
+             "n_items": r.n_items, "n_pairs": r.n_pairs, **{f"d_{k}": v for k, v in r.details.items()
+                                                             if isinstance(v, (int, float, str, bool))}}
+            for key, r in self.per_group.items()
+        ]
+
+
+def _summarise(
+    metric: ConsistencyMetric, results: Dict[Hashable, ConsistencyResult],
+    ci: bool, n_boot: int, seed: int,
+) -> DatasetResult:
+    if not results:
+        raise ValueError("no groups to evaluate")
+    scores = [r.score for r in results.values()]
+    interval = bootstrap_ci(scores, n_boot=n_boot, seed=seed) if ci else None
+    return DatasetResult(metric.name, mean(scores), len(results), interval, results)
+
+
+def evaluate_groups(
+    groups: Mapping[Hashable, Sequence[Any]],
+    metric: ConsistencyMetric,
+    *,
+    contexts: Optional[Mapping[Hashable, Mapping[str, Any]]] = None,
+    ci: bool = True,
+    n_boot: int = 2000,
+    seed: int = 0,
+) -> DatasetResult:
+    """``groups`` maps a group key (e.g. a question ID) to its N items; ``contexts`` optionally
+    maps the same keys to per-group context such as ``{"query": "..."}`` for an LLM judge."""
+    results = {
+        key: metric.evaluate(items, context=(contexts or {}).get(key))
+        for key, items in groups.items()
+    }
+    return _summarise(metric, results, ci, n_boot, seed)
+
+
+async def aevaluate_groups(
+    groups: Mapping[Hashable, Sequence[Any]],
+    metric: ConsistencyMetric,
+    *,
+    contexts: Optional[Mapping[Hashable, Mapping[str, Any]]] = None,
+    concurrency: int = 4,
+    ci: bool = True,
+    n_boot: int = 2000,
+    seed: int = 0,
+) -> DatasetResult:
+    sem = asyncio.Semaphore(concurrency)
+
+    async def one(key: Hashable, items: Sequence[Any]) -> Tuple[Hashable, ConsistencyResult]:
+        async with sem:
+            return key, await metric.aevaluate(items, context=(contexts or {}).get(key))
+
+    pairs = await asyncio.gather(*(one(k, v) for k, v in groups.items()))
+    return _summarise(metric, dict(pairs), ci, n_boot, seed)