trialmark 0.1.0__py3-none-any.whl

trialmark/__init__.py

@@ -0,0 +1,28 @@
+"""Trialmark: model acceptance tests with a pass mark set before the run."""
+
+__version__ = "0.1.0"
+
+from .passmark import Decision, PassMark, decide, parse_pass_mark  # noqa: E402
+from .record import ModelEntry, Result, RunRecord, load_record, parse_record  # noqa: E402
+from .report import decision, summarise, text_report  # noqa: E402
+from .testset import Case, TestSet, hash_cases, load_test_set, parse_test_set  # noqa: E402
+
+__all__ = [
+    "Case",
+    "Decision",
+    "ModelEntry",
+    "PassMark",
+    "Result",
+    "RunRecord",
+    "TestSet",
+    "decide",
+    "decision",
+    "hash_cases",
+    "load_record",
+    "load_test_set",
+    "parse_pass_mark",
+    "parse_record",
+    "parse_test_set",
+    "summarise",
+    "text_report",
+]

trialmark/cli.py

@@ -0,0 +1,85 @@
+"""The trialmark command."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+from . import __version__
+from .record import load_record
+from .report import decision, text_report
+from .testset import load_test_set
+
+
+def _hash(args: argparse.Namespace) -> int:
+    ts = load_test_set(args.test_set)
+    print(ts.hash)
+    return 0
+
+
+def _validate(args: argparse.Namespace) -> int:
+    ts = load_test_set(args.test_set)
+    tags = ts.tags()
+    print(f"{ts.name} v{ts.version}: {len(ts.cases)} cases, tags: {', '.join(tags) if tags else 'none'}")
+    print(ts.hash)
+    return 0
+
+
+def _report(args: argparse.Namespace) -> int:
+    print(text_report(load_record(args.record), by_tag=args.by_tag))
+    return 0
+
+
+def _gate(args: argparse.Namespace) -> int:
+    record = load_record(args.record)
+    if args.test_set:
+        expected = load_test_set(args.test_set).hash
+        if expected != record.test_set["hash"]:
+            print(f"FAIL: the run used test set {record.test_set['hash']}, not {expected}.")
+            return 1
+    d = decision(record)
+    v = d.verdict_for(args.model)
+    if v is None:
+        print(f"FAIL: {args.model} has no results in this run.")
+        return 1
+    if v.passed:
+        print(f"PASS: {args.model} meets the pass mark.")
+        return 0
+    print(f"FAIL: {args.model}: {'; '.join(v.reasons)}")
+    return 1
+
+
+def main(argv: list[str] | None = None) -> int:
+    p = argparse.ArgumentParser(prog="trialmark", description="Model acceptance tests with a pass mark set before the run.")
+    p.add_argument("--version", action="version", version=f"trialmark {__version__}")
+    sub = p.add_subparsers(dest="command", required=True)
+
+    s = sub.add_parser("hash", help="print a test set's hash")
+    s.add_argument("test_set")
+    s.set_defaults(fn=_hash)
+
+    s = sub.add_parser("validate", help="check a test set and list its tags")
+    s.add_argument("test_set")
+    s.set_defaults(fn=_validate)
+
+    s = sub.add_parser("report", help="summarise a run record against its pass mark")
+    s.add_argument("record")
+    s.add_argument("--by-tag", action="store_true", help="also break results down by tag")
+    s.set_defaults(fn=_report)
+
+    s = sub.add_parser("gate", help="exit 0 if a model meets the run's pass mark, 1 if not")
+    s.add_argument("record")
+    s.add_argument("--model", required=True)
+    s.add_argument("--test-set", help="also check that the run used this exact test set")
+    s.set_defaults(fn=_gate)
+
+    args = p.parse_args(argv)
+    try:
+        return args.fn(args)
+    except (OSError, ValueError) as e:
+        print(f"trialmark: {e}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())

trialmark/passmark.py

@@ -0,0 +1,117 @@
+"""Pass marks: the decision rule, written down before the run.
+
+A pass mark says which model to choose once the results are in, so the choice
+is not made after seeing them:
+
+    {
+      "metric": "correct",
+      "tolerance_points": 3,
+      "floors": {"grounded": 0.95},
+      "choose": "cheapest"
+    }
+
+Read as: among the models whose mean "grounded" is at least 0.95, every model
+within 3 points of the best mean "correct" passes, and the cheapest of those is
+chosen. Points are percentage points of a score between 0 and 1.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass, field
+from typing import Any
+
+from .testset import canonical_json
+
+CHOOSE_RULES = ("cheapest", "fastest", "best")
+
+
+@dataclass(frozen=True)
+class PassMark:
+    metric: str
+    tolerance_points: float = 0.0
+    floors: dict[str, float] = field(default_factory=dict)
+    choose: str = "cheapest"
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "metric": self.metric,
+            # Floats throughout, so 3 and 3.0 hash the same after a round trip.
+            "tolerance_points": float(self.tolerance_points),
+            "floors": {k: float(v) for k, v in self.floors.items()},
+            "choose": self.choose,
+        }
+
+    @property
+    def hash(self) -> str:
+        return "sha256:" + hashlib.sha256(canonical_json(self.to_dict()).encode("utf-8")).hexdigest()
+
+
+def parse_pass_mark(data: dict[str, Any]) -> PassMark:
+    if "metric" not in data:
+        raise ValueError("Pass mark is missing 'metric'.")
+    choose = data.get("choose", "cheapest")
+    if choose not in CHOOSE_RULES:
+        raise ValueError(f"Pass mark 'choose' must be one of {', '.join(CHOOSE_RULES)}.")
+    tolerance = float(data.get("tolerance_points", 0))
+    if tolerance < 0:
+        raise ValueError("Pass mark 'tolerance_points' cannot be negative.")
+    floors = {str(k): float(v) for k, v in dict(data.get("floors", {})).items()}
+    return PassMark(metric=str(data["metric"]), tolerance_points=tolerance, floors=floors, choose=choose)
+
+
+@dataclass(frozen=True)
+class ModelVerdict:
+    model: str
+    passed: bool
+    reasons: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class Decision:
+    chosen: str | None
+    best_score: float | None
+    verdicts: tuple[ModelVerdict, ...]
+
+    def verdict_for(self, model: str) -> ModelVerdict | None:
+        return next((v for v in self.verdicts if v.model == model), None)
+
+
+def decide(pass_mark: PassMark, summaries: list[Any]) -> Decision:
+    """Apply a pass mark to model summaries (see report.summarise)."""
+    scored = [s for s in summaries if pass_mark.metric in s.metrics]
+    if not scored:
+        return Decision(chosen=None, best_score=None, verdicts=tuple(
+            ModelVerdict(s.model, False, (f"no '{pass_mark.metric}' results",)) for s in summaries
+        ))
+    best = max(s.metrics[pass_mark.metric].mean for s in scored)
+    threshold = best - pass_mark.tolerance_points / 100
+    verdicts: list[ModelVerdict] = []
+    passing = []
+    for s in summaries:
+        reasons: list[str] = []
+        m = s.metrics.get(pass_mark.metric)
+        if m is None:
+            reasons.append(f"no '{pass_mark.metric}' results")
+        elif m.mean < threshold - 1e-9:
+            reasons.append(
+                f"{pass_mark.metric} {m.mean:.1%} is more than {pass_mark.tolerance_points:g} points below the best {best:.1%}"
+            )
+        for name, floor in pass_mark.floors.items():
+            fm = s.metrics.get(name)
+            if fm is None:
+                reasons.append(f"no '{name}' results")
+            elif fm.mean < floor - 1e-9:
+                reasons.append(f"{name} {fm.mean:.1%} is below the floor of {floor:.1%}")
+        verdicts.append(ModelVerdict(s.model, not reasons, tuple(reasons)))
+        if not reasons:
+            passing.append(s)
+    chosen = None
+    if passing:
+        if pass_mark.choose == "cheapest":
+            chosen = min(passing, key=lambda s: (s.cost_per_100_usd, s.model)).model
+        elif pass_mark.choose == "fastest":
+            chosen = min(passing, key=lambda s: (s.latency_p50_ms, s.model)).model
+        else:
+            chosen = max(passing, key=lambda s: (s.metrics[pass_mark.metric].mean, -s.cost_per_100_usd)).model
+    return Decision(chosen=chosen, best_score=best, verdicts=tuple(verdicts))

trialmark/record.py

@@ -0,0 +1,131 @@
+"""Run records: everything needed to repeat a run and check its result.
+
+A record is started before any model runs, with the test set and the pass mark
+fixed, and results are added as they arrive. The pass mark's hash and the time
+it was set are kept, so a reader can see it was not changed after the results.
+"""
+
+from __future__ import annotations
+
+import json
+import uuid
+from dataclasses import asdict, dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from . import __version__
+from .passmark import PassMark, parse_pass_mark
+from .testset import TestSet
+
+
+@dataclass
+class ModelEntry:
+    id: str
+    provider: str = ""
+    settings: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class Result:
+    model: str
+    repeat: int
+    case_id: str
+    metrics: dict[str, float]
+    tags: list[str] = field(default_factory=list)
+    output: Any = None
+    cost_usd: float = 0.0
+    latency_ms: int = 0
+    input_tokens: int = 0
+    output_tokens: int = 0
+    error: str | None = None
+    note: str = ""
+
+
+@dataclass
+class RunRecord:
+    test_set: dict[str, str]
+    pass_mark: dict[str, Any]
+    pass_mark_hash: str
+    pass_mark_set_at: str
+    models: list[ModelEntry]
+    repeats: int
+    run_id: str = field(default_factory=lambda: str(uuid.uuid4()))
+    trialmark_version: str = __version__
+    judge: dict[str, Any] | None = None
+    environment: dict[str, Any] = field(default_factory=dict)
+    results: list[Result] = field(default_factory=list)
+
+    @classmethod
+    def start(
+        cls,
+        test_set: TestSet,
+        pass_mark: PassMark,
+        models: list[ModelEntry],
+        repeats: int = 3,
+        judge: dict[str, Any] | None = None,
+        environment: dict[str, Any] | None = None,
+    ) -> "RunRecord":
+        if repeats < 1:
+            raise ValueError("A run needs at least one repeat.")
+        if not models:
+            raise ValueError("A run needs at least one model.")
+        return cls(
+            test_set={"name": test_set.name, "version": test_set.version, "hash": test_set.hash},
+            pass_mark=pass_mark.to_dict(),
+            pass_mark_hash=pass_mark.hash,
+            pass_mark_set_at=datetime.now(timezone.utc).isoformat(timespec="seconds"),
+            models=models,
+            repeats=repeats,
+            judge=judge,
+            environment=environment or {},
+        )
+
+    def add(self, result: Result) -> None:
+        if result.model not in {m.id for m in self.models}:
+            raise ValueError(f"Model '{result.model}' is not in this run.")
+        if not 0 <= result.repeat < self.repeats:
+            raise ValueError(f"Repeat {result.repeat} is outside 0..{self.repeats - 1}.")
+        self.results.append(result)
+
+    def parsed_pass_mark(self) -> PassMark:
+        pm = parse_pass_mark(self.pass_mark)
+        if pm.hash != self.pass_mark_hash:
+            raise ValueError("The pass mark does not match the hash recorded when it was set.")
+        return pm
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+    def save(self, path: str | Path) -> None:
+        with open(path, "w", encoding="utf-8") as f:
+            json.dump(self.to_dict(), f, indent=2, ensure_ascii=False)
+            f.write("\n")
+
+
+def parse_record(data: dict[str, Any]) -> RunRecord:
+    try:
+        models = [ModelEntry(**m) for m in data["models"]]
+        results = [Result(**r) for r in data.get("results", [])]
+        return RunRecord(
+            test_set=dict(data["test_set"]),
+            pass_mark=dict(data["pass_mark"]),
+            pass_mark_hash=str(data["pass_mark_hash"]),
+            pass_mark_set_at=str(data["pass_mark_set_at"]),
+            models=models,
+            repeats=int(data["repeats"]),
+            run_id=str(data.get("run_id", "")),
+            trialmark_version=str(data.get("trialmark_version", "")),
+            judge=data.get("judge"),
+            environment=dict(data.get("environment", {})),
+            results=results,
+        )
+    except KeyError as e:
+        raise ValueError(f"Run record is missing {e}.") from None
+    except TypeError as e:
+        raise ValueError(f"Run record has an unexpected field: {e}.") from None
+
+
+def load_record(path: str | Path) -> RunRecord:
+    with open(path, encoding="utf-8") as f:
+        return parse_record(json.load(f))

trialmark/report.py

@@ -0,0 +1,123 @@
+"""Summaries and reports: each model's scores, cost and speed, with the range
+across repeats, overall and for each tag."""
+
+from __future__ import annotations
+
+import statistics
+from dataclasses import dataclass
+
+from .passmark import Decision, decide
+from .record import Result, RunRecord
+
+
+@dataclass(frozen=True)
+class MetricSummary:
+    mean: float
+    low: float
+    high: float
+
+
+@dataclass(frozen=True)
+class ModelSummary:
+    model: str
+    cases: int
+    repeats: int
+    errors: int
+    metrics: dict[str, MetricSummary]
+    cost_per_100_usd: float
+    latency_p50_ms: int
+
+
+def _summarise(model: str, results: list[Result]) -> ModelSummary:
+    by_repeat: dict[int, list[Result]] = {}
+    for r in results:
+        by_repeat.setdefault(r.repeat, []).append(r)
+    names = sorted({k for r in results for k in r.metrics})
+    metrics: dict[str, MetricSummary] = {}
+    for name in names:
+        # Mean within each repeat, then the mean and range across repeats.
+        per_repeat = []
+        for rs in by_repeat.values():
+            vals = [r.metrics[name] for r in rs if name in r.metrics]
+            if vals:
+                per_repeat.append(sum(vals) / len(vals))
+        if per_repeat:
+            metrics[name] = MetricSummary(
+                mean=sum(per_repeat) / len(per_repeat), low=min(per_repeat), high=max(per_repeat)
+            )
+    costs = [sum(r.cost_usd for r in rs) / len(rs) * 100 for rs in by_repeat.values() if rs]
+    latencies = [r.latency_ms for r in results if r.latency_ms > 0 and r.error is None]
+    return ModelSummary(
+        model=model,
+        cases=len({r.case_id for r in results}),
+        repeats=len(by_repeat),
+        errors=sum(1 for r in results if r.error),
+        metrics=metrics,
+        cost_per_100_usd=sum(costs) / len(costs) if costs else 0.0,
+        latency_p50_ms=int(statistics.median(latencies)) if latencies else 0,
+    )
+
+
+def summarise(record: RunRecord, tag: str | None = None) -> list[ModelSummary]:
+    out = []
+    for m in record.models:
+        rs = [r for r in record.results if r.model == m.id and (tag is None or tag in r.tags)]
+        if rs:
+            out.append(_summarise(m.id, rs))
+    return out
+
+
+def decision(record: RunRecord) -> Decision:
+    return decide(record.parsed_pass_mark(), summarise(record))
+
+
+def _pct(m: MetricSummary | None) -> str:
+    if m is None:
+        return "-"
+    if abs(m.high - m.low) < 1e-9:
+        return f"{m.mean:.0%}"
+    return f"{m.mean:.0%} ({m.low:.0%}-{m.high:.0%})"
+
+
+def _table(summaries: list[ModelSummary], metrics: list[str]) -> list[str]:
+    head = ["Model", *metrics, "Cost/100", "Median time"]
+    rows = [
+        [
+            s.model,
+            *(_pct(s.metrics.get(n)) for n in metrics),
+            f"${s.cost_per_100_usd:.2f}",
+            f"{s.latency_p50_ms / 1000:.1f}s",
+        ]
+        for s in summaries
+    ]
+    widths = [max(len(str(x)) for x in col) for col in zip(head, *rows)]
+    fmt = lambda r: "  ".join(str(x).ljust(w) for x, w in zip(r, widths)).rstrip()  # noqa: E731
+    return [fmt(head), fmt(["-" * w for w in widths]), *map(fmt, rows)]
+
+
+def text_report(record: RunRecord, by_tag: bool = False) -> str:
+    summaries = summarise(record)
+    metrics = sorted({n for s in summaries for n in s.metrics})
+    pm = record.parsed_pass_mark()
+    d = decide(pm, summaries)
+    lines = [
+        f"Test set   {record.test_set['name']} v{record.test_set['version']} ({record.test_set['hash'][:19]}...)",
+        f"Run        {record.run_id}, {record.repeats} repeats, Trialmark {record.trialmark_version}",
+    ]
+    if record.judge:
+        lines.append(f"Judge      {record.judge.get('model', '?')}")
+    lines += ["", *_table(summaries, metrics), ""]
+    lines.append(
+        "Ranges in brackets are the lowest and highest repeat. Treat differences inside them as no difference."
+    )
+    lines += ["", f"Pass mark  set {record.pass_mark_set_at}: {pm.metric} within {pm.tolerance_points:g} points of the best"
+              + "".join(f", {k} at least {v:.0%}" for k, v in pm.floors.items())
+              + f"; choose the {pm.choose}."]
+    for v in d.verdicts:
+        lines.append(f"  {'PASS' if v.passed else 'FAIL'}  {v.model}" + (f": {'; '.join(v.reasons)}" if v.reasons else ""))
+    lines.append(f"Chosen     {d.chosen or 'no model met the pass mark'}")
+    if by_tag:
+        tags = sorted({t for r in record.results for t in r.tags})
+        for t in tags:
+            lines += ["", f"Tag: {t}", *_table(summarise(record, tag=t), metrics)]
+    return "\n".join(lines)

trialmark/testset.py

@@ -0,0 +1,97 @@
+"""Test sets: the cases a model is tested on, each with a reference output and tags.
+
+A test set is a JSON file:
+
+    {
+      "name": "contracts",
+      "version": "2026-11-05",
+      "cases": [
+        {"id": "q1", "input": "What is the notice period?",
+         "reference": "Sixty days' written notice.", "tags": ["factual"]}
+      ]
+    }
+
+The hash covers the cases only, so a rerun can prove it used exactly the same
+cases even if the set was renamed.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+@dataclass(frozen=True)
+class Case:
+    id: str
+    input: Any
+    reference: Any
+    tags: tuple[str, ...] = ()
+
+
+@dataclass(frozen=True)
+class TestSet:
+    name: str
+    version: str
+    cases: tuple[Case, ...]
+    description: str = ""
+    extra: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def hash(self) -> str:
+        return hash_cases(self.cases)
+
+    def tags(self) -> list[str]:
+        return sorted({t for c in self.cases for t in c.tags})
+
+
+def canonical_json(value: Any) -> str:
+    return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+
+
+def hash_cases(cases: tuple[Case, ...] | list[Case]) -> str:
+    rows = sorted(
+        ({"id": c.id, "input": c.input, "reference": c.reference, "tags": sorted(c.tags)} for c in cases),
+        key=lambda r: r["id"],
+    )
+    return "sha256:" + hashlib.sha256(canonical_json(rows).encode("utf-8")).hexdigest()
+
+
+def parse_test_set(data: dict[str, Any]) -> TestSet:
+    for key in ("name", "version", "cases"):
+        if key not in data:
+            raise ValueError(f"Test set is missing '{key}'.")
+    if not isinstance(data["cases"], list) or not data["cases"]:
+        raise ValueError("Test set 'cases' must be a non-empty list.")
+    cases: list[Case] = []
+    seen: set[str] = set()
+    for i, raw in enumerate(data["cases"]):
+        if not isinstance(raw, dict):
+            raise ValueError(f"Case {i} must be an object.")
+        for key in ("id", "input", "reference"):
+            if key not in raw:
+                raise ValueError(f"Case {i} is missing '{key}'.")
+        cid = str(raw["id"])
+        if cid in seen:
+            raise ValueError(f"Case id '{cid}' appears more than once.")
+        seen.add(cid)
+        tags = raw.get("tags", [])
+        if not isinstance(tags, list) or not all(isinstance(t, str) for t in tags):
+            raise ValueError(f"Case '{cid}' tags must be a list of strings.")
+        cases.append(Case(id=cid, input=raw["input"], reference=raw["reference"], tags=tuple(tags)))
+    extra = {k: v for k, v in data.items() if k not in ("name", "version", "cases", "description")}
+    return TestSet(
+        name=str(data["name"]),
+        version=str(data["version"]),
+        cases=tuple(cases),
+        description=str(data.get("description", "")),
+        extra=extra,
+    )
+
+
+def load_test_set(path: str | Path) -> TestSet:
+    with open(path, encoding="utf-8") as f:
+        return parse_test_set(json.load(f))

trialmark-0.1.0.dist-info/METADATA

@@ -0,0 +1,152 @@
+Metadata-Version: 2.5
+Name: trialmark
+Version: 0.1.0
+Summary: Model acceptance tests for language models: your own test set, a pass mark set before the run, and a record anyone can rerun.
+Project-URL: Homepage, https://github.com/peshwar9/trialmark
+Project-URL: Issues, https://github.com/peshwar9/trialmark/issues
+Author: PE (peshwar9)
+License-Expression: Apache-2.0
+License-File: LICENSE
+Keywords: acceptance testing,benchmark,evaluation,llm,model selection
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# Trialmark
+
+**Model acceptance tests for language models.** Test models on your own work, decide in advance
+what result is good enough, and keep a record that anyone can rerun and check.
+
+Choosing a language model is a trade between accuracy, cost and speed. Trialmark makes that
+choice systematic:
+
+1. **Your own test set.** Cases from your own work, each with a reference answer written by a
+   person and tagged by kind, such as "factual" or "multi-part". The set is hashed, so a later
+   run can prove it used exactly the same cases.
+2. **A pass mark set before the run.** For example: "choose the cheapest model within 3 points
+   of the best on correctness, as long as at least 95% of its answers are grounded". The rule
+   is stored with its hash and the time it was set, so it cannot be quietly changed after the
+   results are in.
+3. **Repeats, with the range shown.** Models do not give the same answer every time. Each model
+   runs several times, and the report shows the lowest and highest result so that a difference
+   inside the range is read as no difference.
+4. **A report by kind of case.** A small model may be as good as a large one on single facts
+   and worse on multi-part questions. The breakdown by tag shows where.
+5. **A run record and a release gate.** Every run writes one JSON file with the test set's hash,
+   the models and their settings, the judge, the pass mark and every result. `trialmark gate`
+   exits with 0 or 1, so a pipeline can refuse a model change that fails the pass mark.
+
+Trialmark runs on your own machines. Your test sets, reference answers and records stay with you.
+
+## Where it comes from
+
+Trialmark was spun out of the model benchmark in [Forge](https://www.forgeprivate.com), a
+governed layer through which organisations delegate work to AI. That benchmark compares models
+on a fixed set of business-document questions, through Forge's own answer path, with three
+repeats and accuracy, cost and speed side by side. Trialmark makes the same kind of test
+available to any developer or team, with or without Forge.
+
+## Status
+
+Trialmark is at an early stage. Version 0.1 provides the formats and the decision:
+
+| Built in 0.1 | Planned |
+|---|---|
+| The test set format, with hashing and tags | Runners that call models directly (Anthropic, OpenAI and others) |
+| The pass mark and the decision rule | A reference-answer judge, which grades an answer as correct, partly correct or incorrect and checks that it is grounded |
+| The run record, with the pass mark's hash | A field-by-field scorer for extraction and classification |
+| The report, with ranges and the breakdown by tag | A runner that tests a model inside Forge, with Forge's retrieval and rules applied |
+| `trialmark gate` for pipelines | Logging runs to MLflow |
+
+In 0.1 you produce the results yourself, with your own code, and Trialmark records, reports and
+decides on them.
+
+## Install
+
+```bash
+pip install trialmark
+```
+
+Python 3.10 or later. The core has no dependencies.
+
+## Try it
+
+The `examples` folder holds a five-question test set and an illustrative run of three made-up
+models. The scores in it are invented to show the format; they are not measurements of any real
+model.
+
+```bash
+trialmark validate examples/testset.json
+trialmark report examples/run-record.json --by-tag
+trialmark gate examples/run-record.json --model medium-model --test-set examples/testset.json
+```
+
+## Use it from Python
+
+```python
+from trialmark import ModelEntry, PassMark, Result, RunRecord, load_test_set, text_report
+
+test_set = load_test_set("my-testset.json")
+
+# Fix the decision before any model runs.
+pass_mark = PassMark(metric="correct", tolerance_points=3, floors={"grounded": 0.95}, choose="cheapest")
+record = RunRecord.start(
+    test_set,
+    pass_mark,
+    [ModelEntry(id="model-a", provider="provider-x"), ModelEntry(id="model-b", provider="provider-y")],
+    repeats=3,
+    judge={"model": "judge-model", "prompt_version": "1"},
+)
+
+for repeat in range(3):
+    for model in ("model-a", "model-b"):
+        for case in test_set.cases:
+            answer, cost, ms = call_your_model(model, case.input)   # your code
+            score = grade(answer, case.reference)                     # your code: 0, 0.5 or 1
+            record.add(Result(
+                model=model, repeat=repeat, case_id=case.id, tags=list(case.tags),
+                metrics={"correct": score, "grounded": 1.0},
+                output=answer, cost_usd=cost, latency_ms=ms,
+            ))
+
+record.save("run-record.json")
+print(text_report(record, by_tag=True))
+```
+
+## Formats
+
+**Test set**
+
+```json
+{
+  "name": "contract-questions",
+  "version": "2026-11-05",
+  "cases": [
+    { "id": "q1", "input": "What is the notice period?", "reference": "Sixty days' written notice.", "tags": ["factual"] }
+  ]
+}
+```
+
+The hash covers the cases (their ids, inputs, references and tags), not the name or version.
+
+**Pass mark**
+
+| Field | Meaning |
+|---|---|
+| `metric` | The score the decision rests on, such as `correct` |
+| `tolerance_points` | How many percentage points below the best model still passes |
+| `floors` | Minimum mean scores on other metrics, such as `{"grounded": 0.95}` |
+| `choose` | Which passing model to choose: `cheapest`, `fastest` or `best` |
+
+**Metrics** are whatever your scorer produces, each between 0 and 1. A model's score on a
+metric is the mean within each repeat; the report shows the mean of those and their range.
+
+## Licence
+
+Apache-2.0. See [LICENSE](LICENSE).

trialmark-0.1.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+trialmark/__init__.py,sha256=XwXHsiw1Ib4gAxswMDKo0jHbyVVWRJY-CkTPILBislA,743
+trialmark/cli.py,sha256=779mvyqJYrkKYT1hENGWTtIHNthgCo06ET4IZkdIlT8,2700
+trialmark/passmark.py,sha256=UQ3EqNbQpHbZ0ocfptY4TWCsGzC3NghPH7H792CA9_0,4344
+trialmark/record.py,sha256=ZWXcjhfjEHca8luSiyohl0XyKAmCdetY-wpf-4pmfck,4351
+trialmark/report.py,sha256=3ipooxxdIrWGpy8kfC2RIwBvsRWlOwWGMRgWjFTqkbk,4548
+trialmark/testset.py,sha256=-LQMl-tSblQiJ7X4dr2oFAGp2NLZf6DtHy3DIwy5GNI,3083
+trialmark-0.1.0.dist-info/METADATA,sha256=wG1attMcWnl02HGSEQ2fqNkMeCGbPXKA8OzgERXf7cc,6332
+trialmark-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+trialmark-0.1.0.dist-info/entry_points.txt,sha256=V5xoW75gguNdKsK49ManBGrXWh8F2-JdPDCDriwH3jU,49
+trialmark-0.1.0.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+trialmark-0.1.0.dist-info/RECORD,,

trialmark-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

trialmark-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+trialmark = trialmark.cli:main

trialmark-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.