agentic-evals 0.2.0__tar.gz → 0.3.0__tar.gz

{agentic_evals-0.2.0 → agentic_evals-0.3.0}/CHANGELOG.md

@@ -4,6 +4,32 @@ All notable changes to this project will be documented here.
 
 This project follows [Semantic Versioning](https://semver.org/).
 
+## 0.3.0 - 2026-09-13
+
+### Added
+
+- `agentic_evals.skills`: methodology playbooks (`SKILL.md` cards --
+  Trigger/Do/Avoid/Check/Risk, matching the format Braintrust's
+  `eval-library/skills` uses), scoped to this package's own API:
+  `write-a-scorer`, `define-a-release-gate`, `size-a-test-suite`,
+  `instrument-a-trace`. These are guidance documents, not runnable code --
+  consumable by a human or by a coding agent's own skill mechanism.
+- `agentic_evals.packs`: runnable eval-config bundles -- a `TestSuite`
+  paired with the scorer names it needs, loaded from a single YAML/JSON
+  file. `load_pack()`/`load_builtin_pack()` validate that every
+  `required_scorers` entry resolves in the registry before returning.
+  Ships 3 built-in packs: `tool-use-correctness`, `factual-qa`,
+  `json-output-contract`.
+
+### Fixed
+
+- `LLMRubricEvaluator` now reads its reference material (the expert
+  answer for `FACTUALITY`, candidate B for `BATTLE`) from
+  `EvaluatorConfig.config["reference"]` instead of `case.expected_output`.
+  Reusing `expected_output` collided with `evaluate_suite`'s hardcoded
+  exact-match check, which fires whenever that field is set -- silently
+  forcing every rubric-graded case to also require a literal string match.
+
 ## 0.2.0 - 2026-09-13
 
 ### Added

{agentic_evals-0.2.0 → agentic_evals-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: agentic-evals
-Version: 0.2.0
+Version: 0.3.0
 Summary: A standalone, framework-agnostic evaluation and scoring engine for LLM/agent outputs — extracted from AgenticLens's evaluation module.
 Project-URL: Homepage, https://github.com/DeepAgentLabs/agentic-evals
 Project-URL: Repository, https://github.com/DeepAgentLabs/agentic-evals
@@ -275,6 +275,44 @@ Python targets execute local code and HTTP targets can reach arbitrary
 URLs. Only point them at trusted suite files and trusted target
 definitions.
 
+## Eval packs
+
+A pack bundles a `TestSuite` with the scorer names it needs into one
+shareable YAML/JSON file:
+
+```python
+from agentic_evals import (
+    EvalSpan,
+    EvalTrace,
+    EvaluationSample,
+    default_registry,
+    evaluate_suite,
+    load_builtin_pack,
+)
+
+pack = load_builtin_pack("tool-use-correctness")  # validates required_scorers up front
+sample = EvaluationSample(
+    case_id="refund-status-lookup",
+    output="Your refund is on its way.",
+    trace=EvalTrace(spans=[EvalSpan(tool_name="lookup_refund")]),
+)
+report = evaluate_suite(pack.to_suite(), [sample], registry=default_registry())
+```
+
+Ships 3 built-in packs (`list_builtin_packs()`): `tool-use-correctness`,
+`json-output-contract` (both runnable with `default_registry()`), and
+`factual-qa` (needs a registry with an `LLMRubricEvaluator` registered,
+since it uses the `FACTUALITY` rubric). Load your own with `load_pack(path)`.
+
+## Skills
+
+`agentic_evals/skills/` ships methodology playbooks (`SKILL.md` cards --
+Trigger/Do/Avoid/Check/Risk), not runnable code: `write-a-scorer`,
+`define-a-release-gate`, `size-a-test-suite`, `instrument-a-trace`. These
+document how to use this package well and are meant to be read directly,
+or picked up by a coding agent's own skill mechanism -- distinct from
+"packs" above, which are runnable configuration.
+
 ## Using it with AgenticLens's own traces
 
 If you already have an AgenticLens `Run` (from its instrumentation API or

{agentic_evals-0.2.0 → agentic_evals-0.3.0}/README.md

@@ -219,6 +219,44 @@ Python targets execute local code and HTTP targets can reach arbitrary
 URLs. Only point them at trusted suite files and trusted target
 definitions.
 
+## Eval packs
+
+A pack bundles a `TestSuite` with the scorer names it needs into one
+shareable YAML/JSON file:
+
+```python
+from agentic_evals import (
+    EvalSpan,
+    EvalTrace,
+    EvaluationSample,
+    default_registry,
+    evaluate_suite,
+    load_builtin_pack,
+)
+
+pack = load_builtin_pack("tool-use-correctness")  # validates required_scorers up front
+sample = EvaluationSample(
+    case_id="refund-status-lookup",
+    output="Your refund is on its way.",
+    trace=EvalTrace(spans=[EvalSpan(tool_name="lookup_refund")]),
+)
+report = evaluate_suite(pack.to_suite(), [sample], registry=default_registry())
+```
+
+Ships 3 built-in packs (`list_builtin_packs()`): `tool-use-correctness`,
+`json-output-contract` (both runnable with `default_registry()`), and
+`factual-qa` (needs a registry with an `LLMRubricEvaluator` registered,
+since it uses the `FACTUALITY` rubric). Load your own with `load_pack(path)`.
+
+## Skills
+
+`agentic_evals/skills/` ships methodology playbooks (`SKILL.md` cards --
+Trigger/Do/Avoid/Check/Risk), not runnable code: `write-a-scorer`,
+`define-a-release-gate`, `size-a-test-suite`, `instrument-a-trace`. These
+document how to use this package well and are meant to be read directly,
+or picked up by a coding agent's own skill mechanism -- distinct from
+"packs" above, which are runnable configuration.
+
 ## Using it with AgenticLens's own traces
 
 If you already have an AgenticLens `Run` (from its instrumentation API or

{agentic_evals-0.2.0 → agentic_evals-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "agentic-evals"
-version = "0.2.0"
+version = "0.3.0"
 description = "A standalone, framework-agnostic evaluation and scoring engine for LLM/agent outputs — extracted from AgenticLens's evaluation module."
 readme = "README.md"
 license = { file = "LICENSE" }

{agentic_evals-0.2.0 → agentic_evals-0.3.0}/src/agentic_evals/__init__.py

@@ -29,6 +29,7 @@ from agentic_evals.models import (
     TestCase,
     TestSuite,
 )
+from agentic_evals.packs import EvalPack, list_builtin_packs, load_builtin_pack, load_pack
 from agentic_evals.runner import (
     evaluate_suite,
     load_samples,
@@ -59,7 +60,7 @@ from agentic_evals.scorers import (
     valid_json,
 )
 
-__version__ = "0.2.0"
+__version__ = "0.3.0"
 
 __all__ = [
     "BATTLE",
@@ -70,6 +71,7 @@ __all__ = [
     "BusinessRuleEvaluator",
     "CallableEvaluator",
     "CaseEvaluation",
+    "EvalPack",
     "EvalSpan",
     "EvalTrace",
     "EvaluationContext",
@@ -101,6 +103,9 @@ __all__ = [
     "exact_match",
     "json_diff",
     "levenshtein_similarity",
+    "list_builtin_packs",
+    "load_builtin_pack",
+    "load_pack",
     "load_samples",
     "load_suite",
     "no_redundant_tool_calls",

agentic_evals-0.3.0/src/agentic_evals/packs/__init__.py

@@ -0,0 +1,15 @@
+"""Runnable eval packs -- a `TestSuite` bundled with the scorer names it needs.
+
+Distinct from `agentic_evals.skills`, which are methodology playbooks, not
+runnable configuration.
+"""
+
+from agentic_evals.packs.library import list_builtin_packs, load_builtin_pack
+from agentic_evals.packs.loader import EvalPack, load_pack
+
+__all__ = [
+    "EvalPack",
+    "list_builtin_packs",
+    "load_builtin_pack",
+    "load_pack",
+]

agentic_evals-0.3.0/src/agentic_evals/packs/builtin/factual-qa.yaml

@@ -0,0 +1,27 @@
+name: factual-qa
+description: >
+  Grades a submitted answer against a reference answer for factual
+  consistency using the FACTUALITY rubric. Requires the caller to
+  register LLMRubricEvaluator("factuality", FACTUALITY, complete_fn=...)
+  since grading needs a real model call.
+required_scorers:
+  - factuality
+suite:
+  name: factual-qa
+  version: "1"
+  description: >
+    Compares a submitted answer against a reference answer for factual
+    consistency.
+  cases:
+    - id: capital-of-france
+      name: Answer states the correct capital
+      input: What is the capital of France?
+      # No expected_output here on purpose: evaluate_suite runs a
+      # hardcoded exact-match check whenever it's set, which would fight
+      # a rubric meant to allow near-matches. The reference answer for
+      # the rubric lives in the evaluator's own config instead.
+      evaluators:
+        - name: factuality
+          threshold: 0.8
+          config:
+            reference: Paris is the capital of France.

agentic_evals-0.3.0/src/agentic_evals/packs/builtin/json-output-contract.yaml

@@ -0,0 +1,23 @@
+name: json-output-contract
+description: >
+  Checks that an agent's output is valid JSON and matches an expected
+  structure -- catches both malformed JSON and structurally wrong
+  responses.
+required_scorers:
+  - valid_json
+  - json_diff
+suite:
+  name: json-output-contract
+  version: "1"
+  description: >
+    The agent must return a JSON object with the exact structure the
+    caller expects.
+  cases:
+    - id: structured-answer
+      name: Output is valid JSON matching the expected structure
+      expected_output: '{"answer": "42", "confidence": 0.9}'
+      evaluators:
+        - name: valid_json
+          threshold: 1.0
+        - name: json_diff
+          threshold: 1.0

agentic_evals-0.3.0/src/agentic_evals/packs/builtin/tool-use-correctness.yaml

@@ -0,0 +1,27 @@
+name: tool-use-correctness
+description: >
+  Checks that an agent calls exactly the tools a refund-lookup task
+  needs -- the required lookup tool, and never a destructive delete tool.
+required_scorers:
+  - tool_call_precision
+suite:
+  name: tool-use-correctness
+  version: "1"
+  description: >
+    An agent answering a refund-status question must call lookup_refund
+    and never delete_order.
+  cases:
+    - id: refund-status-lookup
+      name: Agent looks up the refund without deleting anything
+      input:
+        order_id: A123
+      required_tools:
+        - lookup_refund
+      forbidden_tools:
+        - delete_order
+      evaluators:
+        - name: tool_call_precision
+          threshold: 1.0
+          config:
+            allowed_tools:
+              - lookup_refund

agentic_evals-0.3.0/src/agentic_evals/packs/library.py

@@ -0,0 +1,17 @@
+from pathlib import Path
+
+from agentic_evals.evaluators import EvaluatorRegistry
+from agentic_evals.packs.loader import EvalPack, load_pack
+
+_BUILTIN_DIR = Path(__file__).parent / "builtin"
+
+
+def list_builtin_packs() -> tuple[str, ...]:
+    return tuple(sorted(path.stem for path in _BUILTIN_DIR.glob("*.yaml")))
+
+
+def load_builtin_pack(name: str, *, registry: EvaluatorRegistry | None = None) -> EvalPack:
+    path = _BUILTIN_DIR / f"{name}.yaml"
+    if not path.is_file():
+        raise ValueError(f"unknown builtin eval pack {name!r}; available: {list_builtin_packs()}")
+    return load_pack(path, registry=registry)

agentic_evals-0.3.0/src/agentic_evals/packs/loader.py

@@ -0,0 +1,48 @@
+"""Runnable eval packs: a `TestSuite` bundled with the scorer names it needs.
+
+Distinct from `agentic_evals.skills` (methodology playbooks) -- a pack is a
+small, shareable, immediately-runnable configuration.
+"""
+
+import json
+from pathlib import Path
+from typing import Any
+
+import yaml
+from pydantic import BaseModel, Field
+
+from agentic_evals.evaluators import EvaluatorRegistry
+from agentic_evals.models import TestSuite
+from agentic_evals.scorers.registry import default_registry
+
+
+class EvalPack(BaseModel):
+    name: str
+    description: str = ""
+    suite: TestSuite
+    required_scorers: list[str] = Field(default_factory=list)
+
+    def to_suite(self) -> TestSuite:
+        return self.suite
+
+
+def _load_data(path: Path) -> Any:
+    text = path.read_text(encoding="utf-8")
+    return json.loads(text) if path.suffix.lower() == ".json" else yaml.safe_load(text)
+
+
+def load_pack(path: Path, *, registry: EvaluatorRegistry | None = None) -> EvalPack:
+    """Load an `EvalPack` from a YAML/JSON file, failing fast if a required scorer is missing.
+
+    `registry` defaults to `default_registry()`; pass your own (e.g. one
+    with an `LLMRubricEvaluator` registered) if the pack needs a scorer
+    that isn't in the default set.
+    """
+    pack = EvalPack.model_validate(_load_data(path))
+    available = (registry or default_registry()).names()
+    missing = [name for name in pack.required_scorers if name not in available]
+    if missing:
+        raise ValueError(
+            f"eval pack {pack.name!r} requires scorer(s) not available in the registry: {missing}"
+        )
+    return pack

{agentic_evals-0.2.0 → agentic_evals-0.3.0}/src/agentic_evals/scorers/rubric.py

@@ -139,7 +139,14 @@ MODERATION = RubricTemplate(
 
 
 class LLMRubricEvaluator(CallableEvaluator):
-    """Grade a sample against a `RubricTemplate` using a caller-supplied model call."""
+    """Grade a sample against a `RubricTemplate` using a caller-supplied model call.
+
+    The rubric's reference material (the expert answer for `FACTUALITY`,
+    candidate B for `BATTLE`) comes from `EvaluatorConfig.config["reference"]`
+    -- deliberately *not* `case.expected_output`, since `evaluate_suite`
+    runs a hardcoded exact-match check whenever `expected_output` is set,
+    which would fight a rubric that's meant to allow near-matches.
+    """
 
     def __init__(
         self, name: str, template: RubricTemplate, complete_fn: Callable[[str], str]
@@ -147,7 +154,7 @@ class LLMRubricEvaluator(CallableEvaluator):
         def _judge(context: EvaluationContext) -> Score:
             prompt = template.render(
                 output=context.sample.output,
-                expected=context.case.expected_output,
+                expected=context.config.config.get("reference"),
                 input=context.case.input,
                 extra=context.config.config.get("extra", {}),
             )

agentic_evals-0.3.0/src/agentic_evals/skills/define-a-release-gate/SKILL.md

@@ -0,0 +1,59 @@
+---
+name: define-a-release-gate
+description: Choose GateConfig thresholds and understand the cost/latency edge cases before gating a release on an EvaluationReport.
+---
+
+# Define a release gate
+
+## Trigger
+
+Use this when you're wiring `evaluate_gate(report, GateConfig(...))` into a
+CI check or release decision and need to pick real threshold values, not
+just accept the defaults.
+
+## Do
+
+1. Start from the strictest sensible defaults (`min_pass_rate=1.0`,
+   `min_average_score=1.0`, `max_failed_cases=0`) and loosen deliberately —
+   it's easier to justify relaxing a gate later than to explain why it was
+   never strict.
+2. Set `max_average_latency_ms`/`max_total_cost_usd` only when you have a
+   real budget in mind; leaving them `None` skips that check entirely
+   rather than silently passing with a made-up number.
+3. Read `GateDecision.reasons` and `.observed` in CI output, not just
+   `.passed` — `reasons` names exactly which threshold failed, which is
+   what someone debugging a red build actually needs.
+4. Treat a gate failure on cost/latency as at least as actionable as a
+   pass-rate failure — regressions there are often the first sign of a
+   prompt or retry-loop change before quality visibly degrades.
+
+## Avoid
+
+- Don't set `max_total_cost_usd` on a suite where cost is routinely
+  unavailable for some cases (e.g. a live target that doesn't report
+  per-span cost) — the gate will fail every run with "cost is unavailable
+  or incomplete," which looks like a real regression but isn't.
+- Don't loosen `min_pass_rate` to make a flaky suite pass — fix the
+  flaky case or evaluator instead; a gate that's routinely overridden
+  stops being trusted.
+- Don't gate on a suite that mixes exploratory and release-blocking cases
+  in one `TestSuite` — split them, since one flaky exploratory case
+  shouldn't block a release.
+
+## Check
+
+- `evaluate_gate`'s cost check treats an incomplete cost picture
+  (`summary.total_cost_usd is None`, or any case missing `cost_usd`) as
+  "unavailable," never as `$0.00` — confirm your traces actually populate
+  `estimated_cost_usd` before relying on this gate for cost control.
+- The gate config you shipped to CI matches the one you tested locally —
+  `GateConfig` is a plain Pydantic model, so diff it explicitly rather than
+  trusting two configs "look the same."
+
+## Risk
+
+A gate that only checks `pass_rate` can pass while cost or latency drifts
+upward every release, since neither is checked unless you explicitly set
+`max_average_latency_ms`/`max_total_cost_usd`. If cost or latency matters
+for the product, set both thresholds, don't just default to the pass-rate
+check.

agentic_evals-0.3.0/src/agentic_evals/skills/instrument-a-trace/SKILL.md

@@ -0,0 +1,67 @@
+---
+name: instrument-a-trace
+description: Build EvalTrace/EvalSpan correctly from your own agent's data, including how to normalize legacy or differently-shaped trace payloads with a trace_adapter.
+---
+
+# Instrument a trace
+
+## Trigger
+
+Use this when you're producing the `trace` field of an `EvaluationSample`
+for the first time, or when a `TestCase`'s latency/cost/tool checks are
+silently passing or failing in a way that doesn't match reality.
+
+## Do
+
+1. Populate `EvalTrace.total_latency_ms` and `.estimated_cost_usd`
+   directly as plain numbers — this is not a computed property here (it
+   is on some richer trace schemas). If you're deriving it from
+   timestamps or per-call costs, do that arithmetic yourself before
+   constructing the `EvalTrace`.
+2. Give every tool-calling `EvalSpan` a `tool_name` — `required_tools`,
+   `forbidden_tools`, `tool_call_precision`, and `tool_call_recall` all
+   filter on `span.tool_name is not None`; a span with `tool_name=None`
+   is invisible to every tool-based check.
+3. Put arguments a `required_tool_arguments` check needs under
+   `span.attributes["tool_args"]` — that's the exact key the built-in
+   check reads.
+4. If your traces come from a system with a different native shape (e.g.
+   timestamp-derived latency, per-span rather than trace-level cost), use
+   the `trace_adapter` parameter on `load_samples()`/`run_live_suite()`:
+   write one small function that maps your raw trace dict into
+   `EvalTrace`'s shape, and pass it once at the call site rather than
+   converting ad hoc everywhere you build a sample.
+5. Set `estimated_cost_usd=None` (the default) when cost genuinely isn't
+   known for a sample — every cost check in this package treats `None` as
+   "unavailable," never as `$0.00`. Never pass `0.0` to mean "unknown."
+
+## Avoid
+
+- Don't leave `total_latency_ms` at its default `0.0` for a real sample —
+  every `max_latency_ms` check will trivially pass, which looks like a
+  release gate working when it's actually blind.
+- Don't put tool arguments somewhere other than
+  `attributes["tool_args"]` and expect `required_tool_arguments` to find
+  them — it looks at that exact key, not the whole `attributes` dict.
+- Don't silently drop a trace field your source system provides just
+  because `EvalSpan`/`EvalTrace` doesn't ask for it — pydantic ignores
+  unknown fields by default, so a payload shaped for a different schema
+  validates "successfully" while quietly losing the data you needed.
+
+## Check
+
+- Build one `EvalTrace` by hand from a real sample and print it — confirm
+  `total_latency_ms`, `estimated_cost_usd`, and every span's `tool_name`
+  match what you know actually happened, not just that validation passed.
+- If you wrote a `trace_adapter`, unit test it directly against a raw
+  payload sample, independent of `evaluate_suite` — a normalization bug
+  there fails silently (defaults to 0/None) rather than raising.
+
+## Risk
+
+A trace payload that predates `EvalTrace`'s shape (timestamps instead of
+`total_latency_ms`, per-span cost instead of trace-level) validates fine
+and produces a *plausible-looking* report with latency/cost silently
+defaulted to 0/None — this doesn't error, so it can pass code review and
+still gate a release on wrong numbers. Always adapt at the boundary
+(a `trace_adapter`), not by hoping downstream fields line up.

agentic_evals-0.3.0/src/agentic_evals/skills/size-a-test-suite/SKILL.md

@@ -0,0 +1,59 @@
+---
+name: size-a-test-suite
+description: Decide how many TestCases a TestSuite needs and how to split criteria across them without bundling unrelated checks into one case.
+---
+
+# Size a test suite
+
+## Trigger
+
+Use this when you're building a new `TestSuite` and unsure whether to add
+more `TestCase`s, add more `evaluators` to an existing case, or split one
+case into several.
+
+## Do
+
+1. One `TestCase` per input/scenario, not per criterion — if the same
+   input needs three checks (contains a fact, calls the right tool, stays
+   under a latency budget), that's one `TestCase` with three entries in
+   `evaluators`/threshold fields, not three cases.
+2. Cover the boundary the criterion actually claims to check: a
+   `max_turns` case needs both a passing sample at the limit and a failing
+   one just over it, not just one happy-path sample.
+3. Include at least one adversarial or edge-case input per suite (empty
+   input, a refusal, a malformed tool response) — `TestCase.require
+   expectation`'s validator forces every case to declare *something* to
+   check, so use that to your advantage: even an edge case needs a real
+   expectation, not a placeholder.
+4. Give every `TestCase.id` a stable, descriptive slug (`refund-large-amount`,
+   not `case-7`) — `TestSuite` rejects duplicate IDs, and stable IDs are
+   what `EvaluationSample.case_id` and dataset splits key off of.
+
+## Avoid
+
+- Don't cram every acceptance criterion for a feature into one
+  mega-`TestCase` with a dozen `evaluators` entries — when it fails, a
+  report should point at one clear thing, not require reading six scores
+  to find the one that mattered.
+- Don't size a suite by "how many cases feels thorough" — size it by
+  "which distinct behaviors would a regression actually break," and stop
+  once you've covered them once each plus their edges.
+- Don't let a suite silently grow stale: a case with an `expected_output`
+  that references a value your system no longer produces will fail forever
+  and gets ignored, which is worse than not having the case.
+
+## Check
+
+- Every `TestCase` has at least one expectation set (`TestCase`'s own
+  validator enforces this) — if you're stuck picking one, that's a signal
+  the case doesn't yet test anything specific.
+- Running the suite against a known-good sample set produces a 100% pass
+  rate; running it against a known-bad set fails on the case(s) you'd
+  expect, not a different one.
+
+## Risk
+
+A suite that's easy to make pass by construction (loose thresholds, only
+happy-path cases) gives false confidence in a release gate — it's the
+gate's job to fail sometimes, so a suite that never produces a genuine
+failure during development hasn't been tested itself.

agentic_evals-0.3.0/src/agentic_evals/skills/write-a-scorer/SKILL.md

@@ -0,0 +1,64 @@
+---
+name: write-a-scorer
+description: Choose and implement the right kind of scorer (deterministic, embedding, LLM rubric, or trajectory) for a new eval criterion in agentic_evals.
+---
+
+# Write a scorer
+
+## Trigger
+
+Use this when you're adding a new grading criterion to a `TestCase` and
+aren't sure whether to reach for a built-in scorer, write a custom
+`CallableEvaluator`, or reach for an LLM judge.
+
+## Do
+
+1. Name one criterion per scorer. If you're tempted to score "correctness
+   and tone" in a single evaluator, that's two evaluators.
+2. Prefer `agentic_evals.scorers.text` first: `exact_match`,
+   `contains_all`/`contains_any`, `valid_json`, `json_diff`, `numeric_diff`
+   are deterministic, free, and never flaky — use them whenever the
+   criterion has an objectively checkable answer.
+3. Reach for `embedding_similarity` only when near-miss wording should
+   still pass and you already have an embedding call available; it needs
+   an `embed_fn` in `EvaluatorConfig.config` since this package never
+   calls an embedding API itself.
+4. Reach for `LLMRubricEvaluator` + a `RubricTemplate` only when the
+   criterion is genuinely subjective (factual consistency, summary
+   quality, pairwise preference). Supply your own `complete_fn` —
+   `agentic_evals` never calls a model.
+5. Reach for `agentic_evals.scorers.trajectory` (`tool_call_precision`,
+   `tool_call_recall`, `no_redundant_tool_calls`, `trajectory_efficiency`)
+   when the criterion is about *how* the agent got the answer, not just
+   the final text.
+6. If none of the above fit, write a plain `CallableEvaluator(name, fn)` —
+   `fn` takes an `EvaluationContext` and returns a `Score` or `list[Score]`.
+
+## Avoid
+
+- Don't wrap an LLM judge around something a deterministic check already
+  covers (`valid_json`, `json_diff`) — it's slower, costs money, and can
+  disagree with itself between runs.
+- Don't average unrelated criteria into a single `Score.value` — register
+  them as separate evaluators so a report shows exactly what failed.
+- Don't hardcode a specific model or embedding provider inside a scorer
+  function — take `complete_fn`/`embed_fn` as a parameter, the way every
+  built-in rubric/embedding scorer does.
+
+## Check
+
+- The scorer raises a clear `ValueError` when its required input
+  (`expected_output`, `case.input`, a trace with tool calls) is missing,
+  rather than silently returning a meaningless score.
+- `Score.value` stays in `[0, 1]`; a hard pass/fail scorer returns exactly
+  `0.0` or `1.0`, a graded one returns a genuine gradient.
+- You've tested the scorer directly against at least one passing and one
+  failing sample (call it with a hand-built `EvaluationContext`), not only
+  through a full suite run.
+
+## Risk
+
+An LLM-graded rubric is only as good as its verdict parsing —
+`RubricTemplate.parse_verdict` raises if the model's completion doesn't
+contain one of the expected tokens. Don't swallow that exception upstream:
+a silently defaulted score is worse than a visible failure.

agentic_evals-0.3.0/tests/test_packs.py

@@ -0,0 +1,71 @@
+import pytest
+
+from agentic_evals import (
+    FACTUALITY,
+    EvalSpan,
+    EvalTrace,
+    EvaluationSample,
+    EvaluatorRegistry,
+    LLMRubricEvaluator,
+    default_registry,
+    evaluate_suite,
+    list_builtin_packs,
+    load_builtin_pack,
+)
+
+
+def test_list_builtin_packs_includes_the_shipped_packs() -> None:
+    names = list_builtin_packs()
+    assert names == tuple(sorted(names))
+    assert set(names) >= {"tool-use-correctness", "factual-qa", "json-output-contract"}
+
+
+def test_load_builtin_pack_rejects_unknown_name() -> None:
+    with pytest.raises(ValueError, match="unknown builtin eval pack"):
+        load_builtin_pack("does-not-exist")
+
+
+def test_tool_use_correctness_pack_runs_end_to_end() -> None:
+    pack = load_builtin_pack("tool-use-correctness")
+    sample = EvaluationSample(
+        case_id="refund-status-lookup",
+        output="Your refund is on its way.",
+        trace=EvalTrace(spans=[EvalSpan(tool_name="lookup_refund")]),
+    )
+
+    report = evaluate_suite(pack.to_suite(), [sample], registry=default_registry())
+
+    assert report.summary.pass_rate == 1.0
+
+
+def test_json_output_contract_pack_runs_end_to_end() -> None:
+    pack = load_builtin_pack("json-output-contract")
+    sample = EvaluationSample(
+        case_id="structured-answer",
+        output='{"answer": "42", "confidence": 0.9}',
+        trace=EvalTrace(),
+    )
+
+    report = evaluate_suite(pack.to_suite(), [sample], registry=default_registry())
+
+    assert report.summary.pass_rate == 1.0
+
+
+def test_factual_qa_pack_requires_a_registry_with_factuality_registered() -> None:
+    with pytest.raises(ValueError, match="factuality"):
+        load_builtin_pack("factual-qa")  # default_registry() has no LLM-judge scorers
+
+    registry = EvaluatorRegistry()
+    registry.register(
+        LLMRubricEvaluator("factuality", FACTUALITY, complete_fn=lambda prompt: "(A)")
+    )
+    pack = load_builtin_pack("factual-qa", registry=registry)
+
+    sample = EvaluationSample(
+        case_id="capital-of-france",
+        output="Paris",
+        trace=EvalTrace(),
+    )
+    report = evaluate_suite(pack.to_suite(), [sample], registry=registry)
+
+    assert report.summary.pass_rate == 1.0

{agentic_evals-0.2.0 → agentic_evals-0.3.0}/tests/test_scorers_rubric.py

@@ -14,13 +14,17 @@ from agentic_evals import (
 
 
 def _context(
-    *, output: str, expected_output: str | None, input_: object = None
+    *, output: str, reference: str | None = None, input_: object = None, threshold: float = 0.8
 ) -> EvaluationContext:
-    case = TestCase(id="case-1", name="case", expected_output=expected_output, input=input_)
+    # Filler expectation unrelated to the rubric under test: TestCase
+    # requires at least one, and expected_output is deliberately not used
+    # here (see LLMRubricEvaluator's docstring on why).
+    case = TestCase(id="case-1", name="case", input=input_, max_turns=1_000_000)
     sample = EvaluationSample(case_id="case-1", output=output, trace=EvalTrace())
-    return EvaluationContext(
-        case=case, sample=sample, config=EvaluatorConfig(name="judge", threshold=0.8)
+    config = EvaluatorConfig(
+        name="judge", threshold=threshold, config={"reference": reference} if reference else {}
     )
+    return EvaluationContext(case=case, sample=sample, config=config)
 
 
 def test_rubric_template_renders_all_fields() -> None:
@@ -67,7 +71,7 @@ def test_battle_template_distinguishes_win_and_tie_tokens() -> None:
 
 def test_llm_rubric_evaluator_scores_using_complete_fn() -> None:
     evaluator = LLMRubricEvaluator("factuality", FACTUALITY, complete_fn=lambda prompt: "(A)")
-    context = _context(output="Paris", expected_output="Paris is the capital of France")
+    context = _context(output="Paris", reference="Paris is the capital of France")
 
     scores = evaluator.evaluate(context)
 
@@ -81,7 +85,7 @@ def test_llm_rubric_evaluator_scores_using_complete_fn() -> None:
 
 def test_llm_rubric_evaluator_applies_threshold_from_config() -> None:
     evaluator = LLMRubricEvaluator("factuality", FACTUALITY, complete_fn=lambda prompt: "(C)")
-    context = _context(output="Paris", expected_output="Paris is the capital of France")
+    context = _context(output="Paris", reference="Paris is the capital of France")
 
     score = evaluator.evaluate(context)[0]
 
@@ -91,7 +95,7 @@ def test_llm_rubric_evaluator_applies_threshold_from_config() -> None:
 
 def test_llm_rubric_evaluator_propagates_unparseable_completion() -> None:
     evaluator = LLMRubricEvaluator("factuality", FACTUALITY, complete_fn=lambda prompt: "unclear")
-    context = _context(output="Paris", expected_output="Paris is the capital of France")
+    context = _context(output="Paris", reference="Paris is the capital of France")
 
     with pytest.raises(ValueError, match="could not parse a verdict"):
         evaluator.evaluate(context)

agentic_evals-0.3.0/tests/test_skills.py

@@ -0,0 +1,40 @@
+"""Structure checks for the methodology skills shipped under `agentic_evals/skills/`.
+
+These are documentation/guidance cards (Trigger/Do/Avoid/Check/Risk), not
+runnable code -- this test verifies the format is intact, not the content.
+"""
+
+from pathlib import Path
+
+import yaml
+
+SKILLS_DIR = Path(__file__).parent.parent / "src" / "agentic_evals" / "skills"
+REQUIRED_SECTIONS = ["## Trigger", "## Do", "## Avoid", "## Check", "## Risk"]
+
+
+def _skill_dirs() -> list[Path]:
+    return sorted(p for p in SKILLS_DIR.iterdir() if p.is_dir())
+
+
+def test_at_least_one_skill_is_shipped() -> None:
+    assert _skill_dirs(), "expected at least one skill directory under agentic_evals/skills/"
+
+
+def test_every_skill_has_a_well_formed_skill_md() -> None:
+    for skill_dir in _skill_dirs():
+        skill_md = skill_dir / "SKILL.md"
+        assert skill_md.is_file(), f"{skill_dir} is missing SKILL.md"
+
+        text = skill_md.read_text(encoding="utf-8")
+        assert text.startswith("---\n"), f"{skill_md} must start with YAML frontmatter"
+
+        _, frontmatter_raw, body = text.split("---\n", 2)
+        frontmatter = yaml.safe_load(frontmatter_raw)
+
+        assert frontmatter.get("name") == skill_dir.name, (
+            f"{skill_md}'s frontmatter name must match its directory name"
+        )
+        assert frontmatter.get("description"), f"{skill_md} must set a non-empty description"
+
+        for section in REQUIRED_SECTIONS:
+            assert section in body, f"{skill_md} is missing the {section!r} section"

{agentic_evals-0.2.0 → agentic_evals-0.3.0}/uv.lock

@@ -9,7 +9,7 @@ resolution-markers = [
 
 [[package]]
 name = "agentic-evals"
-version = "0.2.0"
+version = "0.3.0"
 source = { editable = "." }
 dependencies = [
     { name = "jsonschema" },