evalkit-py 0.2.0.dev12__py3-none-any.whl

evalkit/core/endpoints.py

@@ -0,0 +1,31 @@
+import asyncio
+import ipaddress
+import socket
+
+import httpx
+
+
+class BlockedEndpoint(ValueError):
+    pass
+
+
+def blocked(addr: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
+    return not addr.is_global or addr.is_multicast or addr.is_reserved
+
+
+async def check_endpoint(url: str, *, allow_private: bool) -> None:
+    if allow_private:
+        return
+    host = httpx.URL(url).host
+    if not host:
+        raise BlockedEndpoint(f"{url} has no host")
+    try:
+        infos = await asyncio.get_running_loop().getaddrinfo(host, None)
+    except socket.gaierror as exc:
+        raise BlockedEndpoint(f"{url}: cannot resolve {host}") from exc
+    bad = sorted({str(info[4][0]) for info in infos if blocked(ipaddress.ip_address(info[4][0]))})
+    if bad:
+        raise BlockedEndpoint(
+            f"{url} resolves to a private address ({', '.join(bad)}); "
+            "the operator can allow it with allow_private_endpoints: true"
+        )

evalkit/core/evaluators/__init__.py

@@ -0,0 +1,29 @@
+from evalkit.core.evaluators import gauges as _gauges
+from evalkit.core.evaluators import presentation as _presentation
+from evalkit.core.evaluators import response_time as _response_time
+from evalkit.core.evaluators import rubrics as _rubrics
+from evalkit.core.evaluators import tool_trajectory as _tool_trajectory
+from evalkit.core.evaluators import trajectory_rubrics as _trajectory_rubrics
+from evalkit.core.evaluators.base import (
+    BUILTIN_EVALUATORS,
+    BUILTIN_GAUGES,
+    EvalContext,
+    execute,
+    gate,
+    neutral_skip,
+)
+
+__all__ = [
+    "BUILTIN_EVALUATORS",
+    "BUILTIN_GAUGES",
+    "EvalContext",
+    "execute",
+    "gate",
+    "neutral_skip",
+    "_gauges",
+    "_presentation",
+    "_response_time",
+    "_rubrics",
+    "_tool_trajectory",
+    "_trajectory_rubrics",
+]

evalkit/core/evaluators/base.py

@@ -0,0 +1,121 @@
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+from evalkit.contracts.harness import Capabilities
+from evalkit.contracts.settings import JudgeSettings
+from evalkit.core.judge import JudgeUsage
+from evalkit.core.trajectory import RunMetrics, TrajectoryView
+from evalkit.store.base import CaseView, ScoreRow
+
+PASS_THRESHOLD = 0.8
+WARN_THRESHOLD = 0.5
+
+
+def label_from_score(score: float) -> str:
+    if score >= PASS_THRESHOLD:
+        return "pass"
+    if score >= WARN_THRESHOLD:
+        return "warn"
+    return "fail"
+
+
+# score=None excludes skipped evaluators from aggregates — returning 1.0 would
+# inflate the overall score.
+def neutral_skip(name: str, reason: str = "capability not provided") -> ScoreRow:
+    return ScoreRow(name=name, score=None, label="skip", explanation=reason)
+
+
+@dataclass
+class EvalContext:
+    case: CaseView
+    answer: str
+    trajectory_view: TrajectoryView | None
+    wall_ms: int
+    judge_cfg: JudgeSettings
+    metrics: RunMetrics | None = None
+    session_id: str | None = None
+    cost_map: dict[str, Any] = field(default_factory=dict)
+    judge_usage: JudgeUsage = field(default_factory=JudgeUsage)
+    output: dict[str, Any] | None = None
+    judge_api_key: str | None = None
+
+    def record_judge_usage(self, usage: JudgeUsage) -> None:
+        self.judge_usage.input_tokens += usage.input_tokens
+        self.judge_usage.output_tokens += usage.output_tokens
+
+
+EvalFn = Callable[[EvalContext, dict[str, Any]], Awaitable[list[ScoreRow]]]
+
+
+@dataclass(frozen=True)
+class BuiltinEval:
+    name: str
+    fn: EvalFn
+    requires: frozenset[str]
+    kind: Literal["evaluator", "gauge"]
+    emits: tuple[str, ...]
+    uses_judge: bool = False
+
+
+BUILTIN_EVALUATORS: dict[str, BuiltinEval] = {}
+BUILTIN_GAUGES: dict[str, BuiltinEval] = {}
+
+
+def builtin_evaluator(
+    name: str,
+    requires: set[str] | None = None,
+    emits: tuple[str, ...] | None = None,
+    uses_judge: bool = False,
+):
+    def decorate(fn: EvalFn) -> EvalFn:
+        BUILTIN_EVALUATORS[name] = BuiltinEval(
+            name=name,
+            fn=fn,
+            requires=frozenset(requires or set()),
+            kind="evaluator",
+            emits=emits or (name,),
+            uses_judge=uses_judge,
+        )
+        return fn
+
+    return decorate
+
+
+def builtin_gauge(name: str, requires: set[str] | None = None):
+    def decorate(fn: EvalFn) -> EvalFn:
+        BUILTIN_GAUGES[name] = BuiltinEval(
+            name=name, fn=fn, requires=frozenset(requires or set()), kind="gauge", emits=(name,)
+        )
+        return fn
+
+    return decorate
+
+
+def gate(evaluator: BuiltinEval, capabilities: Capabilities) -> bool:
+    return all(getattr(capabilities.provides, flag, False) for flag in evaluator.requires)
+
+
+async def execute(
+    evaluator: BuiltinEval,
+    ctx: EvalContext,
+    config: dict[str, Any],
+    capabilities: Capabilities,
+) -> list[ScoreRow]:
+    if not gate(evaluator, capabilities):
+        missing = sorted(
+            flag for flag in evaluator.requires if not getattr(capabilities.provides, flag, False)
+        )
+        row = neutral_skip(evaluator.name, f"harness does not provide: {', '.join(missing)}")
+        if evaluator.kind == "gauge":
+            row.kind = "gauge"
+            row.label = "info"
+        return [row]
+    rows = await evaluator.fn(ctx, config)
+    for row in rows:
+        if evaluator.kind == "gauge":
+            row.kind = "gauge"
+            row.label = row.label or "info"
+        elif row.label is None and row.score is not None:
+            row.label = label_from_score(row.score)
+    return rows

evalkit/core/evaluators/gauges.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+from typing import Any
+
+from evalkit.core.costmap import cost_usd, resolve_rates
+from evalkit.core.evaluators.base import EvalContext, builtin_gauge
+from evalkit.store.base import ScoreRow
+
+
+def _reading(name: str, value: float | None, explanation: str | None = None) -> list[ScoreRow]:
+    if value is None:
+        return [
+            ScoreRow(name=name, kind="gauge", score=None, label="info", explanation="no reading")
+        ]
+    return [ScoreRow(name=name, kind="gauge", score=value, label="info", explanation=explanation)]
+
+
+@builtin_gauge("gauge_duration_seconds")
+async def gauge_duration_seconds(ctx: EvalContext, config: dict[str, Any]) -> list[ScoreRow]:
+    return _reading("gauge_duration_seconds", round(ctx.wall_ms / 1000, 1))
+
+
+@builtin_gauge("gauge_input_tokens", requires={"token_usage"})
+async def gauge_input_tokens(ctx: EvalContext, config: dict[str, Any]) -> list[ScoreRow]:
+    value = ctx.metrics.input_tokens if ctx.metrics else None
+    return _reading("gauge_input_tokens", value)
+
+
+@builtin_gauge("gauge_output_tokens", requires={"token_usage"})
+async def gauge_output_tokens(ctx: EvalContext, config: dict[str, Any]) -> list[ScoreRow]:
+    value = ctx.metrics.output_tokens if ctx.metrics else None
+    return _reading("gauge_output_tokens", value)
+
+
+@builtin_gauge("gauge_cached_tokens", requires={"token_usage"})
+async def gauge_cached_tokens(ctx: EvalContext, config: dict[str, Any]) -> list[ScoreRow]:
+    value = ctx.metrics.cached_tokens if ctx.metrics else None
+    return _reading("gauge_cached_tokens", value)
+
+
+@builtin_gauge("gauge_agent_cost_usd", requires={"token_usage"})
+async def gauge_agent_cost_usd(ctx: EvalContext, config: dict[str, Any]) -> list[ScoreRow]:
+    value = ctx.metrics.cost_usd if ctx.metrics else None
+    return _reading("gauge_agent_cost_usd", round(value, 4) if value is not None else None)
+
+
+@builtin_gauge("gauge_judge_cost_usd")
+async def gauge_judge_cost_usd(ctx: EvalContext, config: dict[str, Any]) -> list[ScoreRow]:
+    usage = ctx.judge_usage
+    if usage.input_tokens == 0 and usage.output_tokens == 0:
+        return _reading("gauge_judge_cost_usd", None)
+    rates = resolve_rates(ctx.cost_map, ctx.judge_cfg.model)
+    if rates is None:
+        return _reading("gauge_judge_cost_usd", None)
+    total = cost_usd(rates, usage.input_tokens, usage.output_tokens, 0)
+    return _reading(
+        "gauge_judge_cost_usd",
+        round(total, 4),
+        f"{usage.input_tokens} in / {usage.output_tokens} out judge tokens",
+    )
+
+
+@builtin_gauge("gauge_total_tool_calls", requires={"trajectory", "tool_calls"})
+async def gauge_total_tool_calls(ctx: EvalContext, config: dict[str, Any]) -> list[ScoreRow]:
+    value = len(ctx.trajectory_view.tool_calls()) if ctx.trajectory_view else None
+    return _reading("gauge_total_tool_calls", value)
+
+
+@builtin_gauge("gauge_errored_tool_calls", requires={"trajectory", "tool_calls"})
+async def gauge_errored_tool_calls(ctx: EvalContext, config: dict[str, Any]) -> list[ScoreRow]:
+    value = ctx.trajectory_view.errored_tool_call_count() if ctx.trajectory_view else None
+    return _reading("gauge_errored_tool_calls", value)
+
+
+@builtin_gauge("gauge_subagents", requires={"trajectory"})
+async def gauge_subagents(ctx: EvalContext, config: dict[str, Any]) -> list[ScoreRow]:
+    value = ctx.trajectory_view.subagent_count() if ctx.trajectory_view else None
+    return _reading("gauge_subagents", value)
+
+
+@builtin_gauge("gauge_turns", requires={"trajectory"})
+async def gauge_turns(ctx: EvalContext, config: dict[str, Any]) -> list[ScoreRow]:
+    value = ctx.trajectory_view.turn_count() if ctx.trajectory_view else None
+    return _reading("gauge_turns", value)

evalkit/core/evaluators/presentation.py

@@ -0,0 +1,166 @@
+"""presentation_judge — rubric-scored judge over how the reply presents itself.
+
+Grades form, not content: whether a unit label is shown, whether a table carries
+the columns asked for, whether the reply relays an artifact instead of restating
+it. Correctness of the figures belongs to answer_rubrics.
+
+Score is points-weighted over binary per-rubric verdicts:
+
+    score = sum(points of met rubrics) / sum(points of all rubrics)
+
+Never fatal. There is no `clean` factor and no fabrication guard, because
+presentation cannot fabricate — a missing unit label invents nothing. A `forbid`
+rubric that is violated costs its points, so an assertion whose violation is a
+product bug rather than a quality shortfall carries that weight through `points`.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from evalkit.contracts.cases import PresentationJudge, PresentationRubric
+from evalkit.core.evaluators.base import EvalContext, builtin_evaluator
+from evalkit.core.evaluators.rubrics import missing_verdict_indices
+from evalkit.core.judge import call_judge
+from evalkit.store.base import ScoreRow
+
+LOGGER = logging.getLogger(__name__)
+
+
+class IncompleteRubricVerdicts(Exception):
+    """The judge did not verdict every rubric exactly once."""
+
+
+class PresentationVerdict(BaseModel):
+    index: int = Field(
+        description="The number this item was listed under in the prompt, counting from 1."
+    )
+    met: bool = Field(
+        description="True when the answer has the required property. For an item"
+        " marked MUST NOT, true means the answer avoided it."
+    )
+    evidence: str = Field(
+        description="The span of the answer that decided this verdict, quoted. When"
+        " the property is absent, say what was looked for and not found."
+    )
+
+
+class PresentationJudgeOutput(BaseModel):
+    verdicts: list[PresentationVerdict]
+
+
+SYSTEM_PROMPT = """\
+You grade HOW an answer presents its information, not whether the information is \
+correct. A figure may be wrong and still presented well; a figure may be right and \
+presented badly. Judge only presentation.
+
+You see the answer text and nothing else — not the tool results, not any generated \
+file. If a property cannot be decided from the text, it is not met.
+
+For each numbered item, return met/unmet and quote the span of the answer that \
+decided it. For an item marked MUST NOT, `met` means the answer avoided it.
+
+Return exactly one verdict per item, for every item."""
+
+
+def _rubric_text(rubric: PresentationRubric) -> str:
+    return f"MUST NOT: {rubric.forbid}" if rubric.forbid is not None else str(rubric.rubric)
+
+
+def _render_rubric(index: int, rubric: PresentationRubric) -> str:
+    line = f"{index}. {_rubric_text(rubric)}"
+    if rubric.note:
+        line += f"\n   note: {rubric.note}"
+    return line
+
+
+def build_user_prompt(answer: str, rubrics: list[PresentationRubric]) -> str:
+    items = "\n".join(_render_rubric(i, rubric) for i, rubric in enumerate(rubrics, start=1))
+    return f"ANSWER\n{answer}\n\nPRESENTATION ITEMS\n{items}"
+
+
+def score_rubrics(rubrics: list[PresentationRubric], verdicts: list[PresentationVerdict]) -> float:
+    """Points-weighted fraction of rubrics met. `rubrics` is non-empty by schema,
+    so the denominator is always positive and the score is always a real number."""
+    by_index = {v.index: v for v in verdicts}
+    total = sum(r.points for r in rubrics)
+    earned = sum(r.points for i, r in enumerate(rubrics, start=1) if by_index[i].met)
+    return earned / total
+
+
+async def _judge_verdicts(
+    ctx: EvalContext, rubrics: list[PresentationRubric], base_prompt: str
+) -> PresentationJudgeOutput:
+    """One judge call, retried once when the verdict list has a gap.
+
+    A gap is a judge lapse, not the agent's failure, so it is retried rather than
+    scored — defaulting the missing rubrics to unmet would blame the agent.
+    """
+    current = base_prompt
+    missing: list[int] = []
+    for attempt in (1, 2):
+        parsed, usage = await call_judge(
+            ctx.judge_cfg,
+            api_key=ctx.judge_api_key,
+            system_prompt=SYSTEM_PROMPT,
+            user_prompt=current,
+            output_type=PresentationJudgeOutput,
+            session_id=ctx.session_id,
+            evaluator_name="presentation_judge",
+        )
+        ctx.record_judge_usage(usage)
+        missing = missing_verdict_indices(len(rubrics), [v.index for v in parsed.verdicts])
+        if not missing:
+            return parsed
+        LOGGER.warning(
+            "evals.presentation.incomplete_verdicts attempt=%s missing=%s", attempt, missing
+        )
+        current = (
+            f"{base_prompt}\n\nYour previous response did not return exactly one "
+            f"verdict for items {missing}. Return one verdict per item, for every "
+            f"item from 1 to {len(rubrics)}."
+        )
+    raise IncompleteRubricVerdicts(f"judge left items {missing} unverdicted after a retry")
+
+
+def _explanation(rubrics: list[PresentationRubric], verdicts: list[PresentationVerdict]) -> str:
+    """Aggregates lead and `items` rides last, so a truncated explanation
+    preview shows the counts. Items carry the rubric text — readers render
+    explanations with no case config to join against — and the judge's quoted
+    evidence as `note`."""
+    by_index = {v.index: v for v in verdicts}
+    items: list[dict[str, Any]] = []
+    met_count = 0
+    for i, r in enumerate(rubrics, start=1):
+        v = by_index[i]
+        met_count += v.met
+        item: dict[str, Any] = {
+            "index": i,
+            "text": _rubric_text(r),
+            "verdict": "satisfied" if v.met else "missing",
+            "points_earned": r.points if v.met else 0,
+            "points_possible": r.points,
+        }
+        if v.evidence:
+            item["note"] = v.evidence
+        items.append(item)
+    return json.dumps({"met": met_count, "total": len(rubrics), "items": items}, ensure_ascii=False)
+
+
+@builtin_evaluator("presentation_judge", uses_judge=True)
+async def presentation_judge(ctx: EvalContext, config: dict[str, Any]) -> list[ScoreRow]:
+    cfg = PresentationJudge.model_validate(config)
+    rubrics = cfg.rubrics
+    parsed = await _judge_verdicts(ctx, rubrics, build_user_prompt(ctx.answer, rubrics))
+    return [
+        ScoreRow(
+            name="presentation_judge",
+            score=score_rubrics(rubrics, parsed.verdicts),
+            explanation=_explanation(rubrics, parsed.verdicts),
+            annotator_kind="LLM",
+        )
+    ]

evalkit/core/evaluators/response_time.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from typing import Any
+
+from evalkit.contracts.cases import MaxResponseTime
+from evalkit.core.evaluators.base import EvalContext, builtin_evaluator
+from evalkit.store.base import ScoreRow
+
+
+@builtin_evaluator("max_response_time")
+async def max_response_time(ctx: EvalContext, config: dict[str, Any]) -> list[ScoreRow]:
+    cfg = MaxResponseTime.model_validate(config)
+    seconds = ctx.wall_ms / 1000
+    passed = seconds <= cfg.max_seconds
+    return [
+        ScoreRow(
+            name="max_response_time",
+            score=1.0 if passed else 0.0,
+            explanation=f"run took {seconds:.1f}s against a cap of {cfg.max_seconds:g}s",
+        )
+    ]