aehf 0.1.0__py3-none-any.whl

aehf/__init__.py

@@ -0,0 +1,6 @@
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("aehf")
+except PackageNotFoundError:  # not installed (e.g. running from a raw checkout)
+    __version__ = "0.0.0"

aehf/adapters/anthropic.py

@@ -0,0 +1,92 @@
+import time
+
+from anthropic import AsyncAnthropic
+from anthropic.types import MessageParam, ToolParam, ToolResultBlockParam
+
+from aehf.core.case import EvalCase
+from aehf.core.transcript import Step, Termination, ToolCall, Transcript
+from aehf.tools.base import ToolProviderFactory
+
+
+class AnthropicAdapter:
+
+    def __init__(self, client: AsyncAnthropic, provider_factory: ToolProviderFactory, model: str, max_tokens: int) -> None:
+        self.client = client
+        self.provider_factory = provider_factory
+        self.model = model
+        self.max_tokens = max_tokens
+
+    async def run(self, case: EvalCase) -> Transcript:
+        toolprovider = self.provider_factory(case)
+        tool_spec: list[ToolParam] = [
+            ToolParam(name=t.name, description=t.description or "", input_schema=t.parameters)
+            for t in case.tools
+        ]
+        messages: list[MessageParam] = [{"role": "user", "content": case.task_prompt}]
+        steps: list[Step] = []
+        total_tokens = 0
+        final_text = ""
+        start_time = time.monotonic()
+
+        while True:
+            # enforce budgets before spending, not after
+            if len(steps) >= case.max_steps:
+                termination_reason = Termination.max_steps
+                break
+            if total_tokens >= case.token_budget:
+                termination_reason = Termination.budget_exceeded
+                break
+
+            response = await self.client.messages.create(
+                model=self.model,
+                max_tokens=self.max_tokens,
+                tools=tool_spec,
+                messages=messages,
+            )
+            messages.append({"role": "assistant", "content": response.content})
+            # input tokens count every turn: each call re-reads the whole conversation
+            step_tokens = response.usage.input_tokens + response.usage.output_tokens
+            total_tokens += step_tokens
+
+            step_text = ""
+            toolcalls: list[ToolCall] = []
+            tool_results: list[ToolResultBlockParam] = []
+            for block in response.content:
+                if block.type == "text":
+                    step_text = block.text
+                elif block.type == "tool_use":
+                    tool_start_time = time.monotonic()
+                    result = await toolprovider.execute(block.name, block.input)
+                    tool_results.append({"type": "tool_result", "tool_use_id": block.id, "content": result})
+                    toolcalls.append(ToolCall(
+                        toolname=block.name,
+                        arguments=block.input,
+                        result=result,
+                        latency=time.monotonic() - tool_start_time,
+                    ))
+
+            steps.append(Step(model_output=step_text, tool_calls=toolcalls, token_usage=step_tokens))
+            if step_text:
+                final_text = step_text
+            if tool_results:
+                messages.append({"role": "user", "content": tool_results})
+
+            if response.stop_reason == "end_turn":
+                termination_reason = Termination.finished
+                break
+            if response.stop_reason == "refusal":
+                termination_reason = Termination.refused
+                break
+            if response.stop_reason != "tool_use":
+                # max_tokens, pause_turn, or anything the API adds later: stop, don't spin
+                termination_reason = Termination.unexpected_stop
+                break
+
+        return Transcript(
+            id=case.id,
+            ordered_steps=steps,
+            final_answer=final_text,
+            total_tokens=total_tokens,
+            duration_seconds=time.monotonic() - start_time,
+            termination_reason=termination_reason,
+        )

aehf/adapters/base.py

@@ -0,0 +1,14 @@
+from typing import Protocol, runtime_checkable
+
+from aehf.core.case import EvalCase
+from aehf.core.transcript import Transcript
+
+
+@runtime_checkable
+class Agent(Protocol):
+
+    async def run(self, case: EvalCase) -> Transcript:
+        ...
+
+
+

aehf/adapters/fake.py

@@ -0,0 +1,12 @@
+
+from aehf.core.case import EvalCase
+from aehf.core.transcript import Transcript
+
+
+class FakeAgent:
+    def __init__(self, transcripts:dict[str,Transcript]) -> None:
+        self.transcripts = transcripts
+    
+    async def run(self, case : EvalCase) ->  Transcript:
+            
+        return self.transcripts[case.id]

aehf/cli.py

@@ -0,0 +1,295 @@
+import asyncio
+import os
+import subprocess
+from enum import Enum
+from pathlib import Path
+
+import typer
+from anthropic import AsyncAnthropic
+from dotenv import load_dotenv
+
+from aehf.adapters.anthropic import AnthropicAdapter
+from aehf.adapters.base import Agent
+from aehf.core.loader import SuiteLoadError, load_suite
+from aehf.core.results import SuiteResult, Verdict, load_suite_result, save_suite_result
+from aehf.core.runner import run_suite
+from aehf.core.transcript import Termination
+from aehf.judges.assertionjudge import AssertionJudge
+from aehf.judges.base import Judge
+from aehf.judges.calibration import (
+    LabeledTranscript,
+    LabelLoadError,
+    cohen_kappa,
+    export_unlabeled,
+    load_labeled,
+    render_for_label,
+    save_labeled,
+)
+from aehf.judges.llmjudge import LLMJudge
+from aehf.judges.runner import judge_suite
+from aehf.regression.diff import DiffResult, diff_runs, is_regression
+from aehf.regression.store import RunNotFoundError, load_run, save_run
+from aehf.reporting.markdown import render_markdown
+from aehf.stats.aggregate import CaseStats, aggregate
+from aehf.stats.compare import ComparisonResult, compare_result
+from aehf.tools.base import ToolProviderFactory
+from aehf.tools.mock import mock_provider_factory
+from aehf.tools.replay import record_provider_factory, replay_provider_factory
+
+app = typer.Typer()
+
+
+class AgentChoice(Enum):
+    anthropic = "anthropic"
+    
+class ToolChoice(Enum):
+    mock = "mock"
+    record = "record"
+    replay = "replay"
+
+class JudgeChoice(Enum):
+    assertion = "assertion"
+    llm = "llm"
+def build_agent(agent: AgentChoice, tools : ToolChoice, recordings_dir:Path, model : str, max_tokens:int) -> Agent: 
+    
+    if tools == ToolChoice.mock:
+        factory : ToolProviderFactory = mock_provider_factory
+    elif tools  == ToolChoice.record:
+        factory  = record_provider_factory(recordings_dir,mock_provider_factory)
+    elif tools == ToolChoice.replay:
+        factory = replay_provider_factory(recordings_dir)
+    else:
+        raise ValueError(f"unknown tools choice {tools}")
+
+    if agent == AgentChoice.anthropic:
+        api_key = os.environ.get("ANTHROPIC_API_KEY")
+        if not api_key:
+            print("ANTHROPIC_API_KEY not set")
+            raise typer.Exit(2)
+        client = AsyncAnthropic(api_key = api_key)
+        return AnthropicAdapter(client = client,provider_factory = factory,model = model, max_tokens = max_tokens)
+    raise ValueError(f"unknown agent choice {agent}")
+    
+def build_judge(judgechoice: JudgeChoice, prompt_version:str,model :str,max_tokens:int) -> Judge:
+
+    if judgechoice == JudgeChoice.assertion:
+        judge = AssertionJudge() 
+    elif judgechoice == JudgeChoice.llm:
+        
+        key = os.environ.get("ANTHROPIC_API_KEY")
+        if not key:
+            print("ANTHROPIC_API_KEY not set")
+            raise typer.Exit(2)
+        return LLMJudge(client=AsyncAnthropic(api_key=key), prompt_version=prompt_version, model=model, max_tokens=max_tokens)
+    else:
+        raise ValueError(f"unknown Judge choice : {judgechoice}")
+    
+    return judge
+
+def _current_sha() -> str:
+    # CI hands us the SHA directly; fall back to git for local runs
+    sha = os.environ.get("GITHUB_SHA")
+    if sha:
+        return sha
+    try:
+        return subprocess.run(
+            ["git", "rev-parse", "HEAD"], capture_output=True, text=True, check=True
+        ).stdout.strip()
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        return "unknown"
+
+
+async def _score_labeled(judge:Judge, records : list[LabeledTranscript], concurrency : int) -> list[Verdict]:
+    sem = asyncio.Semaphore(concurrency)
+
+    async def one(record: LabeledTranscript) -> Verdict:
+        async with sem:
+            return await judge.score(record.case,record.transcript)
+    return await asyncio.gather(*(one(r) for r in records))
+
+
+
+
+@app.command()
+def run(path: Path, agent : AgentChoice, tools: ToolChoice, n_samples:int = 1,  concurrency: int = 5, max_tokens : int = 1024, recordings_dir : Path = Path("output/"), model : str = "claude-haiku-4-5-20251001", prompt_version:str = "v1",judge_model :str = "claude-haiku-4-5-20251001",judgechoice : JudgeChoice = JudgeChoice.assertion, output :Path | None = None, store: Path | None = None) ->None:
+    load_dotenv()
+    try:
+        suite = load_suite(path)
+    except (SuiteLoadError, FileNotFoundError) as e:
+        print(e)
+        raise typer.Exit(2)
+    if n_samples == 0:
+        raise ValueError("n_samples value must be greater than 0")
+    if output is not None:
+        output.mkdir(parents=True, exist_ok=True)
+    adapter = build_agent(agent,tools,recordings_dir,model,max_tokens)
+    judge = build_judge(judgechoice,prompt_version,judge_model,max_tokens)
+    runs : list[SuiteResult] = []
+    for _ in range(n_samples):
+        suite_result = asyncio.run(run_suite(adapter,suite,concurrency))
+        runs.append(asyncio.run(judge_suite(judge,suite,suite_result,concurrency)))
+
+    if store is not None:
+        # regression store holds ONE canonical run per (sha, model, judge_version);
+        # diff compares single runs. ponytail: store runs[0]; per-sample storage is
+        # future work if diff ever consumes n-sampled data.
+        save_run(store, _current_sha(), model, prompt_version, runs[0])
+    if n_samples == 1:
+        if output is not None:
+            save_suite_result(suite_result, output / f"{suite_result.suite_name}-{suite_result.run_id}.json")
+        failed = [r for r in suite_result.results if r.transcript.termination_reason != Termination.finished]
+        print(f"{len(suite_result.results)} cases: {len(suite_result.results) - len(failed)} completed, {len(failed)} failed") 
+        print(f"{'id':>23} {'reason':<12} {'steps':>5} {'tokens':>7} {'secs':>6}")
+        for r in suite_result.results:
+            t = r.transcript
+            
+            print(f"{r.case_id:>23} {t.termination_reason.value:<12} {len(t.ordered_steps):>5} {t.total_tokens:>7} {t.duration_seconds:>6.2f}")
+        if failed:
+            raise typer.Exit(1)
+    else:
+        if output is not None:
+            for sr in runs:
+                save_suite_result(sr, output / f"{sr.suite_name}-{sr.run_id}.json")
+        aggregate_results:list[CaseStats] = aggregate(runs)
+        print(f"{'id':>23} {'n':>5} {'pass':>5} {'rate':>7} {'95% CI':>16} {'flaky':>6}")
+        for case_stats in aggregate_results:
+            ci = f"[{case_stats.ci_low:.2f}, {case_stats.ci_high:.2f}]"
+            flag = "flaky" if case_stats.flaky else ""
+            print(f"{case_stats.case_id:>23} {case_stats.n:>5} {case_stats.passes:>5} {case_stats.pass_rate:>7.2f} {ci:>16} {flag:>6}")
+
+    
+
+@app.command()
+def calibrate(label: Path,judgechoice : JudgeChoice, prompt_version :str = "v1", judge_model :str = "claude-haiku-4-5-20251001", max_tokens : int = 1024 ,concurrency : int = 5) -> None:
+    load_dotenv()
+    try:
+        records = load_labeled(label)
+    except (LabelLoadError , FileNotFoundError) as e:
+        print(e)
+        raise typer.Exit(2)
+
+    if not records:
+        print("No labeled records found")
+        raise typer.Exit(2)
+    judge = build_judge(judgechoice,prompt_version,judge_model,max_tokens)
+    verdicts = asyncio.run(_score_labeled(judge, records,concurrency))
+    judge_labels = [v.passed for v in verdicts]
+    human_labels = [r.human_label for r in records]
+    kappa = cohen_kappa(judge_labels,human_labels)
+    agreement = sum(j == h for j,h in zip(judge_labels,human_labels,strict = True))
+
+    print(f"records : {len(records)}")
+    print(f"raw agreement : {agreement}/{len(records)}")
+    print(f"cohen kappa : {kappa:.3f}")
+
+    jp_hp = sum(v.passed and r.human_label for v, r in zip(verdicts, records, strict=True))
+    jp_hf = sum(v.passed and not r.human_label for v, r in zip(verdicts, records, strict=True))
+    jf_hp = sum(not v.passed and r.human_label for v, r in zip(verdicts, records, strict=True))
+    jf_hf = sum(not v.passed and not r.human_label for v, r in zip(verdicts, records, strict=True))
+    print()
+    print(f"{'':12}{'human_pass':>12}{'human_fail':>12}")
+    print(f"{'judge_pass':12}{jp_hp:>12}{jp_hf:>12}")
+    print(f"{'judge_fail':12}{jf_hp:>12}{jf_hf:>12}")
+
+    
+    disagreements = [(v, r) for v, r in zip(verdicts, records, strict=True) if v.passed != r.human_label]
+    print(f"\nDISAGREEMENTS ({len(disagreements)}):")
+    for v, r in disagreements:
+        human = "PASS" if r.human_label else "FAIL"
+        judge_label = "PASS" if v.passed else "FAIL"
+        conf = f"{v.score:.2f}" if v.score is not None else "n/a"
+        print(f"  {r.case.id:12} human={human} judge={judge_label} conf={conf}")
+        print(f"    judge: {v.reasoning}")
+
+@app.command()
+def export_labels(suite_path: Path, out: Path, results: list[Path]) -> None:
+    load_dotenv()
+    try:
+        suite = load_suite(suite_path)
+    except (SuiteLoadError, FileNotFoundError) as e:
+        print(e)
+        raise typer.Exit(2)
+
+    records: list[LabeledTranscript] = []
+    for path in results:
+        try:
+            suite_result = load_suite_result(path)
+        except FileNotFoundError as e:
+            print(e)
+            raise typer.Exit(2)
+        records.extend(export_unlabeled(suite, suite_result))
+
+    save_labeled(records, out)
+    print(f"wrote {len(records)} unlabeled records to {out}")  
+
+@app.command()
+def compare(path_a: Path, path_b: Path) -> None:
+    try:
+        a = load_suite_result(path_a)
+        b = load_suite_result(path_b)
+        result : ComparisonResult = compare_result(a,b)
+        print(f"n_cases   : {result.n_cases}")
+        print(f"a passed  : {result.a_passed}")
+        print(f"b passed  : {result.b_passed}")
+        print(f"both pass : {result.both_pass}")
+        print(f"both fail : {result.both_fail}")
+        print(f" only a   : {result.a_only}")
+        print(f" only b   : {result.b_only}")
+        verdict = "significant" if result.p_value < 0.05 else "not significant"
+        print(f" P value  : {result.p_value:.4f} → {verdict} at alpha=0.05")
+    except (FileNotFoundError,ValueError) as e:
+        print(e)
+        raise typer.Exit(2)
+
+@app.command()
+def diff(base_sha:str, head_sha:str, model: str = "claude-haiku-4-5-20251001", judge_version: str = "v1", store: Path = Path(".aehf"), out: Path = Path("scorecard.md") ) -> None:
+
+    try:
+        base = load_run(base = store,sha = base_sha,model = model, judge_version =judge_version)
+        head = load_run(base = store,sha = head_sha,model = model, judge_version =judge_version)
+        diff_result : DiffResult = diff_runs(base,head)
+    except RunNotFoundError as e:
+        print(e)
+        raise typer.Exit(2)
+    except ValueError as e:
+        print(e)
+        raise typer.Exit(2)
+
+    markdown_string = render_markdown(diff_result,base_sha,head_sha)
+    print(markdown_string)                       # terminal user sees the scorecard
+    out.parent.mkdir(parents = True, exist_ok = True)
+    out.write_text(markdown_string, encoding = "utf-8")
+
+    if is_regression(diff_result):
+        raise typer.Exit(1)
+@app.command()
+def label(path: Path, only_provisional: bool = False) -> None:
+    """Interactively set human_label and note on each record, saving after each."""
+    try:
+        records = load_labeled(path)
+    except (LabelLoadError, FileNotFoundError) as e:
+        print(e)
+        raise typer.Exit(2)
+
+    for i, rec in enumerate(records):
+        if only_provisional and not rec.note.startswith("provisional"):
+            continue
+        print("\n" + render_for_label(rec))
+        current = "PASS" if rec.human_label else "FAIL"
+        print(f"[{i + 1}/{len(records)}] current label: {current}  note={rec.note!r}")
+
+        choice = typer.prompt("[p]ass / [f]ail / [s]kip / [q]uit", default="s")
+        if choice == "q":
+            break
+        if choice not in ("p", "f"):
+            continue  # skip: leave the record as-is
+
+        note = typer.prompt("note", default="reviewed")
+        records[i] = rec.model_copy(update={"human_label": choice == "p", "note": note})
+        save_labeled(records, path)  # persist after every label so quitting is safe
+
+    print("saved.")
+
+
+if __name__== "__main__":
+    app()

aehf/core/case.py

@@ -0,0 +1,47 @@
+import re
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+
+class SuccessCriteria(BaseModel):
+    model_config = ConfigDict(frozen = True)
+    
+    answer_regex : str | None = None
+    required_tools: list[str] = Field(default_factory=list)
+    forbidden_tools: list[str] = Field(default_factory=list)
+    rubric : str | None = None
+    @field_validator("answer_regex")
+    @classmethod
+    def _must_compile(cls, v: str | None) -> str | None:
+        #
+        if v is not None:
+            try:
+                re.compile(v)
+            except re.error as e:
+                raise ValueError(f"invalid answer_regex {v!r}: {e}") from e
+        return v
+
+class ToolSpec(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    name: str
+    description: str | None = None
+    parameters: dict[str,Any] = Field(default_factory = dict[str,Any])
+    
+
+class EvalCase(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    id: str
+    task_prompt: str
+    tools : list[ToolSpec]
+    success_criteria: SuccessCriteria
+    max_steps: int = Field(..., description = "Maximum Number of Steps")
+    timeout_seconds: float = Field(..., description = "Time out Seconds") 
+    token_budget: int = Field(..., description = "Token Budget")
+    tags: str | None = None
+    tool_fixtures: dict[str,str] | None = None
+
+class Suite(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    name: str
+    eval: list[EvalCase]

aehf/core/loader.py

@@ -0,0 +1,31 @@
+from pathlib import Path
+
+import yaml
+from pydantic import ValidationError
+
+from aehf.core.case import Suite
+
+
+class SuiteLoadError(Exception):
+    ...
+
+def load_suite(path: Path) -> Suite:
+
+    
+    with open(path,encoding = "utf-8") as file:
+        try:
+            data = yaml.safe_load(file)
+        except yaml.YAMLError as e:
+            raise SuiteLoadError(f"{path}: {e}") from e
+
+    
+    try:
+        result = Suite.model_validate(data)
+
+    except ValidationError as e:
+        raise SuiteLoadError(f"{path}: {e}") from e
+    
+    return result
+
+
+    

aehf/core/results.py

@@ -0,0 +1,36 @@
+from pathlib import Path
+
+from pydantic import BaseModel, ConfigDict
+
+from aehf.core.transcript import Transcript
+
+
+class Verdict(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    passed : bool
+    score: float | None = None
+    reasoning: str
+    judge_name: str
+    version: str 
+
+
+class CaseResult(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    case_id : str
+    transcript : Transcript
+    verdicts: list[Verdict]
+    run_metadata: dict[str,str]
+
+class SuiteResult(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    suite_name: str
+    results: list[CaseResult]
+    run_id: str
+
+def save_suite_result(result: SuiteResult, path: Path) -> None:
+    path.write_text(result.model_dump_json(indent=2))
+def load_suite_result(path: Path) -> SuiteResult:
+    return SuiteResult.model_validate_json(path.read_text())
+
+def case_passed(cr: CaseResult) -> bool:
+    return bool(cr.verdicts) and cr.verdicts[0].passed

aehf/core/runner.py

@@ -0,0 +1,51 @@
+import asyncio
+import time
+import traceback
+from typing import Any
+from uuid import uuid4
+
+from aehf.adapters.base import Agent
+from aehf.core.case import EvalCase, Suite
+from aehf.core.results import CaseResult, SuiteResult
+from aehf.core.transcript import Termination, Transcript
+
+
+async def run_case(agent: Agent, case:EvalCase) -> CaseResult:
+    run_metadata : dict[str,Any] = {}
+    start_time = time.monotonic()
+    try:
+        transcript = await asyncio.wait_for(agent.run(case),timeout = case.timeout_seconds)
+        duration =  time.monotonic()-start_time
+        transcript = transcript.model_copy(update = {"duration_seconds" : duration})
+        if len(transcript.ordered_steps) > case.max_steps and transcript.total_tokens >case.token_budget:
+            transcript = transcript.model_copy(update ={"termination_reason" : Termination.budget_and_steps_exceeded})
+        elif transcript.total_tokens >case.token_budget:
+            transcript = transcript.model_copy(update ={"termination_reason" : Termination.budget_exceeded})
+
+        elif len(transcript.ordered_steps) > case.max_steps:
+            transcript = transcript.model_copy(update ={"termination_reason" :Termination.max_steps})
+    except TimeoutError:
+        transcript = Transcript(id = case.id,ordered_steps = [], final_answer ="",total_tokens= -1,duration_seconds = time.monotonic()-start_time,termination_reason = Termination.timeout)
+    except Exception as exc:
+        transcript = Transcript(id = case.id,ordered_steps = [], final_answer ="",total_tokens= -1,duration_seconds = time.monotonic() - start_time,termination_reason = Termination.crashed)
+        run_metadata['error_type'] = type(exc).__name__
+        run_metadata['error_message'] = str(exc)
+        run_metadata['error_traceback'] = traceback.format_exc()
+
+    case_result = CaseResult(case_id = case.id,transcript = transcript, verdicts = [], run_metadata = run_metadata) 
+    return case_result
+
+async def _run_case_bounded(agent:Agent, case:EvalCase, sem:asyncio.Semaphore) -> CaseResult:
+    async with sem:
+        return await run_case(agent,case)
+async def run_suite(agent:Agent, suite:Suite, max_concurrency :int = 5) -> SuiteResult:
+    sem = asyncio.Semaphore(max_concurrency)
+    
+    tasks = [_run_case_bounded(agent,c,sem) for c in suite.eval]
+    result = await asyncio.gather(*tasks)
+    
+    suiteresult = SuiteResult(suite_name = suite.name, results = result,run_id = uuid4().hex[:8])
+    return suiteresult
+ 
+
+

aehf/core/transcript.py

@@ -0,0 +1,39 @@
+
+from enum import Enum
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class Termination(Enum):
+    finished = "finished"
+    max_steps = "max_steps"
+    timeout = "timeout"
+    crashed= "crashed"
+    refused = "refused"
+    unexpected_stop = "unexpected_stop"
+    budget_exceeded = "budget_exceeded"
+    budget_and_steps_exceeded = "budget_and_steps_exceeded"
+
+class ToolCall(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    toolname: str
+    arguments: dict[str,Any]
+    result: str
+    error_flag: str | None =None
+    latency: float
+
+class Step(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    model_output:str
+    tool_calls: list[ToolCall] | None=None
+    token_usage: int= Field(..., description ="Token Usage")
+
+class Transcript(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    id: str
+    ordered_steps: list[Step]
+    final_answer: str
+    total_tokens: int
+    duration_seconds: float 
+    termination_reason: Termination

aehf/judges/assertionjudge.py

@@ -0,0 +1,42 @@
+import re
+
+from aehf.core.case import EvalCase
+from aehf.core.results import Verdict
+from aehf.core.transcript import Transcript
+
+
+class AssertionJudge:
+
+    async def score(self, case : EvalCase, transcript: Transcript) -> Verdict:
+        criteria = case.success_criteria
+        reasons: list[str] = []
+        passed = True
+
+        if criteria.answer_regex is not None:
+            if re.search(criteria.answer_regex, transcript.final_answer):
+                reasons.append("answer matched")
+            else:
+                reasons.append("answer did not match")
+                passed = False
+
+        called = [tc.toolname for step in transcript.ordered_steps for tc in (step.tool_calls or [])]
+
+        forbidden_hits = sorted({t for t in called if t in criteria.forbidden_tools})
+        if forbidden_hits:
+            reasons.append(f"forbidden tool(s) called: {', '.join(forbidden_hits)}")
+            passed = False
+
+        missing = sorted(t for t in criteria.required_tools if t not in called)
+        if missing:
+            reasons.append(f"required tool(s) not called: {', '.join(missing)}")
+            passed = False
+        elif criteria.required_tools:
+            reasons.append("all required tools called")
+
+        return Verdict(
+            passed=passed,
+            score=1.0 if passed else 0.0,
+            reasoning="; ".join(reasons) or "no criteria specified",
+            judge_name="AssertionJudge",
+            version="1",
+        )