agentgrader 0.1.0__py3-none-any.whl

agentaudit/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

agentaudit/checks/accuracy.py

@@ -0,0 +1,63 @@
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+import yaml
+
+from agentaudit.checks.base import CheckResult, execute_checks
+from agentaudit.config import JudgeConfig
+from agentaudit.judge.groq_judge import judge
+from agentaudit.targets.base import Target
+
+
+@dataclass
+class AccuracyCase:
+    id: str
+    prompt: str
+    rubric: str
+
+
+def load_accuracy_cases(path: str | Path) -> list[AccuracyCase]:
+    data = yaml.safe_load(Path(path).read_text())
+    cases = []
+    for i, raw in enumerate(data.get("cases", [])):
+        rubric = raw.get("rubric")
+        if not rubric:
+            rubric = f"The response must convey the same meaning as this expected answer: {raw['expected']}"
+        cases.append(AccuracyCase(id=raw.get("id", f"accuracy-{i}"), prompt=raw["prompt"], rubric=rubric))
+    return cases
+
+
+def _process_accuracy_case(target: Target, case: AccuracyCase, judge_config: JudgeConfig) -> CheckResult:
+    started_at = time.perf_counter()
+    response = target.send(case.prompt)
+    verdict = judge(prompt=case.prompt, response=response, rubric=case.rubric, judge_config=judge_config)
+    latency_ms = (time.perf_counter() - started_at) * 1000
+    return CheckResult(
+        check_type="accuracy",
+        case_id=case.id,
+        target_name=target.name,
+        prompt=case.prompt,
+        response=response,
+        passed=verdict.passed,
+        score=verdict.score,
+        reasoning=verdict.reasoning,
+        latency_ms=latency_ms,
+        judge_prompt_tokens=verdict.prompt_tokens,
+        judge_completion_tokens=verdict.completion_tokens,
+    )
+
+
+def run_accuracy_checks(
+    target: Target,
+    cases: list[AccuracyCase],
+    judge_config: JudgeConfig = JudgeConfig(),
+    repeat: int = 1,
+    max_workers: int = 2,
+) -> list[CheckResult]:
+    return execute_checks(
+        lambda case: _process_accuracy_case(target, case, judge_config),
+        cases,
+        repeat=repeat,
+        max_workers=max_workers,
+    )

agentaudit/checks/base.py

@@ -0,0 +1,73 @@
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass, field
+from typing import Any, TypeVar
+
+
+@dataclass
+class CheckResult:
+    check_type: str
+    case_id: str
+    target_name: str
+    prompt: str
+    response: str
+    passed: bool
+    score: float
+    reasoning: str
+    latency_ms: float = 0.0
+    judge_prompt_tokens: int = 0
+    judge_completion_tokens: int = 0
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+CaseT = TypeVar("CaseT")
+
+
+def execute_checks(
+    process_case: Callable[[CaseT], CheckResult],
+    cases: list[CaseT],
+    repeat: int = 1,
+    require_all_attempts_pass: bool = False,
+    max_workers: int = 2,
+) -> list[CheckResult]:
+    def run_one(case: CaseT) -> CheckResult:
+        attempts = [process_case(case) for _ in range(repeat)]
+        return attempts[0] if repeat == 1 else _aggregate_attempts(attempts, require_all_attempts_pass)
+
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        return list(executor.map(run_one, cases))
+
+
+def _aggregate_attempts(attempts: list[CheckResult], require_all_attempts_pass: bool) -> CheckResult:
+    if require_all_attempts_pass:
+        passed = all(attempt.passed for attempt in attempts)
+    else:
+        passed_count = sum(1 for attempt in attempts if attempt.passed)
+        passed = passed_count * 2 > len(attempts)
+
+    failing = [attempt for attempt in attempts if not attempt.passed]
+    if failing:
+        reasoning = f"{len(attempts) - len(failing)}/{len(attempts)} runs passed. Example failure: {failing[0].reasoning}"
+    else:
+        reasoning = f"All {len(attempts)} runs passed. {attempts[0].reasoning}"
+
+    first = attempts[0]
+    return CheckResult(
+        check_type=first.check_type,
+        case_id=first.case_id,
+        target_name=first.target_name,
+        prompt=first.prompt,
+        response=first.response,
+        passed=passed,
+        score=sum(attempt.score for attempt in attempts) / len(attempts),
+        reasoning=reasoning,
+        latency_ms=sum(attempt.latency_ms for attempt in attempts),
+        judge_prompt_tokens=sum(attempt.judge_prompt_tokens for attempt in attempts),
+        judge_completion_tokens=sum(attempt.judge_completion_tokens for attempt in attempts),
+        metadata={
+            **first.metadata,
+            "repeat_attempts": [
+                {"passed": attempt.passed, "score": attempt.score, "reasoning": attempt.reasoning} for attempt in attempts
+            ],
+        },
+    )

agentaudit/checks/fairness.py

@@ -0,0 +1,118 @@
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+import yaml
+
+from agentaudit.checks.base import CheckResult, execute_checks
+from agentaudit.config import JudgeConfig
+from agentaudit.judge.groq_judge import judge_pair
+from agentaudit.targets.base import Target
+
+DEFAULT_COMPARISON_FOCUS = (
+    "Check whether the agent offers the same level of helpfulness, accuracy, tone, and "
+    "willingness to assist regardless of the requester's name."
+)
+
+
+@dataclass
+class FairnessCase:
+    id: str
+    prompt_template: str
+    identities: list[str]
+    comparison_focus: str
+
+    @property
+    def baseline_identity(self) -> str:
+        return self.identities[0]
+
+    @property
+    def compared_identities(self) -> list[str]:
+        return self.identities[1:]
+
+    def prompt_for(self, identity: str) -> str:
+        return self.prompt_template.format(name=identity)
+
+
+def load_fairness_cases(path: str | Path) -> list[FairnessCase]:
+    data = yaml.safe_load(Path(path).read_text())
+    cases = []
+    for i, raw in enumerate(data.get("cases", [])):
+        cases.append(
+            FairnessCase(
+                id=raw.get("id", f"fairness-{i}"),
+                prompt_template=raw["prompt_template"],
+                identities=raw["identities"],
+                comparison_focus=raw.get("comparison_focus", DEFAULT_COMPARISON_FOCUS),
+            )
+        )
+    return cases
+
+
+def _process_fairness_case(target: Target, case: FairnessCase, judge_config: JudgeConfig) -> CheckResult:
+    baseline_prompt = case.prompt_for(case.baseline_identity)
+    baseline_response = target.send(baseline_prompt)
+
+    comparisons = []
+    started_at = time.perf_counter()
+    for identity in case.compared_identities:
+        variant_prompt = case.prompt_for(identity)
+        variant_response = target.send(variant_prompt)
+        verdict = judge_pair(
+            prompt_a=baseline_prompt,
+            response_a=baseline_response,
+            prompt_b=variant_prompt,
+            response_b=variant_response,
+            comparison_focus=case.comparison_focus,
+            judge_config=judge_config,
+        )
+        comparisons.append(
+            {
+                "identity": identity,
+                "prompt": variant_prompt,
+                "response": variant_response,
+                "equivalent": verdict.equivalent,
+                "score": verdict.score,
+                "reasoning": verdict.reasoning,
+                "prompt_tokens": verdict.prompt_tokens,
+                "completion_tokens": verdict.completion_tokens,
+            }
+        )
+    latency_ms = (time.perf_counter() - started_at) * 1000
+
+    failing = [c for c in comparisons if not c["equivalent"]]
+    if failing:
+        failing_identities = ", ".join(c["identity"] for c in failing)
+        reasoning = f"Treatment diverged from the '{case.baseline_identity}' baseline for: {failing_identities}."
+    else:
+        reasoning = f"All identities received equivalent treatment to the '{case.baseline_identity}' baseline."
+
+    return CheckResult(
+        check_type="fairness",
+        case_id=case.id,
+        target_name=target.name,
+        prompt=baseline_prompt,
+        response=baseline_response,
+        passed=not failing,
+        score=sum(c["score"] for c in comparisons) / len(comparisons),
+        reasoning=reasoning,
+        latency_ms=latency_ms,
+        judge_prompt_tokens=sum(c["prompt_tokens"] for c in comparisons),
+        judge_completion_tokens=sum(c["completion_tokens"] for c in comparisons),
+        metadata={"baseline_identity": case.baseline_identity, "comparisons": comparisons},
+    )
+
+
+def run_fairness_checks(
+    target: Target,
+    cases: list[FairnessCase],
+    judge_config: JudgeConfig = JudgeConfig(),
+    repeat: int = 1,
+    max_workers: int = 2,
+) -> list[CheckResult]:
+    return execute_checks(
+        lambda case: _process_fairness_case(target, case, judge_config),
+        cases,
+        repeat=repeat,
+        max_workers=max_workers,
+    )

agentaudit/checks/robustness.py

@@ -0,0 +1,97 @@
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+import yaml
+
+from agentaudit.checks.base import CheckResult, execute_checks
+from agentaudit.config import JudgeConfig
+from agentaudit.judge.groq_judge import judge
+from agentaudit.targets.base import Target
+
+
+@dataclass
+class RobustnessCase:
+    id: str
+    base_prompt: str
+    paraphrases: list[str]
+    rubric: str
+
+    @property
+    def all_prompts(self) -> list[str]:
+        return [self.base_prompt] + self.paraphrases
+
+
+def load_robustness_cases(path: str | Path) -> list[RobustnessCase]:
+    data = yaml.safe_load(Path(path).read_text())
+    cases = []
+    for i, raw in enumerate(data.get("cases", [])):
+        rubric = raw.get("rubric")
+        if not rubric:
+            rubric = f"The response must convey the same meaning as this expected answer: {raw['expected']}"
+        cases.append(
+            RobustnessCase(
+                id=raw.get("id", f"robustness-{i}"),
+                base_prompt=raw["base_prompt"],
+                paraphrases=raw.get("paraphrases", []),
+                rubric=rubric,
+            )
+        )
+    return cases
+
+
+def _process_robustness_case(target: Target, case: RobustnessCase, judge_config: JudgeConfig) -> CheckResult:
+    variants = []
+    started_at = time.perf_counter()
+    for prompt in case.all_prompts:
+        response = target.send(prompt)
+        verdict = judge(prompt=prompt, response=response, rubric=case.rubric, judge_config=judge_config)
+        variants.append(
+            {
+                "prompt": prompt,
+                "response": response,
+                "passed": verdict.passed,
+                "score": verdict.score,
+                "reasoning": verdict.reasoning,
+                "prompt_tokens": verdict.prompt_tokens,
+                "completion_tokens": verdict.completion_tokens,
+            }
+        )
+    latency_ms = (time.perf_counter() - started_at) * 1000
+
+    failing = [v for v in variants if not v["passed"]]
+    if failing:
+        failing_prompts = ", ".join(repr(v["prompt"]) for v in failing)
+        reasoning = f"{len(variants) - len(failing)}/{len(variants)} phrasings passed. Failed on: {failing_prompts}"
+    else:
+        reasoning = f"All {len(variants)} phrasings produced a passing answer."
+
+    return CheckResult(
+        check_type="robustness",
+        case_id=case.id,
+        target_name=target.name,
+        prompt=case.base_prompt,
+        response=variants[0]["response"],
+        passed=not failing,
+        score=sum(v["score"] for v in variants) / len(variants),
+        reasoning=reasoning,
+        latency_ms=latency_ms,
+        judge_prompt_tokens=sum(v["prompt_tokens"] for v in variants),
+        judge_completion_tokens=sum(v["completion_tokens"] for v in variants),
+        metadata={"variants": variants, "phrasing_count": len(variants)},
+    )
+
+
+def run_robustness_checks(
+    target: Target,
+    cases: list[RobustnessCase],
+    judge_config: JudgeConfig = JudgeConfig(),
+    repeat: int = 1,
+    max_workers: int = 2,
+) -> list[CheckResult]:
+    return execute_checks(
+        lambda case: _process_robustness_case(target, case, judge_config),
+        cases,
+        repeat=repeat,
+        max_workers=max_workers,
+    )

agentaudit/checks/security.py

@@ -0,0 +1,77 @@
+import time
+from dataclasses import dataclass
+from importlib import resources
+from pathlib import Path
+
+import yaml
+
+from agentaudit.checks.base import CheckResult, execute_checks
+from agentaudit.config import JudgeConfig
+from agentaudit.judge.groq_judge import judge
+from agentaudit.targets.base import Target
+
+
+@dataclass
+class InjectionPattern:
+    id: str
+    category: str
+    prompt: str
+    detection_rubric: str
+
+
+def _parse_patterns(data: dict) -> list[InjectionPattern]:
+    return [
+        InjectionPattern(
+            id=raw["id"],
+            category=raw["category"],
+            prompt=raw["prompt"],
+            detection_rubric=raw["detection_rubric"],
+        )
+        for raw in data.get("patterns", [])
+    ]
+
+
+def load_injection_patterns(path: str | Path) -> list[InjectionPattern]:
+    return _parse_patterns(yaml.safe_load(Path(path).read_text()))
+
+
+def load_default_injection_patterns() -> list[InjectionPattern]:
+    text = resources.files("agentaudit.data").joinpath("injection_patterns.yaml").read_text()
+    return _parse_patterns(yaml.safe_load(text))
+
+
+def _process_security_pattern(target: Target, pattern: InjectionPattern, judge_config: JudgeConfig) -> CheckResult:
+    started_at = time.perf_counter()
+    response = target.send(pattern.prompt)
+    verdict = judge(prompt=pattern.prompt, response=response, rubric=pattern.detection_rubric, judge_config=judge_config)
+    latency_ms = (time.perf_counter() - started_at) * 1000
+    return CheckResult(
+        check_type="security",
+        case_id=pattern.id,
+        target_name=target.name,
+        prompt=pattern.prompt,
+        response=response,
+        passed=verdict.passed,
+        score=verdict.score,
+        reasoning=verdict.reasoning,
+        latency_ms=latency_ms,
+        judge_prompt_tokens=verdict.prompt_tokens,
+        judge_completion_tokens=verdict.completion_tokens,
+        metadata={"category": pattern.category},
+    )
+
+
+def run_security_checks(
+    target: Target,
+    patterns: list[InjectionPattern],
+    judge_config: JudgeConfig = JudgeConfig(),
+    repeat: int = 1,
+    max_workers: int = 2,
+) -> list[CheckResult]:
+    return execute_checks(
+        lambda pattern: _process_security_pattern(target, pattern, judge_config),
+        patterns,
+        repeat=repeat,
+        require_all_attempts_pass=True,
+        max_workers=max_workers,
+    )

agentaudit/cli.py

@@ -0,0 +1,233 @@
+import sys
+from pathlib import Path
+
+import click
+from rich.console import Console
+
+from agentaudit import __version__
+from agentaudit.checks.accuracy import load_accuracy_cases, run_accuracy_checks
+from agentaudit.checks.fairness import load_fairness_cases, run_fairness_checks
+from agentaudit.checks.robustness import load_robustness_cases, run_robustness_checks
+from agentaudit.checks.security import load_default_injection_patterns, load_injection_patterns, run_security_checks
+from agentaudit.config import (
+    DEFAULT_JUDGE_API_KEY_ENV,
+    DEFAULT_JUDGE_BASE_URL,
+    DEFAULT_JUDGE_MODEL,
+    DEFAULT_SCORE_THRESHOLD,
+    JudgeConfig,
+)
+from agentaudit.preflight import run_preflight
+from agentaudit.reporting.console import render_console_report
+from agentaudit.reporting.html import render_html_report
+from agentaudit.reporting.json_report import render_json_report
+from agentaudit.summary import summarize
+from agentaudit.targets import resolve_target
+
+TARGET_OPTION = click.option(
+    "--target",
+    "target_ref",
+    required=True,
+    help="Callable path (module.path:function_name) or HTTP URL for the agent under test.",
+)
+JUDGE_MODEL_OPTION = click.option(
+    "--judge-model", default=DEFAULT_JUDGE_MODEL, show_default=True, help="Model used to judge responses."
+)
+JUDGE_BASE_URL_OPTION = click.option(
+    "--judge-base-url",
+    default=DEFAULT_JUDGE_BASE_URL,
+    show_default=True,
+    help="OpenAI-compatible base URL for the judge provider (Groq, OpenAI, Together, a local server, etc).",
+)
+JUDGE_API_KEY_ENV_OPTION = click.option(
+    "--judge-api-key-env",
+    default=DEFAULT_JUDGE_API_KEY_ENV,
+    show_default=True,
+    help="Name of the environment variable holding the judge provider's API key.",
+)
+REQUEST_FIELD_OPTION = click.option(
+    "--request-field",
+    default="message",
+    show_default=True,
+    help="JSON field name the prompt is sent under, for HTTP targets.",
+)
+RESPONSE_FIELD_OPTION = click.option(
+    "--response-field",
+    default="response",
+    show_default=True,
+    help="JSON field name the response is read from, for HTTP targets.",
+)
+
+
+def _judge_config(judge_model: str, judge_base_url: str, judge_api_key_env: str) -> JudgeConfig:
+    return JudgeConfig(model=judge_model, base_url=judge_base_url, api_key_env=judge_api_key_env)
+
+
+@click.group()
+@click.version_option(version=__version__, prog_name="agentaudit")
+def main() -> None:
+    pass
+
+
+@main.command()
+@TARGET_OPTION
+@click.option(
+    "--cases",
+    "cases_dir",
+    required=True,
+    type=click.Path(exists=True, file_okay=False),
+    help="Directory containing accuracy_cases.yaml, robustness_cases.yaml, fairness_cases.yaml, and/or security_cases.yaml.",
+)
+@click.option("--output", "output_path", default="report.html", show_default=True, help="Path to write the HTML report.")
+@click.option("--json-output", "json_output_path", default=None, help="Optional path to also write a JSON report.")
+@click.option(
+    "--threshold",
+    default=DEFAULT_SCORE_THRESHOLD,
+    show_default=True,
+    type=float,
+    help="Minimum aggregate score (percent) required to pass.",
+)
+@click.option(
+    "--strict-security/--no-strict-security",
+    default=True,
+    show_default=True,
+    help="Fail the gate if any security check fails, regardless of aggregate score.",
+)
+@click.option(
+    "--strict-fairness/--no-strict-fairness",
+    default=True,
+    show_default=True,
+    help="Fail the gate if any fairness check fails, regardless of aggregate score.",
+)
+@JUDGE_MODEL_OPTION
+@JUDGE_BASE_URL_OPTION
+@JUDGE_API_KEY_ENV_OPTION
+@REQUEST_FIELD_OPTION
+@RESPONSE_FIELD_OPTION
+@click.option(
+    "--judge-cost-per-million-tokens",
+    "judge_cost_per_million_tokens",
+    default=None,
+    type=float,
+    help="If set, estimate judge $ cost using this price per 1M tokens.",
+)
+@click.option(
+    "--skip-preflight", is_flag=True, default=False, help="Skip the setup checks (API key, judge model, target) before running."
+)
+@click.option(
+    "--repeat",
+    default=1,
+    show_default=True,
+    type=int,
+    help="Run each check this many times before deciding pass/fail. Accuracy/robustness/fairness use majority "
+    "vote across the repeats; security fails if even one repeat succeeds. Use this to tell a real, consistent "
+    "finding apart from one-off judge noise.",
+)
+@click.option(
+    "--max-workers", default=2, show_default=True, type=int, help="Maximum number of checks to run concurrently."
+)
+def run(
+    target_ref: str,
+    cases_dir: str,
+    output_path: str,
+    json_output_path: str | None,
+    threshold: float,
+    strict_security: bool,
+    strict_fairness: bool,
+    judge_model: str,
+    judge_base_url: str,
+    judge_api_key_env: str,
+    request_field: str,
+    response_field: str,
+    judge_cost_per_million_tokens: float | None,
+    skip_preflight: bool,
+    repeat: int,
+    max_workers: int,
+) -> None:
+    console = Console()
+    sys.path.insert(0, str(Path.cwd()))
+    target = resolve_target(target_ref, request_field=request_field, response_field=response_field)
+    judge_config = _judge_config(judge_model, judge_base_url, judge_api_key_env)
+
+    if not skip_preflight:
+        preflight_results = run_preflight(target, judge_config)
+        for check_result in preflight_results:
+            icon = "[green]OK[/green]" if check_result.ok else "[red]FAIL[/red]"
+            console.print(f"{icon} {check_result.name}: {check_result.message}")
+        if not all(check_result.ok for check_result in preflight_results):
+            console.print("[red]Preflight checks failed, aborting before spending judge calls.[/red]")
+            sys.exit(1)
+
+    cases_path = Path(cases_dir)
+    results = []
+
+    for label, filename, load_cases, run_checks in (
+        ("accuracy", "accuracy_cases.yaml", load_accuracy_cases, run_accuracy_checks),
+        ("robustness", "robustness_cases.yaml", load_robustness_cases, run_robustness_checks),
+        ("fairness", "fairness_cases.yaml", load_fairness_cases, run_fairness_checks),
+    ):
+        case_file = cases_path / filename
+        if case_file.exists():
+            console.print(f"Running {label} checks from {case_file}")
+            cases = load_cases(case_file)
+            results += run_checks(target, cases, judge_config=judge_config, repeat=repeat, max_workers=max_workers)
+        else:
+            console.print(f"[yellow]No {filename} found in {cases_path}, skipping {label} checks.[/yellow]")
+
+    security_file = cases_path / "security_cases.yaml"
+    if security_file.exists():
+        console.print(f"Running security checks from {security_file}")
+        injection_patterns = load_injection_patterns(security_file)
+    else:
+        console.print(f"No security_cases.yaml found in {cases_path}, using AgentAudit's bundled injection pattern library.")
+        injection_patterns = load_default_injection_patterns()
+    results += run_security_checks(target, injection_patterns, judge_config=judge_config, repeat=repeat, max_workers=max_workers)
+
+    if not results:
+        console.print("[red]No checks were run.[/red]")
+        sys.exit(1)
+
+    summary = summarize(results, threshold=threshold, strict_security=strict_security, strict_fairness=strict_fairness)
+
+    render_console_report(results, summary, console=console, judge_cost_per_million_tokens=judge_cost_per_million_tokens)
+    render_html_report(results, summary, output_path, judge_cost_per_million_tokens=judge_cost_per_million_tokens)
+    console.print(f"\nHTML report written to {output_path}")
+
+    if json_output_path:
+        render_json_report(results, summary, json_output_path, judge_cost_per_million_tokens=judge_cost_per_million_tokens)
+        console.print(f"JSON report written to {json_output_path}")
+
+    if not summary.gate_passed:
+        sys.exit(1)
+
+
+@main.command()
+@TARGET_OPTION
+@JUDGE_MODEL_OPTION
+@JUDGE_BASE_URL_OPTION
+@JUDGE_API_KEY_ENV_OPTION
+@REQUEST_FIELD_OPTION
+@RESPONSE_FIELD_OPTION
+def check(
+    target_ref: str,
+    judge_model: str,
+    judge_base_url: str,
+    judge_api_key_env: str,
+    request_field: str,
+    response_field: str,
+) -> None:
+    console = Console()
+    sys.path.insert(0, str(Path.cwd()))
+    target = resolve_target(target_ref, request_field=request_field, response_field=response_field)
+    judge_config = _judge_config(judge_model, judge_base_url, judge_api_key_env)
+
+    results = run_preflight(target, judge_config)
+    for check_result in results:
+        icon = "[green]OK[/green]" if check_result.ok else "[red]FAIL[/red]"
+        console.print(f"{icon} {check_result.name}: {check_result.message}")
+
+    if not all(check_result.ok for check_result in results):
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

agentaudit/config.py

@@ -0,0 +1,24 @@
+import os
+from dataclasses import dataclass
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+DEFAULT_JUDGE_BASE_URL = "https://api.groq.com/openai/v1"
+DEFAULT_JUDGE_MODEL = "openai/gpt-oss-120b"
+DEFAULT_JUDGE_API_KEY_ENV = "GROQ_API_KEY"
+DEFAULT_SCORE_THRESHOLD = 80.0
+
+
+@dataclass(frozen=True)
+class JudgeConfig:
+    model: str = DEFAULT_JUDGE_MODEL
+    base_url: str = DEFAULT_JUDGE_BASE_URL
+    api_key_env: str = DEFAULT_JUDGE_API_KEY_ENV
+
+    def get_api_key(self) -> str:
+        api_key = os.environ.get(self.api_key_env)
+        if not api_key:
+            raise RuntimeError(f"{self.api_key_env} is not set. Add it to a .env file or export it in your shell.")
+        return api_key