ankora 0.1.0__py3-none-any.whl

ankora/__init__.py

@@ -0,0 +1,8 @@
+"""ankora: local-first, CI-native regression testing for LLM applications."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("ankora")
+except PackageNotFoundError:  # pragma: no cover - only when running from a non-installed tree
+    __version__ = "0.0.0"

ankora/cli.py

@@ -0,0 +1,291 @@
+"""The ankora command-line interface.
+
+Wires the typer app and its subcommands (``init``, ``ingest``, ``run``, ``diff``,
+``gate``, ``baseline set``). ``gate`` is the CI entrypoint: it exits non-zero
+when quality regresses against the baseline.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from ankora import __version__
+from ankora.config import (
+    DEFAULT_CONFIG_YAML,
+    Config,
+    ConfigError,
+    TargetConfig,
+    load_config,
+)
+from ankora.diff import DiffReport, diff_runs
+from ankora.ingest import ingest_traces
+from ankora.models import RunResult
+from ankora.replay import replay
+from ankora.storage import StorageError, get_baseline, load_run, run_path, set_baseline
+from ankora.suites import SuiteError
+
+app = typer.Typer(
+    name="ankora",
+    help="Local-first, CI-native regression testing for LLM applications.",
+    no_args_is_help=True,
+    add_completion=False,
+)
+
+console = Console()
+
+
+def _version_callback(value: bool) -> None:
+    if value:
+        console.print(__version__)
+        raise typer.Exit()
+
+
+@app.callback()
+def _main(
+    _version: bool = typer.Option(
+        False,
+        "--version",
+        callback=_version_callback,
+        is_eager=True,
+        help="Show the ankora version and exit.",
+    ),
+) -> None:
+    """ankora: turn the traces you already capture into CI regression tests."""
+
+
+@app.command()
+def version() -> None:
+    """Print the ankora version."""
+    console.print(__version__)
+
+
+@app.command()
+def init(
+    force: bool = typer.Option(False, "--force", help="Overwrite an existing ankora.yaml."),
+) -> None:
+    """Scaffold ankora.yaml and an evals/ directory in the current repo."""
+    config_path = Path("ankora.yaml")
+    if config_path.exists() and not force:
+        console.print(f"[red]{config_path} already exists.[/] Pass [bold]--force[/] to overwrite.")
+        raise typer.Exit(code=1)
+
+    config_path.write_text(DEFAULT_CONFIG_YAML, encoding="utf-8")
+    evals_dir = Path("evals")
+    evals_dir.mkdir(exist_ok=True)
+
+    console.print(f"[green]Wrote[/] {config_path}")
+    console.print(f"[green]Created[/] {evals_dir}/ (add Case files here)")
+    console.print("Next: [bold]ankora ingest <trace-file>[/] to build your first cases.")
+
+
+@app.command()
+def ingest(
+    trace_file: str = typer.Argument(
+        ..., help="Path to an OpenTelemetry GenAI or Langfuse trace export (JSON)."
+    ),
+    out: str = typer.Option("evals/", "--out", help="Directory to write Case files into."),
+    fmt: str = typer.Option(
+        "auto", "--format", help="Trace format: auto (default), otel, or langfuse."
+    ),
+) -> None:
+    """Build or update regression Cases from an OTel GenAI or Langfuse trace file."""
+    if fmt not in ("auto", "otel", "langfuse"):
+        console.print(f"[red]--format must be one of: auto, otel, langfuse[/] (got {fmt!r})")
+        raise typer.Exit(code=1)
+
+    trace_path = Path(trace_file)
+    if not trace_path.exists():
+        console.print(f"[red]Trace file not found:[/] {trace_path}")
+        raise typer.Exit(code=1)
+
+    try:
+        result, detected = ingest_traces(trace_path, fmt)
+    except json.JSONDecodeError as exc:
+        console.print(f"[red]{trace_path} is not valid JSON:[/] {exc}")
+        raise typer.Exit(code=1) from exc
+
+    out_dir = Path(out)
+    out_dir.mkdir(parents=True, exist_ok=True)
+    for case in result.cases:
+        (out_dir / f"{case.id}.yaml").write_text(case.to_yaml(), encoding="utf-8")
+
+    console.print(
+        f"[green]Wrote {len(result.cases)} case(s)[/] to {out_dir} "
+        f"({detected} format; {result.skipped} skipped, {result.total} record(s) seen)."
+    )
+
+
+@app.command()
+def run(
+    suite: list[str] = typer.Option(None, "--suite", help="Glob(s) of Case files to run."),
+    target: str = typer.Option(None, "--target", help="Override target as provider:model."),
+    concurrency: int = typer.Option(8, "--concurrency", help="Max cases to replay in parallel."),
+    config_path: str = typer.Option("ankora.yaml", "--config", help="Path to ankora.yaml."),
+) -> None:
+    """Replay and score the suite, persist a RunResult, print a summary."""
+    try:
+        config = load_config(config_path)
+        if suite:
+            config = config.model_copy(update={"suites": list(suite)})
+        result = replay(config, target=target, concurrency=concurrency)
+    except (ConfigError, SuiteError) as exc:
+        console.print(f"[red]{exc}[/]")
+        raise typer.Exit(code=1) from exc
+
+    _print_run_table(result)
+    console.print(f"\nSaved run [bold]{result.run_id}[/] to {run_path(result.run_id)}")
+
+
+def _print_run_table(result: RunResult) -> None:
+    table = Table(title=f"ankora run · {result.target.provider}:{result.target.model}")
+    table.add_column("Case", overflow="fold")
+    table.add_column("Result")
+    table.add_column("Scores", overflow="fold")
+    for case_result in result.case_results:
+        status = "[green]PASS[/]" if case_result.passed else "[red]FAIL[/]"
+        scores = ", ".join(
+            f"{s.scorer} {s.score:.2f}{'✓' if s.passed else '✗'}"
+            for s in case_result.scorer_results
+        )
+        table.add_row(case_result.case_id, status, scores or "—")
+    console.print(table)
+
+    summary = result.summary
+    color = "green" if summary.failed == 0 else "red"
+    console.print(f"[{color}]{summary.passed}/{summary.total} passed, {summary.failed} failed[/]")
+
+
+@app.command()
+def diff(
+    baseline: str = typer.Argument(..., help="Baseline run id or path to a run JSON."),
+    current: str = typer.Argument(..., help="Current run id or path to a run JSON."),
+    config_path: str = typer.Option("ankora.yaml", "--config", help="Path to ankora.yaml."),
+    fail_on: str = typer.Option(None, "--fail-on", help="Override gate.fail_on for this diff."),
+) -> None:
+    """Show per-case changes between two runs. Read-only; always exits 0."""
+    config = _load_config_or_default(config_path)
+    if fail_on:
+        gate_config = config.gate.model_copy(update={"fail_on": fail_on})
+        config = config.model_copy(update={"gate": gate_config})
+
+    try:
+        baseline_run = _resolve_run(baseline)
+        current_run = _resolve_run(current)
+    except StorageError as exc:
+        console.print(f"[red]{exc}[/]")
+        raise typer.Exit(code=1) from exc
+
+    _print_diff_report(diff_runs(baseline_run, current_run, config))
+
+
+@app.command()
+def gate(
+    config_path: str = typer.Option("ankora.yaml", "--config", help="Path to ankora.yaml."),
+    target: str = typer.Option(None, "--target", help="Override target as provider:model."),
+    concurrency: int = typer.Option(8, "--concurrency", help="Max cases to replay in parallel."),
+) -> None:
+    """Replay the suite, diff against baseline, and exit non-zero on regression.
+
+    This is the CI entrypoint.
+    """
+    try:
+        config = load_config(config_path)
+        current = replay(config, target=target, concurrency=concurrency)
+    except (ConfigError, SuiteError) as exc:
+        console.print(f"[red]{exc}[/]")
+        raise typer.Exit(code=1) from exc
+
+    _print_run_table(current)
+
+    try:
+        baseline = get_baseline(config)
+    except StorageError:
+        console.print(
+            "\n[yellow]No baseline yet[/] — nothing to regress against. "
+            f"Promote this run with [bold]ankora baseline set {current.run_id}[/]."
+        )
+        raise typer.Exit(code=0) from None
+
+    report = diff_runs(baseline, current, config)
+    _print_diff_report(report)
+    if report.has_regressions:
+        console.print(f"\n[red]{report.regressions} regression(s) detected — failing the gate.[/]")
+        raise typer.Exit(code=1)
+    console.print("\n[green]No regressions — gate passed.[/]")
+
+
+baseline_app = typer.Typer(
+    name="baseline",
+    help="Manage the regression baseline.",
+    no_args_is_help=True,
+)
+app.add_typer(baseline_app)
+
+
+@baseline_app.command("set")
+def baseline_set(
+    run_id: str = typer.Argument(..., help="The run id to promote to baseline."),
+    config_path: str = typer.Option("ankora.yaml", "--config", help="Path to ankora.yaml."),
+) -> None:
+    """Promote a stored run to the baseline."""
+    try:
+        config = load_config(config_path)
+        path = set_baseline(config, run_id)
+    except (ConfigError, StorageError) as exc:
+        console.print(f"[red]{exc}[/]")
+        raise typer.Exit(code=1) from exc
+    console.print(f"[green]Baseline set[/] to run [bold]{run_id}[/] at {path}")
+
+
+def _load_config_or_default(config_path: str) -> Config:
+    """Load config, or fall back to a minimal default (used by read-only diff)."""
+    try:
+        return load_config(config_path)
+    except ConfigError:
+        return Config(target=TargetConfig(provider="unknown", model="unknown"))
+
+
+def _resolve_run(reference: str) -> RunResult:
+    """Resolve a run reference that is either a path to a run JSON or a run id."""
+    path = Path(reference)
+    if path.exists():
+        return RunResult.from_json(path.read_text(encoding="utf-8"))
+    return load_run(reference)
+
+
+def _print_diff_report(report: DiffReport) -> None:
+    table = Table(title=f"ankora diff · fail_on={report.fail_on}")
+    table.add_column("Case", overflow="fold")
+    table.add_column("Status")
+    table.add_column("Baseline")
+    table.add_column("Current")
+    table.add_column("Δ")
+    table.add_column("Regression")
+    for case in report.cases:
+        table.add_row(
+            case.case_id,
+            case.status.value,
+            _fmt_score(case.baseline_score),
+            _fmt_score(case.current_score),
+            f"{case.delta:+.3f}" if case.delta is not None else "—",
+            "[red]yes[/]" if case.is_regression else "no",
+        )
+    console.print(table)
+    console.print(
+        f"regressed={report.regressed} new_failures={report.new_failures} "
+        f"fixed={report.fixed} unchanged={report.unchanged} "
+        f"new_passes={report.new_passes} removed={report.removed}"
+    )
+
+
+def _fmt_score(score: float | None) -> str:
+    return f"{score:.3f}" if score is not None else "—"
+
+
+if __name__ == "__main__":
+    app()

ankora/config.py

@@ -0,0 +1,228 @@
+"""Load and validate ankora.yaml into a pydantic Config.
+
+See CLAUDE.md "Config file" section for the target schema. Provider API keys are
+never stored on the model — they are read from the configured env var at call
+time via :meth:`Config.resolve_api_key`.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Annotated, Any, Literal
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field, ValidationError
+
+
+class ConfigError(Exception):
+    """Raised for any human-facing problem loading or validating a config.
+
+    Carries a message intended to be printed straight to a user, rather than a
+    raw pydantic error dump.
+    """
+
+
+class ProviderConfig(BaseModel):
+    """How to reach a provider (keys read from env, never inlined)."""
+
+    api_key_env: str
+
+
+class TargetConfig(BaseModel):
+    """The provider/model replays run against by default."""
+
+    provider: str
+    model: str
+
+
+class ModelRef(BaseModel):
+    """A provider/model reference used by scorers (judge, embedding model)."""
+
+    provider: str
+    model: str
+
+
+class LLMJudgeScorerConfig(BaseModel):
+    """Rubric-based judge scorer (``type: llm_judge``)."""
+
+    type: Literal["llm_judge"]
+    judge: ModelRef
+    rubric: str
+    threshold: float = 0.7
+
+
+class EmbeddingSimilarityScorerConfig(BaseModel):
+    """Cosine-similarity scorer over an embedding model (``type: embedding_similarity``)."""
+
+    type: Literal["embedding_similarity"]
+    model: ModelRef
+    threshold: float = 0.85
+
+
+class ExactScorerConfig(BaseModel):
+    """Exact-match scorer (``type: exact``).
+
+    ``normalize`` strips surrounding whitespace and lowercases both sides before
+    comparing.
+    """
+
+    type: Literal["exact"]
+    threshold: float = 1.0
+    normalize: bool = True
+
+
+class RegexScorerConfig(BaseModel):
+    """Regex-match scorer (``type: regex``)."""
+
+    type: Literal["regex"]
+    pattern: str
+    threshold: float = 1.0
+
+
+class JSONSchemaScorerConfig(BaseModel):
+    """JSON-schema validation scorer (``type: json_schema``).
+
+    The schema is provided under the ``schema`` key in YAML; it is exposed on
+    the model as ``json_schema`` to avoid shadowing pydantic internals.
+    """
+
+    model_config = ConfigDict(populate_by_name=True)
+
+    type: Literal["json_schema"]
+    json_schema: dict[str, Any] = Field(alias="schema")
+    threshold: float = 1.0
+
+
+# Discriminated on ``type`` so an unknown scorer type yields a clear error that
+# names the valid tags. Kept under the name ``ScorerConfig`` for callers that
+# import it (e.g. scorers.registry.build_scorer).
+ScorerConfig = Annotated[
+    LLMJudgeScorerConfig
+    | EmbeddingSimilarityScorerConfig
+    | ExactScorerConfig
+    | RegexScorerConfig
+    | JSONSchemaScorerConfig,
+    Field(discriminator="type"),
+]
+
+
+class GateConfig(BaseModel):
+    """How the gate decides pass/fail."""
+
+    fail_on: Literal["regression", "absolute"] = "regression"
+    baseline: str = ".ankora/baseline.json"
+
+
+class Config(BaseModel):
+    """The fully-parsed ankora.yaml."""
+
+    version: int = 1
+    suites: list[str] = Field(default_factory=lambda: ["evals/**/*.yaml"])
+    target: TargetConfig
+    providers: dict[str, ProviderConfig] = Field(default_factory=dict)
+    scorers: list[ScorerConfig] = Field(default_factory=list)
+    gate: GateConfig = Field(default_factory=GateConfig)
+
+    def resolve_api_key(self, provider: str) -> str:
+        """Read the API key for ``provider`` from its configured env var.
+
+        The key is read at call time and never stored on the model. Raises
+        :class:`ConfigError` if the provider is not configured or its env var is
+        unset/empty.
+        """
+        provider_config = self.providers.get(provider)
+        if provider_config is None:
+            configured = ", ".join(sorted(self.providers)) or "(none)"
+            raise ConfigError(
+                f"Provider {provider!r} is not configured under `providers`. "
+                f"Configured providers: {configured}."
+            )
+        key = os.environ.get(provider_config.api_key_env)
+        if not key:
+            raise ConfigError(
+                f"Environment variable {provider_config.api_key_env!r} "
+                f"(for provider {provider!r}) is not set. "
+                "Export it before running, e.g. "
+                f"`export {provider_config.api_key_env}=...`."
+            )
+        return key
+
+
+# The default config written by `ankora init`; mirrors the CLAUDE.md example.
+DEFAULT_CONFIG_YAML = """\
+version: 1
+suites: ["evals/**/*.yaml"]
+target:
+  provider: openai
+  model: gpt-4o-mini
+providers:
+  openai: {api_key_env: OPENAI_API_KEY}
+  anthropic: {api_key_env: ANTHROPIC_API_KEY}
+scorers:
+  - type: llm_judge
+    judge: {provider: openai, model: gpt-4o}
+    rubric: "Score 1 if the answer is factually consistent with the reference, else 0."
+    threshold: 0.7
+  - type: embedding_similarity
+    model: {provider: openai, model: text-embedding-3-small}
+    threshold: 0.85
+gate:
+  fail_on: regression
+  baseline: .ankora/baseline.json
+"""
+
+
+def _format_location(loc: tuple[Any, ...]) -> str:
+    """Render a pydantic error location as a readable dotted/indexed path."""
+    parts: list[str] = []
+    for item in loc:
+        if isinstance(item, int):
+            parts.append(f"[{item}]")
+        elif parts:
+            parts.append(f".{item}")
+        else:
+            parts.append(str(item))
+    return "".join(parts) or "(root)"
+
+
+def _humanize_validation_error(error: ValidationError, source: str) -> ConfigError:
+    """Turn a pydantic ValidationError into a clear, user-facing ConfigError."""
+    lines = [f"Invalid ankora config ({source}):"]
+    for err in error.errors():
+        location = _format_location(err["loc"])
+        lines.append(f"  - {location}: {err['msg']}")
+    return ConfigError("\n".join(lines))
+
+
+def load_config(path: str | Path = "ankora.yaml") -> Config:
+    """Read an ankora.yaml file and validate it into a :class:`Config`.
+
+    Raises :class:`ConfigError` with a human-readable message for a missing
+    file, malformed YAML, or a schema violation (unknown scorer ``type``,
+    missing ``api_key_env``, malformed ``gate.fail_on``, etc.).
+    """
+    config_path = Path(path)
+    try:
+        raw = config_path.read_text(encoding="utf-8")
+    except FileNotFoundError as exc:
+        raise ConfigError(
+            f"Config file not found: {config_path}. Run `ankora init` to create one."
+        ) from exc
+
+    try:
+        data = yaml.safe_load(raw)
+    except yaml.YAMLError as exc:
+        raise ConfigError(f"Config file {config_path} is not valid YAML: {exc}") from exc
+
+    if data is None:
+        raise ConfigError(f"Config file {config_path} is empty.")
+    if not isinstance(data, dict):
+        raise ConfigError(
+            f"Config file {config_path} must be a YAML mapping, got {type(data).__name__}."
+        )
+
+    try:
+        return Config.model_validate(data)
+    except ValidationError as exc:
+        raise _humanize_validation_error(exc, str(config_path)) from exc

ankora/diff.py

@@ -0,0 +1,158 @@
+"""Compare a current RunResult against a baseline to classify per-case changes.
+
+The gate uses this to decide whether quality regressed. Interpretation of what
+counts as a *regression* depends on ``config.gate.fail_on``:
+
+* ``"regression"`` — a case regresses only if it passed in the baseline and now
+  fails (crossing pass -> fail, i.e. its aggregate score dropped past the pass
+  threshold). Cases absent from the baseline are reported as new, never as
+  regressions.
+* ``"absolute"`` — any case failing its scorer thresholds in the current run is
+  a regression, regardless of the baseline.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from enum import StrEnum
+
+from pydantic import BaseModel, Field
+
+from ankora.config import Config
+from ankora.models import CaseResult, RunResult
+
+
+class CaseStatus(StrEnum):
+    """How a single case changed between baseline and current."""
+
+    NEW_FAILURE = "new_failure"  # only in current, failing
+    NEW_PASS = "new_pass"  # only in current, passing
+    FIXED = "fixed"  # failed baseline, passes now
+    REGRESSED = "regressed"  # passed baseline, fails now
+    UNCHANGED = "unchanged"  # same pass/fail state in both
+    REMOVED = "removed"  # only in baseline, missing from current
+
+
+class CaseDiff(BaseModel):
+    """The per-case comparison result."""
+
+    case_id: str
+    status: CaseStatus
+    baseline_passed: bool | None = None
+    current_passed: bool | None = None
+    baseline_score: float | None = None
+    current_score: float | None = None
+    delta: float | None = None
+    is_regression: bool = False
+    detail: str = ""
+
+
+class DiffReport(BaseModel):
+    """The full comparison plus roll-up counts and the gate signal."""
+
+    fail_on: str
+    cases: list[CaseDiff] = Field(default_factory=list)
+    has_regressions: bool = False
+    regressions: int = 0
+    regressed: int = 0
+    new_failures: int = 0
+    new_passes: int = 0
+    fixed: int = 0
+    unchanged: int = 0
+    removed: int = 0
+
+
+def diff_runs(baseline: RunResult, current: RunResult, config: Config) -> DiffReport:
+    """Compare ``current`` to ``baseline`` under ``config.gate.fail_on``."""
+    mode = config.gate.fail_on
+    baseline_by_id = {cr.case_id: cr for cr in baseline.case_results}
+    current_by_id = {cr.case_id: cr for cr in current.case_results}
+
+    cases: list[CaseDiff] = []
+    # Current run order first, so the report reads in suite order.
+    for current_case in current.case_results:
+        cases.append(_diff_case(baseline_by_id.get(current_case.case_id), current_case, mode))
+    # Then any baseline cases that vanished from the current run.
+    for baseline_case in baseline.case_results:
+        if baseline_case.case_id not in current_by_id:
+            cases.append(_removed_case(baseline_case))
+
+    counts = Counter(case.status for case in cases)
+    return DiffReport(
+        fail_on=mode,
+        cases=cases,
+        has_regressions=any(case.is_regression for case in cases),
+        regressions=sum(1 for case in cases if case.is_regression),
+        regressed=counts[CaseStatus.REGRESSED],
+        new_failures=counts[CaseStatus.NEW_FAILURE],
+        new_passes=counts[CaseStatus.NEW_PASS],
+        fixed=counts[CaseStatus.FIXED],
+        unchanged=counts[CaseStatus.UNCHANGED],
+        removed=counts[CaseStatus.REMOVED],
+    )
+
+
+def _diff_case(baseline: CaseResult | None, current: CaseResult, mode: str) -> CaseDiff:
+    current_passed = current.passed
+    current_score = _aggregate_score(current)
+
+    if baseline is None:
+        status = CaseStatus.NEW_FAILURE if not current_passed else CaseStatus.NEW_PASS
+        # A brand-new case is never a regression under "regression"; under
+        # "absolute" it regresses if it fails.
+        is_regression = mode == "absolute" and not current_passed
+        detail = "new case, failing" if not current_passed else "new case, passing"
+        return CaseDiff(
+            case_id=current.case_id,
+            status=status,
+            current_passed=current_passed,
+            current_score=current_score,
+            is_regression=is_regression,
+            detail=detail,
+        )
+
+    baseline_passed = baseline.passed
+    baseline_score = _aggregate_score(baseline)
+    delta = current_score - baseline_score
+
+    if baseline_passed and not current_passed:
+        status = CaseStatus.REGRESSED
+    elif not baseline_passed and current_passed:
+        status = CaseStatus.FIXED
+    else:
+        status = CaseStatus.UNCHANGED
+
+    if mode == "absolute":
+        is_regression = not current_passed
+    else:  # "regression": only a genuine pass -> fail crossing counts
+        is_regression = status == CaseStatus.REGRESSED
+
+    return CaseDiff(
+        case_id=current.case_id,
+        status=status,
+        baseline_passed=baseline_passed,
+        current_passed=current_passed,
+        baseline_score=baseline_score,
+        current_score=current_score,
+        delta=delta,
+        is_regression=is_regression,
+        detail=f"score {baseline_score:.3f} -> {current_score:.3f} (Δ{delta:+.3f})",
+    )
+
+
+def _removed_case(baseline: CaseResult) -> CaseDiff:
+    return CaseDiff(
+        case_id=baseline.case_id,
+        status=CaseStatus.REMOVED,
+        baseline_passed=baseline.passed,
+        baseline_score=_aggregate_score(baseline),
+        is_regression=False,
+        detail="case missing from current run",
+    )
+
+
+def _aggregate_score(case: CaseResult) -> float:
+    """Mean of the case's scorer scores; falls back to pass/fail as 1.0/0.0."""
+    if not case.scorer_results:
+        return 1.0 if case.passed else 0.0
+    return sum(result.score for result in case.scorer_results) / len(case.scorer_results)