evalkeep 0.1.0__py3-none-any.whl

evalkeep/commands/compare_cmd.py

@@ -0,0 +1,169 @@
+"""``evalkeep compare``, ``runs`` and ``baseline`` -- reading two runs together."""
+
+from __future__ import annotations
+
+import uuid
+from dataclasses import dataclass
+from pathlib import Path
+
+from evalkeep.commands.detect_cmd import default_reviewer
+from evalkeep.comparison import ComparisonReport, compare_results
+from evalkeep.config import Project
+from evalkeep.errors import CommandError
+from evalkeep.runs import BaselinePromotion, CaseResult, EvaluationRun, Outcome
+from evalkeep.storage import TraceStore
+from evalkeep.storage.runs import AmbiguousRun
+from evalkeep.targets import BASELINE, CANDIDATE
+
+SUITE_DRIFT_HINT = (
+    "The two runs covered different tests, so their pass rates are not "
+    "comparable. Re-run both against the current suite, or pass "
+    "--allow-suite-drift to compare only the tests they share."
+)
+
+
+@dataclass(frozen=True)
+class RunSummary:
+    run: EvaluationRun
+    counts: dict[Outcome, int]
+    is_baseline: bool = False
+
+
+def compare(
+    *,
+    project_root: Path = Path(),
+    baseline: str | None = None,
+    candidate: str | None = None,
+    allow_suite_drift: bool = False,
+) -> ComparisonReport:
+    """Compare two runs, refusing to compare incompatible suites."""
+    project = Project.load(project_root.expanduser().resolve())
+    with TraceStore.open(project.database_path) as store:
+        baseline_run = _resolve_run(store, baseline, default=BASELINE, role="baseline")
+        candidate_run = _resolve_run(store, candidate, default=CANDIDATE, role="candidate")
+
+        if baseline_run.run_id == candidate_run.run_id:
+            raise CommandError(
+                "The baseline and the candidate are the same run.",
+                hint="Pass --baseline and --candidate explicitly.",
+            )
+
+        report = compare_results(
+            baseline_run,
+            store.runs.results(baseline_run.run_id),
+            candidate_run,
+            store.runs.results(candidate_run.run_id),
+        )
+
+    if not report.suite_compatible and not allow_suite_drift:
+        raise CommandError(
+            f"These runs used different test suites "
+            f"({baseline_run.suite_hash} vs {candidate_run.suite_hash}).",
+            hint=SUITE_DRIFT_HINT,
+        )
+    return report
+
+
+def list_runs(*, project_root: Path = Path(), limit: int = 20) -> list[RunSummary]:
+    project = Project.load(project_root.expanduser().resolve())
+    with TraceStore.open(project.database_path) as store:
+        promotion = store.runs.current_baseline()
+        return [
+            RunSummary(
+                run=run,
+                counts=store.runs.counts(run.run_id),
+                is_baseline=promotion is not None and promotion.run_id == run.run_id,
+            )
+            for run in store.runs.recent(limit=limit)
+        ]
+
+
+def show_run(run_id: str, *, project_root: Path = Path()) -> tuple[EvaluationRun, list[CaseResult]]:
+    project = Project.load(project_root.expanduser().resolve())
+    with TraceStore.open(project.database_path) as store:
+        run = _lookup(store, run_id)
+        if run is None:
+            raise CommandError(
+                f"No run with ID {run_id.strip()!r}.",
+                hint="Run 'evalkeep runs list' to see what exists.",
+            )
+        return run, store.runs.results(run.run_id)
+
+
+def promote_baseline(
+    run_id: str,
+    *,
+    project_root: Path = Path(),
+    reviewer: str | None = None,
+    reason: str | None = None,
+) -> BaselinePromotion:
+    """Make a run the reference point. Only ever an explicit, recorded decision."""
+    project = Project.load(project_root.expanduser().resolve())
+    with TraceStore.open(project.database_path) as store:
+        run = _lookup(store, run_id)
+        if run is None:
+            raise CommandError(
+                f"No run with ID {run_id.strip()!r}.",
+                hint="Run 'evalkeep runs list' to see what exists.",
+            )
+        errored = store.runs.counts(run.run_id).get(Outcome.ERROR, 0)
+        if errored:
+            raise CommandError(
+                f"That run had {errored} test(s) that never executed, so it is not "
+                "a sound reference point.",
+                hint="Fix the target and re-run before promoting.",
+            )
+        promotion = BaselinePromotion(
+            promotion_id=uuid.uuid4().hex,
+            run_id=run.run_id,
+            target_id=run.target_id,
+            reviewer=reviewer or default_reviewer(),
+            reason=reason,
+        )
+        store.runs.promote(promotion)
+        return promotion
+
+
+def current_baseline(
+    *, project_root: Path = Path()
+) -> tuple[BaselinePromotion, EvaluationRun] | None:
+    project = Project.load(project_root.expanduser().resolve())
+    with TraceStore.open(project.database_path) as store:
+        promotion = store.runs.current_baseline()
+        if promotion is None:
+            return None
+        run = store.runs.get(promotion.run_id)
+        return (promotion, run) if run is not None else None
+
+
+def _lookup(store: TraceStore, identifier: str) -> EvaluationRun | None:
+    try:
+        return store.runs.resolve(identifier)
+    except AmbiguousRun as exc:
+        raise CommandError(str(exc), hint="Use more characters of the run ID.") from exc
+
+
+def _resolve_run(
+    store: TraceStore, identifier: str | None, *, default: str, role: str
+) -> EvaluationRun:
+    """Accept a run ID, a target name, or nothing at all.
+
+    With nothing given, the baseline is whichever run was explicitly promoted --
+    falling back to the newest run for the conventional target name only when no
+    promotion has ever been recorded.
+    """
+    if identifier is None and role == "baseline":
+        promotion = store.runs.current_baseline()
+        if promotion is not None:
+            run = store.runs.get(promotion.run_id)
+            if run is not None:
+                return run
+
+    name = (identifier or default).strip()
+    run = _lookup(store, name) or store.runs.latest(name)
+    if run is None:
+        raise CommandError(
+            f"No {role} run matching {name!r}.",
+            hint="Run 'evalkeep runs list', or 'evalkeep run --target ...' first.",
+        )
+    return run

evalkeep/commands/dataset_cmd.py

@@ -0,0 +1,182 @@
+"""``evalkeep dataset`` -- generate and inspect regression-test drafts.
+
+By default only cluster *representatives* get a test. That is the entire point
+of clustering: a suite wants one good test per failure family, not forty copies
+of the same bug. ``--all`` overrides it when you want coverage of every failure.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from evalkeep.clusters import Cluster
+from evalkeep.config import Project
+from evalkeep.errors import CommandError
+from evalkeep.failures import Failure, FailureStatus
+from evalkeep.generation import GENERATOR_VERSION, build_test
+from evalkeep.regression import RegressionTest, ReviewStatus
+from evalkeep.storage import TraceStore
+
+
+@dataclass
+class BuildReport:
+    """What one ``dataset build`` did."""
+
+    considered: int = 0
+    created: int = 0
+    regenerated: int = 0
+    skipped: int = 0
+    reviewed_kept: int = 0
+    unanalyzed: int = 0
+    needs_expectation: int = 0
+    contradictions: int = 0
+    generator_version: int = GENERATOR_VERSION
+    warnings: list[tuple[str, str]] = field(default_factory=list)
+
+    @property
+    def changed(self) -> bool:
+        return bool(self.created or self.regenerated)
+
+
+@dataclass(frozen=True)
+class DatasetListing:
+    tests: list[RegressionTest]
+    total: int
+    counts: dict[ReviewStatus, int]
+
+
+def build_dataset(
+    *,
+    project_root: Path = Path(),
+    representatives_only: bool = True,
+    regenerate: bool = False,
+    limit: int | None = None,
+) -> BuildReport:
+    """Generate pending drafts for the failures worth covering."""
+    project = Project.load(project_root.expanduser().resolve())
+    report = BuildReport()
+
+    with TraceStore.open(project.database_path) as store:
+        clusters = store.clusters.list(include_dismissed=False)
+        if representatives_only and not clusters:
+            raise CommandError(
+                "No clusters to draw representatives from.",
+                hint="Run 'evalkeep discover' first, or pass --all to cover every failure.",
+            )
+
+        targets = _targets(store, clusters, representatives_only=representatives_only)
+        if not targets:
+            raise CommandError(
+                "Nothing to generate tests from.",
+                hint="Run 'evalkeep detect' and 'evalkeep analyze' first.",
+            )
+
+        for failure, cluster, roles in targets:
+            if limit is not None and report.created + report.regenerated >= limit:
+                break
+            report.considered += 1
+
+            existing = store.tests.get_by_failure(failure.failure_id)
+            if existing is not None and not regenerate:
+                report.skipped += 1
+                continue
+            if existing is not None and existing.reviewed:
+                # Regeneration rewrites a draft; it does not undo a review.
+                report.reviewed_kept += 1
+                continue
+
+            analysis = store.failures.get_analysis(failure.failure_id)
+            if analysis is None:
+                # Generated anyway: detection found evidence, and the trace
+                # still shows what the agent did. The draft says plainly that
+                # nobody has diagnosed it.
+                report.unanalyzed += 1
+            stored = store.get(failure.trace_id)
+            if stored is None:  # pragma: no cover - the foreign key prevents this
+                continue
+
+            test = build_test(
+                stored.trace,
+                failure,
+                analysis,
+                cluster_id=cluster.cluster_id if cluster else None,
+                cluster_label=cluster.label if cluster else None,
+                representative_roles=roles,
+            )
+            if existing is not None:
+                test.created_at = existing.created_at
+                report.regenerated += 1
+            else:
+                report.created += 1
+
+            if not test.has_positive_expectation:
+                report.needs_expectation += 1
+            report.contradictions += len(test.contradictions)
+            report.warnings.extend((test.test_id, warning) for warning in test.warnings)
+
+            store.tests.save(test)
+
+    return report
+
+
+def list_tests(
+    *,
+    project_root: Path = Path(),
+    status: ReviewStatus | None = None,
+    limit: int = 50,
+    offset: int = 0,
+) -> DatasetListing:
+    project = Project.load(project_root.expanduser().resolve())
+    with TraceStore.open(project.database_path) as store:
+        return DatasetListing(
+            tests=store.tests.list(status=status, limit=limit, offset=offset),
+            total=store.tests.count(status=status),
+            counts=store.tests.counts_by_status(),
+        )
+
+
+def show_test(identifier: str, *, project_root: Path = Path()) -> RegressionTest:
+    """Look one up by test ID, failure ID or the trace it came from."""
+    project = Project.load(project_root.expanduser().resolve())
+    with TraceStore.open(project.database_path) as store:
+        cleaned = identifier.strip()
+        test = store.tests.get(cleaned) or store.tests.get_by_failure(cleaned)
+        if test is None:
+            failure = store.failures.get_by_trace(cleaned)
+            if failure is not None:
+                test = store.tests.get_by_failure(failure.failure_id)
+        if test is None:
+            raise CommandError(
+                f"No regression test matching {cleaned!r}.",
+                hint="Run 'evalkeep dataset list', or 'evalkeep dataset build' first.",
+            )
+        return test
+
+
+def _targets(
+    store: TraceStore, clusters: list[Cluster], *, representatives_only: bool
+) -> list[tuple[Failure, Cluster | None, list[str]]]:
+    """Which failures get a test, and what cluster context to record with each."""
+    context: dict[str, tuple[Cluster, list[str]]] = {}
+    for cluster in clusters:
+        for member in cluster.members:
+            context[member.failure_id] = (
+                cluster,
+                [role.value for role in member.roles],
+            )
+
+    targets: list[tuple[Failure, Cluster | None, list[str]]] = []
+    for failure in store.failures.iter_all():
+        if failure.status is FailureStatus.DISMISSED:
+            continue
+        entry = context.get(failure.failure_id)
+        if representatives_only:
+            if entry is None or not entry[1]:
+                continue
+            targets.append((failure, entry[0], entry[1]))
+        elif entry is None:
+            targets.append((failure, None, []))
+        else:
+            targets.append((failure, entry[0], entry[1]))
+    return targets

evalkeep/commands/detect_cmd.py

@@ -0,0 +1,154 @@
+"""``evalkeep detect`` and ``evalkeep failures`` -- evidence and review."""
+
+from __future__ import annotations
+
+import getpass
+from dataclasses import dataclass
+from pathlib import Path
+
+from evalkeep.analysis import FailureAnalysis
+from evalkeep.config import Project
+from evalkeep.detection import DetectionReport, detect_failures
+from evalkeep.errors import CommandError
+from evalkeep.failures import Failure, FailureStatus, failure_id_for
+from evalkeep.storage import FailureSummary, StoredTrace, TraceStore
+
+
+@dataclass(frozen=True)
+class FailureListing:
+    summaries: list[FailureSummary]
+    total: int
+    counts: dict[FailureStatus, int]
+
+
+@dataclass(frozen=True)
+class FailureDetail:
+    """A failure together with the trace it describes and how it was labelled."""
+
+    failure: Failure
+    trace: StoredTrace
+    analysis: FailureAnalysis | None = None
+
+
+def default_reviewer() -> str:
+    """Who to record as the reviewer when the caller did not say."""
+    try:
+        return getpass.getuser()
+    except Exception:  # pragma: no cover - getuser can fail on odd systems
+        return "unknown"
+
+
+def run_detection(*, project_root: Path = Path()) -> DetectionReport:
+    project = Project.load(project_root.expanduser().resolve())
+    with TraceStore.open(project.database_path) as store:
+        if store.count() == 0:
+            raise CommandError(
+                "No traces have been ingested yet.",
+                hint="Run 'evalkeep ingest traces.jsonl' first.",
+            )
+        return detect_failures(store)
+
+
+def list_failures(
+    *,
+    project_root: Path = Path(),
+    status: FailureStatus | None = None,
+    limit: int = 50,
+    offset: int = 0,
+) -> FailureListing:
+    project = Project.load(project_root.expanduser().resolve())
+    with TraceStore.open(project.database_path) as store:
+        return FailureListing(
+            summaries=store.failures.list(status=status, limit=limit, offset=offset),
+            total=store.failures.count(status=status),
+            counts=store.failures.counts_by_status(),
+        )
+
+
+def show_failure(identifier: str, *, project_root: Path = Path()) -> FailureDetail:
+    """Look one up by failure ID or by the trace ID it belongs to."""
+    project = Project.load(project_root.expanduser().resolve())
+    with TraceStore.open(project.database_path) as store:
+        failure = resolve_failure(store, identifier, project=project)
+        stored = store.get(failure.trace_id)
+        if stored is None:  # pragma: no cover - the foreign key prevents this
+            raise CommandError(f"Trace {failure.trace_id!r} is missing from the store.")
+        return FailureDetail(
+            failure=failure,
+            trace=stored,
+            analysis=store.failures.get_analysis(failure.failure_id),
+        )
+
+
+def review_failure(
+    identifier: str,
+    status: FailureStatus,
+    *,
+    project_root: Path = Path(),
+    reviewer: str | None = None,
+    reason: str | None = None,
+) -> Failure:
+    """Record a human decision on an existing failure."""
+    project = Project.load(project_root.expanduser().resolve())
+    with TraceStore.open(project.database_path) as store:
+        failure = resolve_failure(store, identifier, project=project)
+        failure.review(status, reviewer=reviewer or default_reviewer(), reason=reason)
+        store.failures.save(failure)
+        return failure
+
+
+def add_failure(
+    trace_id: str,
+    *,
+    project_root: Path = Path(),
+    reviewer: str | None = None,
+    reason: str | None = None,
+) -> Failure:
+    """Mark a trace as a failure by hand, with no detector evidence."""
+    project = Project.load(project_root.expanduser().resolve())
+    with TraceStore.open(project.database_path) as store:
+        stored_id = next((c for c in project.identify(trace_id) if store.get(c) is not None), None)
+        if stored_id is None:
+            raise CommandError(
+                f"No stored trace with ID {trace_id.strip()!r}.",
+                hint="Run 'evalkeep trace list' to see what has been ingested.",
+            )
+        existing = store.failures.get_by_trace(stored_id)
+        if existing is not None:
+            raise CommandError(
+                f"Trace {trace_id.strip()!r} already has failure {existing.failure_id} "
+                f"({existing.status.value}).",
+                hint=f"Use 'evalkeep failures confirm {existing.failure_id}' instead.",
+            )
+        failure = Failure.manual(stored_id, reviewer=reviewer or default_reviewer(), reason=reason)
+        store.failures.save(failure)
+        return failure
+
+
+def resolve_failure(
+    store: TraceStore, identifier: str, *, project: Project | None = None
+) -> Failure:
+    """Accept a failure ID, or the trace ID it was derived from.
+
+    With pseudonymization on, the trace ID someone types may be the original
+    rather than the stored token, so every candidate spelling is tried.
+    """
+    cleaned = identifier.strip()
+    candidates = project.identify(cleaned) if project is not None else [cleaned]
+
+    failure = store.failures.get(cleaned)
+    for candidate in candidates:
+        if failure is not None:
+            break
+        failure = store.failures.get_by_trace(candidate)
+    for candidate in candidates:
+        if failure is not None:
+            break
+        if not candidate.startswith("fail-"):
+            failure = store.failures.get(failure_id_for(candidate))
+    if failure is None:
+        raise CommandError(
+            f"No failure matching {cleaned!r}.",
+            hint="Run 'evalkeep failures list', or 'evalkeep detect' if you have not detected yet.",
+        )
+    return failure