evalkeep 0.1.0__py3-none-any.whl

evalkeep/commands/discover_cmd.py

@@ -0,0 +1,274 @@
+"""``evalkeep discover`` and ``evalkeep clusters`` -- group failures and edit groups.
+
+Editing a clustering is a first-class operation, not an escape hatch. A distance
+metric over short summaries will always split a family that a person can see is
+one, and merge two that a person can see are not; merge, split, rename and
+dismiss are how that judgement gets recorded.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from evalkeep.analysis import FailureAnalysis
+from evalkeep.cache import EmbeddingCache
+from evalkeep.clustering import ClusterInput, assign_roles, derive_label
+from evalkeep.clusters import Cluster, ClusterMember
+from evalkeep.commands.analyze_cmd import run_analysis
+from evalkeep.commands.detect_cmd import default_reviewer
+from evalkeep.config import Project
+from evalkeep.discovery import ClusterEditsWouldBeLost, DiscoveryReport, discover
+from evalkeep.embeddings import get_embedder
+from evalkeep.errors import CommandError
+from evalkeep.storage import TraceStore
+
+EDITS_WOULD_BE_LOST_HINT = "Re-run with --force to discard them, or leave the clustering as it is."
+
+
+@dataclass(frozen=True)
+class ClusterDetail:
+    """A cluster together with what each member is."""
+
+    cluster: Cluster
+    analyses: dict[str, FailureAnalysis]
+    trace_ids: dict[str, str]
+
+
+def run_discovery(
+    *,
+    project_root: Path = Path(),
+    analyze: bool = True,
+    force: bool = False,
+    use_cache: bool = True,
+    group_undescribed: bool = False,
+) -> DiscoveryReport:
+    """Analyze (when a provider is configured), embed, cluster and select."""
+    project = Project.load(project_root.expanduser().resolve())
+
+    if analyze and project.config.analyzer.provider != "manual":
+        run_analysis(project_root=project_root, use_cache=use_cache)
+
+    embedder = get_embedder(project.config.clustering)
+    cache = EmbeddingCache(project.subdir("cache"), enabled=use_cache)
+
+    with TraceStore.open(project.database_path) as store:
+        if store.failures.count() == 0:
+            raise CommandError(
+                "No failure candidates to group.", hint="Run 'evalkeep detect' first."
+            )
+        try:
+            report = discover(
+                store,
+                embedder,
+                cache,
+                project.config.clustering,
+                force=force,
+                group_undescribed=group_undescribed,
+            )
+        except ClusterEditsWouldBeLost as exc:
+            names = ", ".join(cluster.label for cluster in exc.clusters)
+            raise CommandError(
+                f"Re-clustering would discard reviewer edits on {len(exc.clusters)} "
+                f"cluster(s): {names}.",
+                hint=EDITS_WOULD_BE_LOST_HINT,
+            ) from exc
+
+        if report.clusters == 0:
+            raise CommandError(
+                "No analyzed failures to group.",
+                hint="Run 'evalkeep analyze', or label failures by hand with "
+                "'evalkeep failures label'.",
+            )
+        return report
+
+
+def list_clusters(*, project_root: Path = Path(), include_dismissed: bool = True) -> list[Cluster]:
+    project = Project.load(project_root.expanduser().resolve())
+    with TraceStore.open(project.database_path) as store:
+        return store.clusters.list(include_dismissed=include_dismissed)
+
+
+def show_cluster(cluster_id: str, *, project_root: Path = Path()) -> ClusterDetail:
+    project = Project.load(project_root.expanduser().resolve())
+    with TraceStore.open(project.database_path) as store:
+        cluster = _resolve(store, cluster_id)
+        return _detail(store, cluster)
+
+
+def rename_cluster(
+    cluster_id: str, label: str, *, project_root: Path = Path(), reviewer: str | None = None
+) -> Cluster:
+    cleaned = label.strip()
+    if not cleaned:
+        raise CommandError("A cluster label cannot be empty.")
+
+    project = Project.load(project_root.expanduser().resolve())
+    with TraceStore.open(project.database_path) as store:
+        cluster = _resolve(store, cluster_id)
+        cluster.label = cleaned
+        cluster.labelled_by = reviewer or default_reviewer()
+        store.clusters.save(cluster)
+        return cluster
+
+
+def dismiss_cluster(
+    cluster_id: str, *, project_root: Path = Path(), reviewer: str | None = None
+) -> Cluster:
+    """Mark a family as not worth regression coverage. Kept, not deleted."""
+    project = Project.load(project_root.expanduser().resolve())
+    with TraceStore.open(project.database_path) as store:
+        cluster = _resolve(store, cluster_id)
+        cluster.dismissed = True
+        cluster.labelled_by = cluster.labelled_by or reviewer or default_reviewer()
+        store.clusters.save(cluster)
+        return cluster
+
+
+def restore_cluster(cluster_id: str, *, project_root: Path = Path()) -> Cluster:
+    project = Project.load(project_root.expanduser().resolve())
+    with TraceStore.open(project.database_path) as store:
+        cluster = _resolve(store, cluster_id)
+        cluster.dismissed = False
+        store.clusters.save(cluster)
+        return cluster
+
+
+def merge_clusters(
+    cluster_ids: list[str], *, project_root: Path = Path(), reviewer: str | None = None
+) -> Cluster:
+    """Combine several families into one the reviewer says is really one."""
+    if len(cluster_ids) < 2:
+        raise CommandError("Merging needs at least two clusters.")
+
+    project = Project.load(project_root.expanduser().resolve())
+    with TraceStore.open(project.database_path) as store:
+        clusters = [_resolve(store, cluster_id) for cluster_id in cluster_ids]
+        if len({cluster.cluster_id for cluster in clusters}) != len(clusters):
+            raise CommandError("Each cluster can only be merged once.")
+
+        members = [member for cluster in clusters for member in cluster.members]
+        named = next((cluster for cluster in clusters if cluster.labelled_by), None)
+        merged = Cluster.build(
+            label=named.label if named else _combined_label(store, members),
+            members=_rerank(store, members),
+        )
+        # A merge is a judgement, so the result counts as reviewer-edited and
+        # will not be silently rebuilt away by the next `discover`.
+        merged.labelled_by = named.labelled_by if named else (reviewer or default_reviewer())
+
+        for cluster in clusters:
+            store.clusters.delete(cluster.cluster_id)
+        store.clusters.save(merged)
+        return merged
+
+
+def split_cluster(
+    cluster_id: str,
+    failure_ids: list[str],
+    *,
+    project_root: Path = Path(),
+    reviewer: str | None = None,
+) -> tuple[Cluster, Cluster]:
+    """Move some members out of a family into a new one of their own."""
+    if not failure_ids:
+        raise CommandError("Splitting needs at least one failure to move out.")
+
+    project = Project.load(project_root.expanduser().resolve())
+    who = reviewer or default_reviewer()
+    with TraceStore.open(project.database_path) as store:
+        cluster = _resolve(store, cluster_id)
+        wanted = {identifier.strip() for identifier in failure_ids}
+        moving = [member for member in cluster.members if member.failure_id in wanted]
+        staying = [member for member in cluster.members if member.failure_id not in wanted]
+
+        missing = wanted - {member.failure_id for member in moving}
+        if missing:
+            raise CommandError(
+                f"Not in cluster {cluster.cluster_id}: {', '.join(sorted(missing))}.",
+                hint="Run 'evalkeep clusters show <id>' to see its members.",
+            )
+        if not staying:
+            raise CommandError(
+                "Splitting out every member would leave the cluster empty.",
+                hint="Nothing to do: the cluster already contains exactly these failures.",
+            )
+
+        store.clusters.delete(cluster.cluster_id)
+        remainder = Cluster.build(
+            label=_combined_label(store, staying), members=_rerank(store, staying)
+        )
+        remainder.labelled_by = who
+        extracted = Cluster.build(
+            label=_combined_label(store, moving), members=_rerank(store, moving)
+        )
+        extracted.labelled_by = who
+        store.clusters.save(remainder)
+        store.clusters.save(extracted)
+        return remainder, extracted
+
+
+def _rerank(store: TraceStore, members: list[ClusterMember]) -> list[ClusterMember]:
+    """Re-mark representatives after an edit changed the membership.
+
+    Distances to the old centroid are kept: recomputing one would need the
+    vectors, and the ordering they induce is what the roles depend on. The roles
+    themselves are re-derived through the same function the algorithm uses, so
+    an edited cluster carries no stale marks and no missing ones.
+    """
+    ranked = sorted(members, key=lambda member: (member.distance, member.failure_id))
+    fresh = [
+        ClusterMember(failure_id=member.failure_id, distance=member.distance) for member in ranked
+    ]
+    severities = {}
+    for member in fresh:
+        analysis = store.failures.get_analysis(member.failure_id)
+        if analysis is not None:
+            severities[member.failure_id] = analysis.severity
+    assign_roles(fresh, severities)
+    return fresh
+
+
+def _combined_label(store: TraceStore, members: list[ClusterMember]) -> str:
+    inputs: list[ClusterInput] = []
+    for member in members:
+        analysis = store.failures.get_analysis(member.failure_id)
+        if analysis is not None:
+            inputs.append(ClusterInput.from_analysis(member.failure_id, analysis))
+    return derive_label(inputs) if inputs else "unlabelled"
+
+
+def _detail(store: TraceStore, cluster: Cluster) -> ClusterDetail:
+    analyses: dict[str, FailureAnalysis] = {}
+    trace_ids: dict[str, str] = {}
+    for member in cluster.members:
+        analysis = store.failures.get_analysis(member.failure_id)
+        if analysis is not None:
+            analyses[member.failure_id] = analysis
+        failure = store.failures.get(member.failure_id)
+        if failure is not None:
+            trace_ids[member.failure_id] = failure.trace_id
+    return ClusterDetail(cluster=cluster, analyses=analyses, trace_ids=trace_ids)
+
+
+def _resolve(store: TraceStore, identifier: str) -> Cluster:
+    """Accept a cluster ID, or the ID of any failure or trace inside it."""
+    cleaned = identifier.strip()
+    cluster = store.clusters.get(cleaned)
+    if cluster is not None:
+        return cluster
+
+    cluster = store.clusters.find_by_failure(cleaned)
+    if cluster is not None:
+        return cluster
+
+    failure = store.failures.get_by_trace(cleaned)
+    if failure is not None:
+        cluster = store.clusters.find_by_failure(failure.failure_id)
+        if cluster is not None:
+            return cluster
+
+    raise CommandError(
+        f"No cluster matching {cleaned!r}.",
+        hint="Run 'evalkeep clusters list', or 'evalkeep discover' first.",
+    )

evalkeep/commands/ingest_cmd.py

@@ -0,0 +1,50 @@
+"""``evalkeep ingest`` -- validate, redact, deduplicate and store traces."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from evalkeep.adapters import DEFAULT_ADAPTER, get_adapter
+from evalkeep.config import Project
+from evalkeep.errors import CommandError
+from evalkeep.ingest import DEFAULT_SAMPLE_LIMIT, IngestReport, ingest_file
+from evalkeep.redaction import Redactor
+from evalkeep.storage import TraceStore
+
+
+def ingest_traces(
+    path: Path,
+    *,
+    project_root: Path = Path(),
+    adapter_name: str = DEFAULT_ADAPTER,
+    validate_only: bool = False,
+    dry_run: bool = False,
+    error_path: Path | None = None,
+    sample_limit: int = DEFAULT_SAMPLE_LIMIT,
+) -> IngestReport:
+    """Run the ingest pipeline over ``path``."""
+    adapter = get_adapter(adapter_name)
+    if validate_only and dry_run:
+        raise CommandError(
+            "--validate-only and --dry-run cannot be combined.",
+            hint="--validate-only checks the file alone; --dry-run also checks it "
+            "against the stored traces.",
+        )
+
+    path = path.expanduser()
+    resolved_errors = error_path.expanduser() if error_path is not None else None
+
+    if validate_only:
+        return ingest_file(path, adapter, error_path=resolved_errors, sample_limit=sample_limit)
+
+    project = Project.load(project_root.expanduser().resolve())
+    with TraceStore.open(project.database_path) as store:
+        return ingest_file(
+            path,
+            adapter,
+            store=store,
+            redactor=Redactor(project.config.redaction, pseudonymizer=project.pseudonymizer()),
+            dry_run=dry_run,
+            error_path=resolved_errors,
+            sample_limit=sample_limit,
+        )

evalkeep/commands/init_cmd.py

@@ -0,0 +1,151 @@
+"""``evalkeep init`` -- create a safe, idempotent local project structure.
+
+Initialization never destroys existing work. Re-running it fills in whatever is
+missing and leaves everything else untouched, so it is safe to run in a
+half-configured project or as part of a setup script. ``--force`` is the only
+way to rewrite an existing configuration file, and even then nothing under the
+state directory is deleted.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field
+from enum import StrEnum
+from pathlib import Path
+
+from evalkeep.config import (
+    CONFIG_FILENAME,
+    GITIGNORE_ENTRIES,
+    GITIGNORE_HEADER,
+    STATE_SUBDIRS,
+    Project,
+    ProjectConfig,
+)
+from evalkeep.errors import CommandError
+
+
+class Action(StrEnum):
+    """What initialization did to one path."""
+
+    CREATED = "created"
+    EXISTS = "exists"
+    UPDATED = "updated"
+    OVERWRITTEN = "overwritten"
+
+
+@dataclass(frozen=True)
+class Step:
+    action: Action
+    path: Path
+    detail: str = ""
+
+
+@dataclass
+class InitReport:
+    project: Project
+    steps: list[Step] = field(default_factory=list)
+
+    @property
+    def changed(self) -> bool:
+        return any(step.action is not Action.EXISTS for step in self.steps)
+
+
+def initialize_project(
+    root: Path,
+    *,
+    project_name: str | None = None,
+    force: bool = False,
+) -> InitReport:
+    """Create or complete an Evalkeep project rooted at ``root``."""
+    root = root.expanduser().resolve()
+    _check_writable_directory(root)
+
+    config_path = root / CONFIG_FILENAME
+    steps: list[Step] = []
+
+    if config_path.is_file() and not force:
+        config = ProjectConfig.from_yaml(
+            config_path.read_text(encoding="utf-8"), source=config_path
+        )
+        if project_name is not None and project_name != config.project_name:
+            raise CommandError(
+                f"{config_path} already names this project "
+                f"{config.project_name!r}, not {project_name!r}.",
+                hint="Edit the file directly, or re-run with --force.",
+            )
+        steps.append(Step(Action.EXISTS, config_path, "left unchanged"))
+    else:
+        overwriting = config_path.is_file()
+        config = ProjectConfig(project_name=project_name or root.name)
+        _write_text(config_path, config.to_yaml())
+        steps.append(Step(Action.OVERWRITTEN if overwriting else Action.CREATED, config_path))
+
+    project = Project(root, config)
+    steps.extend(_ensure_state_dirs(project))
+    steps.append(_ensure_gitignore(root))
+    return InitReport(project=project, steps=steps)
+
+
+def _ensure_state_dirs(project: Project) -> list[Step]:
+    steps: list[Step] = []
+    state_dir = project.state_dir
+    steps.append(_ensure_dir(state_dir))
+    for name, purpose in STATE_SUBDIRS.items():
+        steps.append(_ensure_dir(project.subdir(name), detail=purpose))
+    # Git does not track empty directories; a placeholder keeps the layout
+    # intact for anyone who clones the repository.
+    keep = project.subdir("exports") / ".gitkeep"
+    if not keep.exists():
+        _write_text(keep, "")
+    return steps
+
+
+def _ensure_dir(path: Path, *, detail: str = "") -> Step:
+    if path.is_dir():
+        return Step(Action.EXISTS, path, detail)
+    if path.exists():
+        raise CommandError(f"{path} exists but is not a directory.")
+    try:
+        path.mkdir(parents=True)
+    except OSError as exc:
+        raise CommandError(f"Could not create {path}: {exc}") from exc
+    return Step(Action.CREATED, path, detail)
+
+
+def _ensure_gitignore(root: Path) -> Step:
+    """Add only the entries that are missing, preserving the existing file."""
+    path = root / ".gitignore"
+    existing_lines = path.read_text(encoding="utf-8").splitlines() if path.is_file() else []
+    present = {line.strip() for line in existing_lines}
+    missing = [entry for entry in GITIGNORE_ENTRIES if entry not in present]
+
+    if not missing:
+        return Step(Action.EXISTS, path, "all entries present")
+
+    block = [GITIGNORE_HEADER, *missing]
+    if existing_lines:
+        prefix = existing_lines + ([""] if existing_lines[-1].strip() else [])
+        action = Action.UPDATED
+    else:
+        prefix = []
+        action = Action.CREATED
+    _write_text(path, "\n".join([*prefix, *block]) + "\n")
+    return Step(action, path, f"{len(missing)} entr{'y' if len(missing) == 1 else 'ies'} added")
+
+
+def _check_writable_directory(root: Path) -> None:
+    if not root.exists():
+        raise CommandError(f"{root} does not exist.")
+    if not root.is_dir():
+        raise CommandError(f"{root} is not a directory.")
+    if not os.access(root, os.W_OK):
+        raise CommandError(f"{root} is not writable.")
+
+
+def _write_text(path: Path, text: str) -> None:
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(text, encoding="utf-8")
+    except OSError as exc:
+        raise CommandError(f"Could not write {path}: {exc}") from exc

evalkeep/commands/pipeline_cmd.py

@@ -0,0 +1,156 @@
+"""``evalkeep from-traces`` -- the whole pipeline up to the review gate.
+
+The stages exist because each is a real decision with its own evidence, and
+anyone tuning a suite will end up running them one at a time. But five commands
+and their options is a lot to understand before seeing whether the tool is worth
+anything, so this runs them in order and reports what came out.
+
+It deliberately stops at review. Everything before it is derived and can be
+re-run; approving a test is a judgement, and a command that quietly approved
+things on your behalf would defeat the point of the gate.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from evalkeep.adapters import DEFAULT_ADAPTER
+from evalkeep.commands.analyze_cmd import run_analysis
+from evalkeep.commands.dataset_cmd import build_dataset
+from evalkeep.commands.detect_cmd import run_detection
+from evalkeep.commands.discover_cmd import run_discovery
+from evalkeep.commands.ingest_cmd import ingest_traces
+from evalkeep.config import Project
+from evalkeep.discovery import CLUSTERABLE
+from evalkeep.errors import CommandError
+from evalkeep.regression import ReviewStatus
+from evalkeep.storage import TraceStore
+
+
+@dataclass
+class PipelineReport:
+    """What one end-to-end pass produced, in the order it happened."""
+
+    traces: int = 0
+    already_known: int = 0
+    invalid: int = 0
+    failures: int = 0
+    evidence: dict[str, int] = field(default_factory=dict)
+    analyzed: int = 0
+    described: bool = False
+    families: int = 0
+    representatives: int = 0
+    drafts: int = 0
+    ready: int = 0
+    needs_expectation: int = 0
+    pending_review: int = 0
+    notes: list[str] = field(default_factory=list)
+
+    @property
+    def found_nothing(self) -> bool:
+        return self.failures == 0
+
+
+def from_traces(
+    path: Path,
+    *,
+    project_root: Path = Path(),
+    adapter_name: str = DEFAULT_ADAPTER,
+    limit: int | None = None,
+) -> PipelineReport:
+    """Ingest a trace file and carry it as far as the review queue."""
+    project = Project.load(project_root.expanduser().resolve())
+    report = PipelineReport()
+
+    ingested = ingest_traces(path, project_root=project_root, adapter_name=adapter_name)
+    report.traces = ingested.stored
+    report.already_known = ingested.already_stored + ingested.content_duplicates
+    report.invalid = ingested.invalid
+    if ingested.identifier_risks:
+        report.notes.append(
+            f"{ingested.identifier_risks} trace(s) have identifiers that look like "
+            "they carry personal data; see redaction.pseudonymize_identifiers."
+        )
+
+    if report.traces == 0 and report.already_known == 0:
+        raise CommandError(
+            "No traces were stored, so there is nothing to work with.",
+            hint="Check the file, or pass --format if it is not Evalkeep's own JSONL.",
+        )
+
+    detected = run_detection(project_root=project_root)
+    report.failures = detected.failures
+    report.evidence = {kind.value: count for kind, count in detected.by_kind.items()}
+    if report.found_nothing:
+        return report
+
+    # Describing failures is what lets them be grouped by *what they are*. With
+    # no provider configured that is a person's job, so the run continues on
+    # observed behaviour and says so rather than stopping.
+    if project.config.analyzer.provider != "manual":
+        analysis = run_analysis(project_root=project_root)
+        report.analyzed = analysis.analyzed + analysis.from_cache + analysis.skipped
+        if analysis.failed:
+            # Configured is not the same as working. Saying which provider
+            # refused, and why, beats a later error about nothing to group.
+            report.notes.append(
+                f"{analysis.failed} failure(s) could not be described by "
+                f"{analysis.analyzer}: {analysis.errors[0][1] if analysis.errors else 'unknown'}"
+            )
+
+    # Ask the store, not the analyzer: descriptions also arrive from
+    # 'evalkeep failures label', and a run after that should not be told it has
+    # none. Anything still undescribed is grouped on observed behaviour rather
+    # than dropped, which is the only way a first run reaches the review queue.
+    undescribed = _undescribed(project)
+    report.described = undescribed == 0
+    if undescribed:
+        report.notes.append(
+            f"{undescribed} failure(s) were grouped by what was observed, not by what "
+            "they are. Describe them with 'evalkeep failures label', or set a working "
+            "analyzer.provider in evalkeep.yaml, and re-run for tighter families."
+        )
+
+    discovered = run_discovery(
+        project_root=project_root,
+        analyze=False,
+        force=True,
+        group_undescribed=undescribed > 0,
+    )
+    report.families = discovered.clusters
+    report.representatives = discovered.representatives
+
+    # Regenerate: everything up to the review gate is derived, so a second run
+    # after describing a failure should reflect it rather than leave the first
+    # run's draft in place. Reviewed tests are kept regardless -- rebuilding a
+    # draft is not the same as undoing a decision.
+    built = build_dataset(project_root=project_root, limit=limit, regenerate=True)
+    report.drafts = built.created + built.regenerated + built.skipped
+
+    with TraceStore.open(project.database_path) as store:
+        drafts = store.tests.list(status=ReviewStatus.DRAFT, limit=10_000)
+        report.pending_review = len(drafts)
+        # Enough evidence means the draft carries a check that fails when the
+        # bug returns -- which is what a regression test is for, and what
+        # drives the compare numbers even when all it does is forbid the
+        # action that was observed. `needs_expectation` then qualifies that
+        # same set rather than naming a separate one: those drafts cannot yet
+        # confirm the agent did the right thing instead, only that it avoided
+        # the one wrong thing. Reported as a remainder, it read as six
+        # findings from three failures.
+        report.ready = sum(1 for test in drafts if test.deterministic_expectations)
+        report.needs_expectation = sum(1 for test in drafts if not test.has_positive_expectation)
+
+    return report
+
+
+def _undescribed(project: Project) -> int:
+    """How many clusterable failures nobody has said anything about yet."""
+    with TraceStore.open(project.database_path) as store:
+        return sum(
+            1
+            for failure in store.failures.iter_all()
+            if failure.status in CLUSTERABLE
+            and store.failures.get_analysis(failure.failure_id) is None
+        )