circa-ci 0.2.2__py3-none-any.whl

circa/analyzer.py

@@ -0,0 +1,405 @@
+"""Deterministic local CI failure analysis."""
+
+from __future__ import annotations
+
+import re
+from collections import defaultdict
+from collections.abc import Iterable
+from dataclasses import dataclass
+from pathlib import Path
+
+from .fingerprint import failure_fingerprint, normalize_environment
+from .knowledge import find_similar_incidents, incident_tokens, load_incidents, tokenize
+from .models import AnalysisResult, CauseCategory, CIFailure, HistoricalIncident
+from .resources import data_path
+
+_OWNERS = {
+    "hardware": "hardware-ops",
+    "network": "network-ops",
+    "infrastructure": "ci-ops",
+    "software": "developer",
+    "unknown": "unknown",
+}
+_CATEGORY_KEYWORDS: dict[CauseCategory, set[str]] = {
+    "hardware": {"disk", "memory", "oom", "filesystem", "cpu"},
+    "software": {"assertion", "assertionerror", "modulenotfounderror", "import", "regression", "traceback"},
+    "network": {"timeout", "dns", "resolve", "connection", "hostname", "registry"},
+    "infrastructure": {"runner", "image", "provisioning", "artifact", "503", "unavailable"},
+    "unknown": set(),
+}
+
+
+# Words in nearly every CI or pytest log; sharing them with an incident is not evidence.
+_GENERIC_TOKENS = frozenset(
+    {"test", "tests", "process", "python", "module", "run", "job", "step", "exit", "code", "completed", "passed"}
+)
+
+
+@dataclass(frozen=True)
+class _ExplicitError:
+    """An error that names its own cause, such as an exception type in a traceback."""
+
+    kind: str
+    pattern: re.Pattern[str]
+    # Error names, never ordinary words: an incident naming one describes this family of error.
+    # ModuleNotFoundError is a subclass of ImportError, and pytest reports both, so they share a family.
+    incident_tokens: frozenset[str]
+    # The subset whose incidents may explain this error: the same error or a more general one,
+    # never a more specific one (a ModuleNotFoundError incident does not explain "cannot import name").
+    cause_tokens: frozenset[str]
+    root_cause: str
+    recommendation: str
+    # Where and what was reported (file, line, message), appended to the root cause when found.
+    details: tuple[re.Pattern[str], ...] = ()
+    # Some tools print their failing summary at an informational level (Prettier uses "[warn]").
+    any_level: bool = False
+
+
+# Ordered by pipeline stage: a dependency or import problem stops the code before
+# any test runs, so it outranks an assertion reported in the same log.
+_EXPLICIT_ERRORS = (
+    _ExplicitError(
+        "dependency installation failure",
+        re.compile(
+            r"No matching distribution found for|Could not find a version that satisfies the requirement"
+            r"|ResolutionImpossible|npm ERR! code (?:ERESOLVE|ETARGET|E404)"
+        ),
+        frozenset({"resolutionimpossible", "eresolve", "etarget"}),
+        frozenset({"resolutionimpossible", "eresolve", "etarget"}),
+        "Dependency installation failed: the package manager could not install a required package or version.",
+        "Check the requirement's name, version pin, and package index, and reproduce the install locally.",
+    ),
+    _ExplicitError(
+        "compiler error",
+        re.compile(r"\S+\.(?:c|cc|cpp|cxx|h|hh|hpp|m|mm):\d+:\d+: (?:fatal )?error: "),
+        frozenset({"gcc", "clang"}),
+        frozenset({"gcc", "clang"}),
+        "Compilation failed: the compiler reported an error in the source.",
+        "Fix the first reported compiler error; later errors are often consequences of it.",
+        (re.compile(r"\S+\.(?:c|cc|cpp|cxx|h|hh|hpp|m|mm):\d+:\d+: (?:fatal )?error: [^\n]*"),),
+    ),
+    _ExplicitError(
+        "syntax error (SyntaxError)",
+        re.compile(r"(?<!except )\b(?:SyntaxError|IndentationError|TabError): "),
+        frozenset({"syntaxerror", "indentationerror", "taberror"}),
+        frozenset({"syntaxerror", "indentationerror", "taberror"}),
+        "The code could not be parsed (SyntaxError).",
+        "Fix the syntax at the reported file and line, and run the code or its tests locally.",
+        # The last traceback frame is the file that failed to parse.
+        (
+            re.compile(r'File "[^"\n]+", line \d+(?![\s\S]*File ")'),
+            re.compile(r"\b(?:SyntaxError|IndentationError|TabError): [^\n]*"),
+        ),
+    ),
+    _ExplicitError(
+        "missing module (ModuleNotFoundError)",
+        re.compile(r"(?<!except )\bModuleNotFoundError(?::| while)|\bNo module named ['\"]"),
+        frozenset({"modulenotfounderror", "importerror"}),
+        frozenset({"modulenotfounderror", "importerror"}),
+        "A required module could not be imported (ModuleNotFoundError): it is missing from the environment"
+        " or the module path is wrong.",
+        "Declare and install the missing package in the test environment, or fix the module path.",
+    ),
+    _ExplicitError(
+        "import failure (ImportError)",
+        re.compile(r"(?<!except )\bImportError(?::| while)|\bcannot import name ['\"]"),
+        frozenset({"importerror", "modulenotfounderror"}),
+        frozenset({"importerror"}),
+        "An import failed (ImportError): a module, or a name inside it, could not be imported.",
+        "Check the installed version of the module and the imported name, and look for circular imports.",
+    ),
+    _ExplicitError(
+        "test assertion failure (AssertionError)",
+        re.compile(r"\bAssertionError\b|(?:^|\s)E\s+assert\s|\bFAILED \S+::\S+ - assert\b|\bDID NOT RAISE\b"),
+        frozenset({"assertion", "assertionerror"}),
+        frozenset({"assertion", "assertionerror"}),
+        "A test assertion failed (AssertionError): the code under test did not produce the expected result.",
+        "Inspect the first failing test's assertion and the change under test, and reproduce it locally.",
+    ),
+    # Static checks: summaries and error signatures that only these tools print, never their names alone.
+    _ExplicitError(
+        "type-check failure (mypy)",
+        re.compile(
+            r"\bFound [1-9]\d* errors? in \d+ files? \(checked \d+ source files?\)"
+            r"|\S+\.pyi?:\d+: error: .*\[[a-z][a-z-]*\]\s*$"
+        ),
+        frozenset({"mypy"}),
+        frozenset({"mypy"}),
+        "Type checking failed (mypy): the code has type errors.",
+        "Fix the reported type errors and rerun mypy locally.",
+        (re.compile(r"\S+\.pyi?:\d+: error: [^\n]*"),),
+    ),
+    _ExplicitError(
+        "lint failure",
+        re.compile(r"\bFound [1-9]\d* errors?\.(?:\s|$)|\[\*\] \d+ fixable with\b"),
+        frozenset({"ruff"}),
+        frozenset({"ruff"}),
+        "The linter reported rule violations.",
+        "Fix the reported violations, or apply the linter's safe fixes, and rerun it locally.",
+        (re.compile(r"\S+\.\w+:\d+:\d+: [A-Z]+\d+ [^\n]*"),),
+    ),
+    _ExplicitError(
+        "formatting check failure",
+        re.compile(r"\b[1-9]\d* files? would be reformatted\b|\b[Ww]ould reformat:? \S|\bCode style issues found in\b"),
+        frozenset({"prettier", "ruff"}),
+        frozenset({"prettier", "ruff"}),
+        "A formatting check failed: files are not formatted as the project's formatter requires.",
+        "Run the project's formatter (for example `ruff format` or `prettier --write`) and commit the result.",
+        (re.compile(r"\b[1-9]\d* files? would be reformatted[^\n]*|\bCode style issues found in[^\n]*"),),
+        any_level=True,
+    ),
+    _ExplicitError(
+        "security scan finding (Bandit)",
+        re.compile(r">> Issue: \[B\d{3}:\w+\]"),
+        frozenset({"bandit"}),
+        frozenset({"bandit"}),
+        "The security scanner (Bandit) reported findings that fail the build.",
+        "Review each reported issue: fix it, or add a narrow, justified suppression.",
+        (re.compile(r"\[B\d{3}:\w+\][^\n]*"), re.compile(r"Location: \S+:\d+:\d+")),
+    ),
+)
+# Pipeline stages a historical root cause can place a failure in. A root cause claiming a stage the
+# log does not show (dependency installation, for a Git connection timeout) is not reused.
+_STAGE_TOKENS = {
+    "dependency installation": frozenset(
+        {"dependency", "dependencies", "install", "installation", "requirement", "requirements"}
+    ),
+}
+# An explicit error counts like three matching keywords toward confidence.
+_EXPLICIT_ERROR_WEIGHT = 3.0
+_ALL_ERROR_TOKENS = frozenset().union(*(error.incident_tokens for error in _EXPLICIT_ERRORS))
+# Explicit environment errors often cause a software error downstream (a full disk fails a test,
+# a registry timeout fails an install), so when one is present the explicit software error is
+# treated as a symptom and the general evidence decides.
+_ENVIRONMENT_ERROR = re.compile(
+    r"no space left on device|out of memory|\boom(?:killed)?\b|\bMemoryError\b|timed out|\btimeout\b"
+    r"|could not resolve host|name resolution|name or service not known|connection (?:refused|reset)"
+    r"|\b503\b|service unavailable",
+    re.IGNORECASE,
+)
+
+
+# Informational log levels report conditions, not failures (for example an optional import that is skipped).
+_NON_FAILURE_LEVEL = re.compile(
+    r"^(?:[^\t]*\t[^\t]*\t)?\ufeff?(?:\S*\d{2}:\d{2}:\d{2}\S*\s+)?(?:##)?\[?(?:WARNING|WARN|INFO|DEBUG|NOTICE)\b",
+    re.IGNORECASE,
+)
+_TEST_FAILURE = next(error for error in _EXPLICIT_ERRORS if error.kind.startswith("test assertion"))
+_DETAIL_LIMIT = 160
+
+
+def _explicit_error(failure_text: str) -> _ExplicitError | None:
+    """Return the earliest-stage explicit error actually reported, judging each log line on its own.
+
+    A test-failure line counts only as a test failure: an exception named in its message (``expected
+    ImportError``, ``DID NOT RAISE ModuleNotFoundError``) is what the test checked, not what was raised.
+    """
+
+    found: set[str] = set()
+    for line in failure_text.splitlines():
+        if _NON_FAILURE_LEVEL.match(line):
+            found.update(error.kind for error in _EXPLICIT_ERRORS if error.any_level and error.pattern.search(line))
+            continue
+        if _TEST_FAILURE.pattern.search(line):
+            found.add(_TEST_FAILURE.kind)
+            continue
+        error = next((error for error in _EXPLICIT_ERRORS if error.pattern.search(line)), None)
+        if error is not None:
+            found.add(error.kind)
+    return next((error for error in _EXPLICIT_ERRORS if error.kind in found), None)
+
+
+def _reported_details(failure_text: str, error: _ExplicitError) -> str:
+    """The first file, line, and message the tool reported, as one clipped string ("" if none)."""
+
+    found = (pattern.search(failure_text) for pattern in error.details)
+    return "; ".join(match.group(0).strip()[:_DETAIL_LIMIT] for match in found if match)
+
+
+def _contradicts(incident: HistoricalIncident, error: _ExplicitError) -> bool:
+    """Whether the incident describes a different explicit error than the one observed."""
+
+    tokens = incident_tokens(incident)
+    return bool(tokens & _ALL_ERROR_TOKENS) and not tokens & error.incident_tokens
+
+
+def _environment_matches(failure: CIFailure, incident: HistoricalIncident) -> int:
+    failure_environment = dict(normalize_environment(failure.environment))
+    incident_environment = dict(normalize_environment(incident.environment))
+    return sum(incident_environment.get(key) == value for key, value in failure_environment.items())
+
+
+def _data_path() -> Path:
+    return data_path("incidents.json")
+
+
+def analyze_failure(
+    failure: CIFailure,
+    incidents: Iterable[HistoricalIncident] | None = None,
+) -> AnalysisResult:
+    """Analyze one failure using local keyword and environment evidence only.
+
+    An explicit error (dependency installation failure, ModuleNotFoundError,
+    ImportError, or a test assertion) identifies a software failure unless the
+    log also states an explicit environment error, such as a full disk that
+    made a test fail. Otherwise a category requires at least two matching
+    incident keywords beyond generic CI words, or two direct category keywords.
+    This favors ``unknown`` when the evidence is sparse.
+    """
+
+    available_incidents = list(incidents) if incidents is not None else load_incidents(_data_path())
+    matches = find_similar_incidents(failure, available_incidents)
+    failure_tokens = tokenize(failure.failure_text)
+    explicit = _explicit_error(failure.failure_text)
+    if explicit is not None and not _ENVIRONMENT_ERROR.search(failure.failure_text):
+        return _explicit_result(failure, explicit, matches, failure_tokens)
+
+    category_scores: dict[CauseCategory, float] = defaultdict(float)
+    incident_overlap: dict[str, int] = {}
+    environment_overlap: dict[str, int] = {}
+    for incident in matches:
+        keyword_overlap = len(failure_tokens & incident_tokens(incident))
+        environment_matches = _environment_matches(failure, incident)
+        incident_overlap[incident.id] = keyword_overlap
+        environment_overlap[incident.id] = environment_matches
+        category_scores[incident.category] += keyword_overlap + (0.5 * environment_matches)
+
+    direct_signals = {
+        category: failure_tokens & keywords
+        for category, keywords in _CATEGORY_KEYWORDS.items()
+        if category != "unknown"
+    }
+    for category, signals in direct_signals.items():
+        category_scores[category] += 0.25 * len(signals)
+
+    candidate = max(
+        (category for category in _CATEGORY_KEYWORDS if category != "unknown"),
+        key=lambda category: category_scores[category],
+    )
+    strongest_match = next((match for match in matches if match.category == candidate), None)
+    strong_incident_evidence = (
+        strongest_match is not None and len((failure_tokens & incident_tokens(strongest_match)) - _GENERIC_TOKENS) >= 2
+    )
+    strong_direct_evidence = len(direct_signals[candidate]) >= 2
+
+    if not (strong_incident_evidence or strong_direct_evidence):
+        return AnalysisResult(
+            fingerprint=failure_fingerprint(failure),
+            category="unknown",
+            confidence=0.0,
+            root_cause="Insufficient deterministic evidence to identify a root cause.",
+            evidence=[],
+            recommendation="Collect additional failure output and environment details, then retry analysis.",
+            owner=_OWNERS["unknown"],
+            matched_incident_ids=[],
+        )
+
+    evidence: list[str] = []
+    for incident in matches:
+        if incident.category != candidate:
+            continue
+        evidence.append(
+            f"Historical incident {incident.id} shares {incident_overlap[incident.id]} keyword(s)"
+            f" and {environment_overlap[incident.id]} environment value(s)."
+        )
+    if direct_signals[candidate]:
+        evidence.append(f"Failure text contains {', '.join(sorted(direct_signals[candidate]))}.")
+
+    confidence = min(0.95, 0.35 + (0.1 * category_scores[candidate]))
+    if strongest_match is not None and _root_cause_applies(failure_tokens, explicit, strongest_match):
+        # Packaged incidents have no root_cause, so their summary remains the output.
+        root_cause = strongest_match.root_cause or strongest_match.summary
+        recommendation = strongest_match.recommendation
+    else:
+        root_cause = f"Failure text indicates a likely {candidate} issue."
+        recommendation = f"Investigate the {candidate} condition reported in the failure output."
+
+    relevant_matches = [match.id for match in matches if match.category == candidate]
+    return AnalysisResult(
+        fingerprint=failure_fingerprint(failure),
+        category=candidate,
+        confidence=confidence,
+        root_cause=root_cause,
+        evidence=evidence,
+        recommendation=recommendation,
+        owner=_OWNERS[candidate],
+        matched_incident_ids=relevant_matches,
+    )
+
+
+def _root_cause_applies(
+    failure_tokens: set[str], explicit: _ExplicitError | None, incident: HistoricalIncident
+) -> bool:
+    """Whether every pipeline stage the incident's root cause claims is shown by the log's words or explicit error.
+
+    Matching is unaffected: an incident can be a relevant match without its root cause applying.
+    """
+
+    claimed = {stage for stage, words in _STAGE_TOKENS.items() if incident_tokens(incident) & words}
+    shown = {
+        stage
+        for stage, words in _STAGE_TOKENS.items()
+        if failure_tokens & words or (explicit is not None and explicit.kind.startswith(stage))
+    }
+    return claimed <= shown
+
+
+def _explicit_result(
+    failure: CIFailure, error: _ExplicitError, matches: list[HistoricalIncident], failure_tokens: set[str]
+) -> AnalysisResult:
+    """Classify a failure from its explicit error, keeping only incidents consistent with it."""
+
+    # Support: distinctive terms shared with the log, plus one when the incident names this error or a more
+    # general one.
+    # An incident is used only with some support, and its root cause only with at least two.
+    support = {
+        match.id: len((failure_tokens & incident_tokens(match)) - _GENERIC_TOKENS - _ALL_ERROR_TOKENS)
+        + bool(incident_tokens(match) & error.cause_tokens)
+        for match in matches
+    }
+    consistent = [
+        match
+        for match in matches
+        if match.category == "software"
+        and not _contradicts(match, error)
+        and (incident_tokens(match) & error.incident_tokens or support[match.id] >= 1)
+    ]
+    overlap = {match.id: len(failure_tokens & incident_tokens(match)) for match in consistent}
+    evidence = [
+        f"Historical incident {match.id} shares {overlap[match.id]} keyword(s)"
+        f" and {_environment_matches(failure, match)} environment value(s)."
+        for match in consistent
+    ]
+    evidence.append(f"Failure text contains an explicit {error.kind}.")
+    details = _reported_details(failure.failure_text, error)
+    if details:
+        evidence.append(f"Reported: {details}")
+    direct = failure_tokens & _CATEGORY_KEYWORDS["software"]
+    if direct:
+        evidence.append(f"Failure text contains {', '.join(sorted(direct))}.")
+
+    # An incident naming only a more specific error (ModuleNotFoundError for an ImportError) is related,
+    # but its root cause is not established by this log, however many other terms it shares.
+    explaining = [
+        match
+        for match in consistent
+        if not incident_tokens(match) & _ALL_ERROR_TOKENS or incident_tokens(match) & error.cause_tokens
+    ]
+    source = max(explaining, key=lambda match: support[match.id], default=None)
+    if source is not None and support[source.id] >= 2:
+        root_cause, recommendation = source.root_cause or source.summary, source.recommendation
+    else:
+        root_cause, recommendation = error.root_cause, error.recommendation
+        if details:
+            root_cause = f"{root_cause} First reported: {details}"
+    score = _EXPLICIT_ERROR_WEIGHT + sum(overlap.values()) + 0.25 * len(direct)
+    return AnalysisResult(
+        fingerprint=failure_fingerprint(failure),
+        category="software",
+        confidence=min(0.95, 0.35 + (0.1 * score)),
+        root_cause=root_cause,
+        evidence=evidence,
+        recommendation=recommendation,
+        owner=_OWNERS["software"],
+        matched_incident_ids=[match.id for match in consistent],
+    )

circa/benchmark.py

@@ -0,0 +1,226 @@
+"""Reproducible evaluation of local historical-incident retrieval."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable, Sequence
+from pathlib import Path
+from typing import Literal
+
+from pydantic import BaseModel, Field, ValidationError
+
+from .knowledge import find_similar_incidents, load_incidents
+from .models import CauseCategory, CIFailure, HistoricalIncident, SemanticIncidentMatch
+from .resources import data_path
+from .retrieval import retrieve_historical_candidates
+from .semantic import SemanticIncidentRetriever
+
+
+class EvaluationCase(BaseModel):
+    """One synthetic retrieval query with zero or more supported relevance labels."""
+
+    id: str = Field(min_length=1)
+    failure_text: str = Field(min_length=1)
+    expected_category: CauseCategory
+    relevant_incident_ids: list[str] = Field(default_factory=list)
+    relevance_note: str | None = None
+    synthetic: Literal[True] = True
+
+
+class RetrievalMetrics(BaseModel):
+    """Metrics with explicit denominators for incident-retrieval evaluation."""
+
+    evaluated_cases: int = Field(ge=0)
+    skipped_cases: int = Field(ge=0)
+    relevant_cases: int = Field(ge=0)
+    no_known_relevance_cases: int = Field(ge=0)
+    top_1_accuracy: float | None = Field(default=None, ge=0.0, le=1.0)
+    recall_at_k: float | None = Field(default=None, ge=0.0, le=1.0)
+    mean_reciprocal_rank: float | None = Field(default=None, ge=0.0, le=1.0)
+    no_result_rate: float | None = Field(default=None, ge=0.0, le=1.0)
+    evaluation_coverage: float | None = Field(default=None, ge=0.0, le=1.0)
+
+
+class BenchmarkMethodResult(BaseModel):
+    """Metrics and availability state for one retrieval method."""
+
+    name: str
+    status: str
+    metrics: RetrievalMetrics
+
+
+class BenchmarkReport(BaseModel):
+    """All retrieval benchmark results for one fixed dataset and top-K value."""
+
+    dataset_size: int = Field(ge=0)
+    top_k: int = Field(ge=1)
+    methods: list[BenchmarkMethodResult]
+
+
+def load_evaluation_cases(
+    path: str | Path,
+    incidents: Iterable[HistoricalIncident] | None = None,
+) -> list[EvaluationCase]:
+    """Load and validate synthetic cases against the available incident IDs."""
+
+    source = Path(path)
+    try:
+        payload = json.loads(source.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as error:
+        raise ValueError(f"Invalid evaluation dataset JSON in {source}: {error.msg}") from error
+    if not isinstance(payload, dict) or not isinstance(payload.get("cases"), list):
+        raise TypeError(f"Invalid evaluation dataset {source}: expected an object with a 'cases' list.")
+
+    available_incidents = incidents if incidents is not None else load_incidents(data_path("incidents.json"))
+    known_ids = {incident.id for incident in available_incidents}
+    cases: list[EvaluationCase] = []
+    seen_ids: set[str] = set()
+    for index, record in enumerate(payload["cases"]):
+        try:
+            case = EvaluationCase.model_validate(record)
+        except ValidationError as error:
+            raise ValueError(f"Invalid evaluation case at index {index} in {source}: {error}") from error
+        if case.id in seen_ids:
+            raise ValueError(f"Duplicate evaluation case ID '{case.id}' in {source}.")
+        unknown_ids = sorted(set(case.relevant_incident_ids) - known_ids)
+        if unknown_ids:
+            raise ValueError(
+                f"Evaluation case '{case.id}' references unknown historical incident ID(s): {', '.join(unknown_ids)}."
+            )
+        seen_ids.add(case.id)
+        cases.append(case)
+    return cases
+
+
+def calculate_retrieval_metrics(
+    cases: Sequence[EvaluationCase],
+    retrieved_ids: Sequence[Sequence[str] | None],
+    top_k: int,
+) -> RetrievalMetrics:
+    """Calculate retrieval metrics without treating unlabeled cases as misses.
+
+    Top-1, Recall@K, and MRR use only cases with one or more known relevant
+    incident IDs. No-result rate uses all successfully evaluated cases.
+    Coverage uses every dataset case, including deliberately ambiguous cases.
+    """
+
+    if top_k < 1:
+        raise ValueError("top_k must be at least 1")
+    if len(cases) != len(retrieved_ids):
+        raise ValueError("cases and retrieved_ids must have the same length")
+
+    evaluated = [(case, ids[:top_k]) for case, ids in zip(cases, retrieved_ids, strict=True) if ids is not None]
+    eligible = [(case, ids) for case, ids in evaluated if case.relevant_incident_ids]
+    no_results = sum(not ids for _, ids in evaluated)
+    top_1_hits = sum(bool(ids) and ids[0] in case.relevant_incident_ids for case, ids in eligible)
+    recall_hits = sum(bool(set(ids) & set(case.relevant_incident_ids)) for case, ids in eligible)
+    reciprocal_ranks = [
+        next(
+            (1 / rank for rank, incident_id in enumerate(ids, start=1) if incident_id in case.relevant_incident_ids),
+            0.0,
+        )
+        for case, ids in eligible
+    ]
+    eligible_count = len(eligible)
+    evaluated_count = len(evaluated)
+    dataset_size = len(cases)
+    return RetrievalMetrics(
+        evaluated_cases=evaluated_count,
+        skipped_cases=dataset_size - evaluated_count,
+        relevant_cases=eligible_count,
+        no_known_relevance_cases=evaluated_count - eligible_count,
+        top_1_accuracy=top_1_hits / eligible_count if eligible_count else None,
+        recall_at_k=recall_hits / eligible_count if eligible_count else None,
+        mean_reciprocal_rank=sum(reciprocal_ranks) / eligible_count if eligible_count else None,
+        no_result_rate=no_results / evaluated_count if evaluated_count else None,
+        evaluation_coverage=evaluated_count / dataset_size if dataset_size else None,
+    )
+
+
+def _failure(case: EvaluationCase) -> CIFailure:
+    return CIFailure(failure_text=case.failure_text)
+
+
+class _PrecomputedSemanticRetriever:
+    """Replay one semantic result so combined retrieval cannot query the backend again."""
+
+    def __init__(self, matches: list[SemanticIncidentMatch]) -> None:
+        self.matches = matches
+
+    def retrieve(
+        self, failure: CIFailure, incidents: Iterable[HistoricalIncident], limit: int = 3
+    ) -> list[SemanticIncidentMatch]:
+        return self.matches
+
+
+def run_retrieval_benchmark(
+    cases: Sequence[EvaluationCase],
+    incidents: Iterable[HistoricalIncident],
+    *,
+    top_k: int = 3,
+    semantic_retriever: SemanticIncidentRetriever | None = None,
+) -> BenchmarkReport:
+    """Run existing keyword, semantic, and combined retrieval on labeled cases.
+
+    A semantic ``None`` result means its embedding backend was unavailable for
+    that case. Combined retrieval still returns its documented keyword fallback
+    and labels that condition in its method status.
+    """
+
+    if top_k < 1:
+        raise ValueError("top_k must be at least 1")
+    available_incidents = list(incidents)
+    keyword_ids: list[list[str]] = []
+    semantic_ids: list[list[str] | None] = []
+    combined_ids: list[list[str]] = []
+    semantic_unavailable = 0
+    active_semantic_retriever = semantic_retriever
+
+    for case in cases:
+        failure = _failure(case)
+        keyword_ids.append([incident.id for incident in find_similar_incidents(failure, available_incidents, top_k)])
+        semantic_matches = (
+            active_semantic_retriever.retrieve(failure, available_incidents, top_k)
+            if active_semantic_retriever
+            else None
+        )
+        if semantic_matches is None:
+            semantic_ids.append(None)
+            semantic_unavailable += 1
+            # Avoid retrying an unavailable local backend for every case.
+            active_semantic_retriever = None
+        else:
+            semantic_ids.append([match.incident.id for match in semantic_matches])
+        combined = retrieve_historical_candidates(
+            failure,
+            available_incidents,
+            semantic_retriever=_PrecomputedSemanticRetriever(semantic_matches)
+            if semantic_matches is not None
+            else None,
+            keyword_limit=top_k,
+            semantic_limit=top_k,
+            limit=top_k,
+        )
+        combined_ids.append([candidate.incident.id for candidate in combined])
+
+    semantic_status = (
+        "available" if semantic_unavailable == 0 else "unavailable" if semantic_unavailable == len(cases) else "partial"
+    )
+    combined_status = (
+        "combined" if semantic_unavailable == 0 else f"keyword fallback for {semantic_unavailable} case(s)"
+    )
+    return BenchmarkReport(
+        dataset_size=len(cases),
+        top_k=top_k,
+        methods=[
+            BenchmarkMethodResult(
+                name="keyword", status="available", metrics=calculate_retrieval_metrics(cases, keyword_ids, top_k)
+            ),
+            BenchmarkMethodResult(
+                name="semantic", status=semantic_status, metrics=calculate_retrieval_metrics(cases, semantic_ids, top_k)
+            ),
+            BenchmarkMethodResult(
+                name="combined", status=combined_status, metrics=calculate_retrieval_metrics(cases, combined_ids, top_k)
+            ),
+        ],
+    )