bront 0.1.0__py3-none-any.whl

bront/__init__.py

@@ -0,0 +1 @@
+"""Bront: repeatable inference reliability experiments and release evidence."""

bront/artifacts.py

@@ -0,0 +1,30 @@
+"""Atomic UTF-8 artifact output shared by JSON evidence and HTML reports."""
+
+import json
+import os
+import tempfile
+from pathlib import Path
+
+from pydantic import BaseModel
+
+
+def write_artifact(report: BaseModel, path: Path) -> None:
+    write_text_artifact(
+        json.dumps(report.model_dump(mode="json"), indent=2, allow_nan=False) + "\n", path
+    )
+
+
+def write_text_artifact(content: str, path: Path) -> None:
+    """Atomic replacement avoids presenting a partially written artifact as evidence."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = None
+    try:
+        with tempfile.NamedTemporaryFile(
+            mode="w", encoding="utf-8", dir=path.parent, delete=False
+        ) as stream:
+            temporary = Path(stream.name)
+            stream.write(content)
+        os.replace(temporary, path)
+    finally:
+        if temporary is not None:
+            temporary.unlink(missing_ok=True)

bront/checks.py

@@ -0,0 +1,104 @@
+"""Explicit release checks, also used to verify assertions when reading reports."""
+
+from bront.results import AssertionResult, Phase
+from bront.scenario_models import RecoveryEvidence, ScenarioEvent, ScenarioEvidence
+from bront.suites import Assertions
+
+
+def evaluate_measurement(phase: Phase, policy: Assertions) -> list[AssertionResult]:
+    values = [
+        ("max_failed_requests", phase.failed, policy.max_failed_requests),
+        ("max_p95_ms", phase.latency_ms.get("p95"), policy.max_p95_ms),
+        ("max_p99_ms", phase.latency_ms.get("p99"), policy.max_p99_ms),
+        (
+            "max_error_rate",
+            phase.failed / phase.requests if phase.requests else None,
+            policy.max_error_rate,
+        ),
+        (
+            "max_validation_failures",
+            phase.errors.get("validation", 0),
+            policy.max_validation_failures,
+        ),
+    ]
+    return [
+        AssertionResult(
+            name=name, actual=actual, limit=limit, passed=actual is not None and actual <= limit
+        )
+        for name, actual, limit in values
+        if limit is not None
+    ]
+
+
+def evaluate_recovery(recovery: RecoveryEvidence, policy: Assertions) -> list[AssertionResult]:
+    results = []
+    for name, observed, limit in (
+        (
+            "recovery_within_seconds",
+            recovery.first_valid_response_ms,
+            policy.recovery_within_seconds,
+        ),
+        (
+            "preferred_backend_within_seconds",
+            recovery.first_preferred_backend_ms,
+            policy.preferred_backend_within_seconds,
+        ),
+    ):
+        if limit is not None:
+            actual = observed / 1000 if observed is not None else None
+            results.append(
+                AssertionResult(
+                    name=name,
+                    actual=actual,
+                    limit=limit,
+                    passed=actual is not None and actual <= limit,
+                )
+            )
+    return results
+
+
+def observe_recovery(
+    phase: Phase, events: list[ScenarioEvent], policy: Assertions, has_identity: bool
+) -> RecoveryEvidence | None:
+    restores = [e for e in events if e.action == "restore" and e.status == "completed"]
+    if not restores:
+        return None
+    start = restores[-1].finished_ms
+    if start is None:
+        return None
+    recovery = RecoveryEvidence(
+        from_ms=start,
+        backend_identity_available=has_identity,
+        preferred_backend=policy.preferred_backend,
+    )
+    valid = [s for s in phase.samples if s.error is None and s.started_ms >= start]
+    if valid:
+        recovery.first_valid_response_ms = min(s.started_ms + s.duration_ms for s in valid) - start
+    preferred = [
+        s
+        for s in valid
+        if policy.preferred_backend is not None and s.backend == policy.preferred_backend
+    ]
+    if preferred:
+        recovery.first_preferred_backend_ms = (
+            min(s.started_ms + s.duration_ms for s in preferred) - start
+        )
+    return recovery
+
+
+def traffic_windows(phase: Phase, scenario: ScenarioEvidence) -> dict[str, int]:
+    events = {event.action: event for event in scenario.events}
+    applied = events["apply"]
+    reversible = scenario.kind != "kubernetes-restart"
+    terminal = events["restored" if reversible else "observed"]
+    end = terminal.finished_ms
+    during_start = events["observed"].finished_ms if reversible else applied.finished_ms
+    during_end = events["restore"].started_ms if reversible else end
+    if end is None or during_start is None or during_end is None:
+        raise ValueError("scenario_events_incomplete")
+    starts = [sample.started_ms for sample in phase.samples]
+    return {
+        "before": sum(start < applied.started_ms for start in starts),
+        "during": sum(during_start <= start <= during_end for start in starts),
+        "after": sum(start > end for start in starts),
+    }

bront/cli.py

@@ -0,0 +1,21 @@
+"""Bront: repeatable inference experiments, release comparisons, and reports."""
+
+import typer
+
+from bront.compare_cli import compare
+from bront.report_cli import report
+from bront.run_cli import run
+
+app = typer.Typer(no_args_is_help=True, pretty_exceptions_show_locals=False)
+app.command()(run)
+app.command()(compare)
+app.command()(report)
+
+
+@app.callback()
+def main() -> None:
+    """Validate inference deployments with repeatable experiments and release evidence."""
+
+
+if __name__ == "__main__":
+    app()

bront/compare_cli.py

@@ -0,0 +1,81 @@
+"""CLI for reproducible baseline/candidate release decisions."""
+
+import tomllib
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich.console import Console
+
+from bront.comparison_models import ComparisonPolicy
+from bront.comparisons import compare_files
+from bront.suites import SuiteError
+
+
+def compare(
+    baseline: Annotated[Path, typer.Argument(help="Baseline run or repeated-run manifest.")],
+    candidate: Annotated[Path, typer.Argument(help="Candidate run or repeated-run manifest.")],
+    policy: Annotated[Path, typer.Option(help="TOML evidence and regression policy.")],
+    baseline_run: Annotated[
+        list[Path] | None, typer.Option(help="Additional baseline run; repeatable.")
+    ] = None,
+    candidate_run: Annotated[
+        list[Path] | None, typer.Option(help="Additional candidate run; repeatable.")
+    ] = None,
+    output: Annotated[Path, typer.Option(help="Comparison JSON artifact.")] = Path(
+        "reports/comparison.json"
+    ),
+    json_output: Annotated[
+        bool, typer.Option("--json", help="Print the comparison as JSON.")
+    ] = False,
+) -> None:
+    """Compare compatible repeated runs; exit 2 when evidence cannot support a verdict."""
+    try:
+        if output.resolve() == policy.resolve():
+            raise SuiteError("Comparison output must not overwrite its policy")
+        if policy.stat().st_size > 1_048_576:
+            raise SuiteError("Policy exceeds the 1 MiB limit")
+        settings = ComparisonPolicy.model_validate(tomllib.loads(policy.read_text()))
+        report = compare_files(
+            [baseline, *(baseline_run or [])], [candidate, *(candidate_run or [])], settings, output
+        )
+    except SuiteError as exc:
+        typer.echo(f"Configuration error: {exc}", err=True)
+        raise typer.Exit(2) from None
+    except (ValueError, OSError) as exc:
+        typer.echo(
+            f"Cannot compare artifacts ({type(exc).__name__}); check policy and output.", err=True
+        )
+        raise typer.Exit(2) from None
+    if json_output:
+        typer.echo(report.model_dump_json(indent=2))
+    else:
+        console = Console(markup=False)
+        console.print(
+            f"{report.status.upper()}: {len(report.baseline_run_ids)} baseline runs, "
+            f"{len(report.candidate_run_ids)} candidate runs"
+        )
+        for difference in report.differences[:10]:
+            console.print(
+                f"{'Allowed' if difference.allowed else 'Incompatible'} change: {difference.field}"
+            )
+        for issue in report.issues[:10]:
+            console.print(f"Evidence: {issue}")
+        for name, metric in report.metrics.items():
+            console.print(
+                f"{name}: baseline median {metric.baseline.median:.3g} "
+                f"[{metric.baseline.minimum:.3g}, {metric.baseline.maximum:.3g}]; "
+                f"candidate median {metric.candidate.median:.3g} "
+                f"[{metric.candidate.minimum:.3g}, {metric.candidate.maximum:.3g}] {metric.unit}"
+            )
+        for check in report.checks:
+            console.print(
+                f"{check.name}: {check.status}; limit {check.limit:g}"
+                + (f" ({check.reason})" if check.reason else "")
+            )
+        if any(run["status"] == "failed" for run in report.run_summaries):
+            console.print(
+                "One or more runs failed their own release assertions; see run_summaries."
+            )
+        console.print(f"Report: {output}")
+    raise typer.Exit({"passed": 0, "failed": 1}.get(report.status, 2))

bront/comparison_models.py

@@ -0,0 +1,83 @@
+"""Comparison policy and evidence models. No statistical significance claim is implied."""
+
+import re
+from typing import Literal, Self
+
+from pydantic import Field, JsonValue, model_validator
+
+from bront.results import timestamp
+from bront.validation import StrictModel
+
+
+class ComparisonPolicy(StrictModel):
+    version: Literal[1]
+    minimum_runs: int = Field(default=3, ge=3, le=100)
+    minimum_requests: int = Field(default=100, ge=1, le=1_000_000)
+    minimum_duration_seconds: float = Field(default=0, ge=0, le=3600)
+    required_labels: list[str] = Field(default_factory=list, max_length=100)
+    allowed_changes: list[str] = Field(default_factory=list, max_length=100)
+    max_p95_increase_percent: float | None = Field(default=None, ge=0)
+    max_p99_increase_percent: float | None = Field(default=None, ge=0)
+    max_throughput_drop_percent: float | None = Field(default=None, ge=0, le=100)
+    max_error_rate_increase: float | None = Field(default=None, ge=0, le=1)
+
+    @model_validator(mode="after")
+    def intentional_changes(self) -> Self:
+        permitted = (
+            r"deployment\.(model_revision|serving_revision|architecture|resources(\.[\w.-]+)?|labels\.[\w.-]+)"
+            r"|contract\.(route_sha256|scenario\.(context|namespace|deployment|proxy))"
+        )
+        if any(not re.fullmatch(permitted, path) for path in self.allowed_changes):
+            raise ValueError(
+                "Only deployment metadata, route, and scenario scope changes can be allowed"
+            )
+        if len(set(self.allowed_changes)) != len(self.allowed_changes):
+            raise ValueError("Duplicate allowed change")
+        return self
+
+
+class Distribution(StrictModel):
+    values: list[float]
+    minimum: float
+    median: float
+    maximum: float
+
+
+class MetricComparison(StrictModel):
+    unit: str
+    baseline: Distribution
+    candidate: Distribution
+    median_change_percent: float | None
+
+
+class ComparisonCheck(StrictModel):
+    name: str
+    status: Literal["passed", "failed", "inconclusive"]
+    limit: float
+    observed_lower: float | None
+    observed_upper: float | None
+    reason: str | None = None
+
+
+class Difference(StrictModel):
+    field: str
+    baseline: JsonValue
+    candidate: JsonValue
+    allowed: bool = False
+
+
+class ComparisonReport(StrictModel):
+    report_type: Literal["bront.comparison"] = "bront.comparison"
+    schema_version: Literal[1] = 1
+    created_at: str = Field(default_factory=timestamp)
+    status: Literal["passed", "failed", "inconclusive", "incompatible"] = "inconclusive"
+    policy: ComparisonPolicy
+    baseline_run_ids: list[str] = Field(default_factory=list)
+    candidate_run_ids: list[str] = Field(default_factory=list)
+    issues: list[str] = Field(default_factory=list)
+    differences: list[Difference] = Field(default_factory=list)
+    baseline_context: dict[str, JsonValue] = Field(default_factory=dict)
+    candidate_context: dict[str, JsonValue] = Field(default_factory=dict)
+    run_summaries: list[dict[str, JsonValue]] = Field(default_factory=list)
+    metrics: dict[str, MetricComparison] = Field(default_factory=dict)
+    checks: list[ComparisonCheck] = Field(default_factory=list)

bront/comparisons.py

@@ -0,0 +1,231 @@
+"""Conservative release verdicts from compatible, completed, repeated experiments."""
+
+import math
+from pathlib import Path
+from statistics import median
+
+from pydantic import JsonValue
+
+from bront.artifacts import write_artifact
+from bront.comparison_models import (
+    ComparisonCheck,
+    ComparisonPolicy,
+    ComparisonReport,
+    Difference,
+    Distribution,
+    MetricComparison,
+)
+from bront.evidence import Evidence, ReadBudget, load_evidence
+from bront.results import RunReport
+from bront.suites import SuiteError
+
+
+def context(report: RunReport) -> dict[str, JsonValue]:
+    return {
+        "contract": report.provenance.contract,
+        "runner": {key: value for key, value in report.provenance.runner.items()},
+        "deployment": report.provenance.deployment.model_dump(mode="json"),
+    }
+
+
+def differences(left: JsonValue, right: JsonValue, path: str = "") -> list[Difference]:
+    if left == right:
+        return []
+    if isinstance(left, dict) and isinstance(right, dict):
+        result = []
+        for key in sorted(left.keys() | right.keys()):
+            location = f"{path}.{key}" if path else key
+            if key not in left or key not in right:
+                result.append(
+                    Difference(field=location, baseline=left.get(key), candidate=right.get(key))
+                )
+            else:
+                result.extend(differences(left[key], right[key], location))
+        return result
+    return [Difference(field=path, baseline=left, candidate=right)]
+
+
+def distribution(values: list[float]) -> Distribution:
+    return Distribution(
+        values=values, minimum=min(values), median=median(values), maximum=max(values)
+    )
+
+
+def percent_change(base: float, candidate: float) -> float | None:
+    if base == 0:
+        return 0 if candidate == 0 else None
+    result = (candidate / base - 1) * 100
+    return result if math.isfinite(result) else None
+
+
+def metric_checks(
+    report: ComparisonReport,
+    baseline: list[RunReport],
+    candidate: list[RunReport],
+    *,
+    gate: bool = True,
+) -> None:
+    policy = report.policy
+    metrics = (
+        (
+            "p95_ms",
+            "ms",
+            lambda r: r.measurement.latency_ms["p95"],
+            policy.max_p95_increase_percent,
+            False,
+        ),
+        (
+            "p99_ms",
+            "ms",
+            lambda r: r.measurement.latency_ms["p99"],
+            policy.max_p99_increase_percent,
+            False,
+        ),
+        (
+            "requests_per_second",
+            "requests/s",
+            lambda r: r.measurement.requests_per_second,
+            policy.max_throughput_drop_percent,
+            True,
+        ),
+        (
+            "error_rate",
+            "fraction",
+            lambda r: r.measurement.failed / r.measurement.requests,
+            policy.max_error_rate_increase,
+            False,
+        ),
+    )
+    for name, unit, extract, limit, reverse in metrics:
+        left = distribution([extract(run) for run in baseline])
+        right = distribution([extract(run) for run in candidate])
+        report.metrics[name] = MetricComparison(
+            unit=unit,
+            baseline=left,
+            candidate=right,
+            median_change_percent=percent_change(left.median, right.median),
+        )
+        if limit is None or not gate:
+            continue
+        changes = []
+        for before in (left.minimum, left.maximum):
+            for after in (right.minimum, right.maximum):
+                value = after - before if name == "error_rate" else percent_change(before, after)
+                changes.append(-value if reverse and value is not None else value)
+        if any(value is None for value in changes):
+            report.checks.append(
+                ComparisonCheck(
+                    name=name,
+                    status="inconclusive",
+                    limit=limit,
+                    observed_lower=None,
+                    observed_upper=None,
+                    reason="relative_change_undefined",
+                )
+            )
+            continue
+        defined = [value for value in changes if value is not None]
+        lower, upper = min(defined), max(defined)
+        verdict = "passed" if upper <= limit else "failed" if lower > limit else "inconclusive"
+        report.checks.append(
+            ComparisonCheck(
+                name=name,
+                status=verdict,
+                limit=limit,
+                observed_lower=lower,
+                observed_upper=upper,
+                reason="variation_crosses_limit" if verdict == "inconclusive" else None,
+            )
+        )
+
+
+def compare_evidence(
+    baseline: Evidence, candidate: Evidence, policy: ComparisonPolicy
+) -> ComparisonReport:
+    report = ComparisonReport(policy=policy, issues=[*baseline.issues, *candidate.issues])
+    report.baseline_run_ids = [r.provenance.run_id for r in baseline.runs]
+    report.candidate_run_ids = [r.provenance.run_id for r in candidate.runs]
+    ids = report.baseline_run_ids + report.candidate_run_ids
+    if len(set(ids)) != len(ids):
+        report.issues.append("duplicate_run_identity")
+    for side, evidence in (("baseline", baseline), ("candidate", candidate)):
+        if len(evidence.runs) < policy.minimum_runs:
+            report.issues.append(f"{side}:insufficient_repeats")
+        for index, run in enumerate(evidence.runs):
+            label = f"{side}[{index}]"
+            report.run_summaries.append(
+                {
+                    "side": side,
+                    "run_id": run.provenance.run_id,
+                    "status": run.status,
+                    "requests": run.measurement.requests,
+                    "failed": run.measurement.failed,
+                    "errors": {key: value for key, value in run.measurement.errors.items()},
+                    "failed_assertions": [a.name for a in run.assertions if not a.passed],
+                }
+            )
+            if run.status not in ("passed", "failed") or run.reason is not None:
+                report.issues.append(f"{label}:run_incomplete")
+            for missing in run.provenance.deployment.missing():
+                report.issues.append(f"{label}:missing_metadata:{missing}")
+            for key in policy.required_labels:
+                if not run.provenance.deployment.labels.get(key):
+                    report.issues.append(f"{label}:required_label_missing:{key}")
+            if run.measurement.requests < policy.minimum_requests:
+                report.issues.append(f"{label}:insufficient_requests")
+            if run.measurement.elapsed_seconds < policy.minimum_duration_seconds:
+                report.issues.append(f"{label}:insufficient_duration")
+            if side == "baseline" and run.status != "passed":
+                report.issues.append(f"{label}:baseline_assertions_not_passed")
+    if not baseline.runs or not candidate.runs:
+        return report
+    report.baseline_context = context(baseline.runs[0])
+    report.candidate_context = context(candidate.runs[0])
+    for side, evidence in (("baseline", baseline), ("candidate", candidate)):
+        for index, run in enumerate(evidence.runs[1:], 1):
+            for difference in differences(context(evidence.runs[0]), context(run)):
+                difference.field = f"{side}[{index}].{difference.field}"
+                report.differences.append(difference)
+    for difference in differences(report.baseline_context, report.candidate_context):
+        difference.allowed = any(
+            difference.field == path or difference.field.startswith(path + ".")
+            for path in policy.allowed_changes
+        )
+        report.differences.append(difference)
+    if any(not d.allowed for d in report.differences):
+        report.status = "incompatible"
+        return report
+    complete = all(
+        run.status in ("passed", "failed") and run.reason is None and run.measurement.requests > 0
+        for run in baseline.runs + candidate.runs
+    )
+    if complete:
+        metric_checks(report, baseline.runs, candidate.runs, gate=not report.issues)
+    if report.issues:
+        return report
+    if any(r.status == "failed" for r in candidate.runs) or any(
+        c.status == "failed" for c in report.checks
+    ):
+        report.status = "failed"
+    elif any(c.status == "inconclusive" for c in report.checks):
+        report.status = "inconclusive"
+    else:
+        report.status = "passed"
+    return report
+
+
+def compare_files(
+    baseline: list[Path],
+    candidate: list[Path],
+    policy: ComparisonPolicy,
+    output: Path | None = None,
+) -> ComparisonReport:
+    budget = ReadBudget()
+    left = load_evidence(baseline, "baseline", budget)
+    right = load_evidence(candidate, "candidate", budget)
+    report = compare_evidence(left, right, policy)
+    if output is not None:
+        if output.resolve() in budget.paths:
+            raise SuiteError("Comparison output must not overwrite an input artifact")
+        write_artifact(report, output)
+    return report