invarlock 0.3.6__py3-none-any.whl → 0.3.8__py3-none-any.whl

invarlock/cli/commands/report.py

@@ -8,20 +8,101 @@ Provides the `invarlock report` group with:
 """
 
 import json
+import math
 from pathlib import Path
+from time import perf_counter
+from typing import Any
 
 import typer
 from rich.console import Console
 
-from invarlock.reporting import certificate as certificate_lib
+from invarlock.cli.output import print_event, resolve_output_style
 from invarlock.reporting import report as report_lib
+from invarlock.reporting import report_builder as report_builder
 
 console = Console()
 
+SECTION_WIDTH = 67
+KV_LABEL_WIDTH = 16
+GATE_LABEL_WIDTH = 32
+ARTIFACT_LABEL_WIDTH = 18
+
+
+def _format_section_title(title: str, *, suffix: str | None = None) -> str:
+    if not suffix:
+        return title
+    combined = f"{title} {suffix}"
+    if len(combined) > SECTION_WIDTH:
+        return combined
+    pad = max(1, SECTION_WIDTH - len(title) - len(suffix))
+    return f"{title}{' ' * pad}{suffix}"
+
+
+def _print_section_header(
+    console: Console, title: str, *, suffix: str | None = None
+) -> None:
+    bar = "═" * SECTION_WIDTH
+    console.print(bar)
+    console.print(_format_section_title(title, suffix=suffix))
+    console.print(bar)
+
+
+def _format_kv_line(label: str, value: str, *, width: int = KV_LABEL_WIDTH) -> str:
+    return f"  {label:<{width}}: {value}"
+
+
+def _format_status(ok: bool) -> str:
+    return "PASS" if ok else "FAIL"
+
+
+def _fmt_metric_value(value: Any) -> str:
+    try:
+        val = float(value)
+    except (TypeError, ValueError):
+        return "N/A"
+    if not math.isfinite(val):
+        return "N/A"
+    return f"{val:.3f}"
+
+
+def _fmt_ci_95(ci: Any) -> str | None:
+    if isinstance(ci, (list, tuple)) and len(ci) == 2:
+        try:
+            lo = float(ci[0])
+            hi = float(ci[1])
+        except (TypeError, ValueError):
+            return None
+        if math.isfinite(lo) and math.isfinite(hi):
+            return f"[{lo:.3f}, {hi:.3f}]"
+    return None
+
+
+def _artifact_entries(
+    saved_files: dict[str, str], output_dir: str
+) -> list[tuple[str, str]]:
+    order = [
+        ("report", "Evaluation Report (JSON)"),
+        ("report_md", "Evaluation Report (MD)"),
+        ("json", "JSON"),
+        ("markdown", "Markdown"),
+        ("html", "HTML"),
+    ]
+    entries: list[tuple[str, str]] = [("Output", output_dir)]
+    used: set[str] = set()
+    for key, label in order:
+        if key in saved_files:
+            entries.append((label, str(saved_files[key])))
+            used.add(key)
+    for key in sorted(saved_files.keys()):
+        if key in used:
+            continue
+        entries.append((key.upper(), str(saved_files[key])))
+    return entries
+
 
 # Group with callback so `invarlock report` still generates reports
 report_app = typer.Typer(
-    help="Operations on reports and certificates (verify, explain, html, validate).",
+    help="Operations on run reports and evaluation reports (verify, explain, html, validate).",
     invoke_without_command=True,
 )
 
@@ -33,6 +114,11 @@ def _generate_reports(
     compare: str | None = None,
     baseline: str | None = None,
     output: str | None = None,
+    style: str = "audit",
+    no_color: bool = False,
+    summary_baseline_seconds: float | None = None,
+    summary_subject_seconds: float | None = None,
+    summary_report_start: float | None = None,
 ) -> None:
     # This callback runs only when invoked without subcommand (default Click behavior)
     try:
@@ -55,21 +141,37 @@ def _generate_reports(
         compare = _coerce_option(compare)
         baseline = _coerce_option(baseline)
         output = _coerce_option(output)
+        style = _coerce_option(style, "audit")
+        no_color = bool(_coerce_option(no_color, False))
+        summary_baseline_seconds = _coerce_option(summary_baseline_seconds)
+        summary_subject_seconds = _coerce_option(summary_subject_seconds)
+        summary_report_start = _coerce_option(summary_report_start)
+
+        output_style = resolve_output_style(
+            style=str(style),
+            profile="ci",
+            progress=False,
+            timing=False,
+            no_color=no_color,
+        )
+
+        def _event(tag: str, message: str, *, emoji: str | None = None) -> None:
+            print_event(console, tag, message, style=output_style, emoji=emoji)
 
         # Load primary report
-        console.print(f"📊 Loading run report: {run}")
+        _event("DATA", f"Loading run report: {run}", emoji="📊")
         primary_report = _load_run_report(run)
 
         # Load comparison report if specified
         compare_report = None
         if compare:
-            console.print(f"📊 Loading comparison report: {compare}")
+            _event("DATA", f"Loading comparison report: {compare}", emoji="📊")
             compare_report = _load_run_report(compare)
 
         # Load baseline report if specified
         baseline_report = None
         if baseline:
-            console.print(f"📊 Loading baseline report: {baseline}")
+            _event("DATA", f"Loading baseline report: {baseline}", emoji="📊")
             baseline_report = _load_run_report(baseline)
 
         # Determine output directory
@@ -80,25 +182,39 @@ def _generate_reports(
             output_dir = output
 
         # Determine formats
+        allowed_formats = {"json", "md", "markdown", "html", "report", "all"}
+        if format not in allowed_formats:
+            _event("FAIL", f"Unknown --format '{format}'", emoji="❌")
+            raise typer.Exit(2)
+
+        if format == "md":
+            format = "markdown"
         if format == "all":
             formats = ["json", "markdown", "html"]
         else:
             formats = [format]
 
-        # Validate certificate requirements
-        if "cert" in formats:
+        # Validate evaluation report requirements
+        if "report" in formats:
             if baseline_report is None:
-                console.print(
-                    "[red]❌ Certificate format requires --baseline parameter[/red]"
+                _event(
+                    "FAIL",
+                    "Evaluation report format requires --baseline",
+                    emoji="❌",
                 )
-                console.print(
-                    "Use: invarlock report --run <run_dir> --format cert --baseline <baseline_run_dir>"
+                _event(
+                    "INFO",
+                    "Use: invarlock report --run <run_dir> --format report --baseline <baseline_run_dir>",
                 )
                 raise typer.Exit(1)
-            console.print("📜 Generating safety certificate with baseline comparison")
+            _event(
+                "EXEC",
+                "Generating evaluation report with baseline comparison",
+                emoji="📜",
+            )
 
         # Generate reports
-        console.print(f"📝 Generating reports in formats: {formats}")
+        _event("EXEC", f"Generating reports in formats: {formats}", emoji="📝")
         saved_files = report_lib.save_report(
             primary_report,
             output_dir,
@@ -109,82 +225,151 @@ def _generate_reports(
         )
 
         # Show results
-        console.print("[green]✅ Reports generated successfully![/green]")
-        console.print(f"📁 Output directory: {output_dir}")
-
-        for fmt, file_path in saved_files.items():
-            if fmt == "cert":
-                console.print(f"  📜 CERTIFICATE (JSON): {file_path}")
-            elif fmt == "cert_md":
-                console.print(f"  📜 CERTIFICATE (MD): {file_path}")
-            else:
-                console.print(f"  📄 {fmt.upper()}: {file_path}")
-
-        # Show key metrics (PM-first). Avoid PPL-first wording.
-        console.print("\n📈 Key Metrics:")
-        console.print(f"  Model: {primary_report['meta']['model_id']}")
-        console.print(f"  Edit: {primary_report['edit']['name']}")
-        pm = (primary_report.get("metrics", {}) or {}).get("primary_metric", {})
-        if isinstance(pm, dict) and pm:
-            kind = str(pm.get("kind") or "primary")
-            console.print(f"  Primary Metric: {kind}")
-            final = pm.get("final")
-            if isinstance(final, int | float):
-                console.print(f"  point (final): {final:.3f}")
-            dci = pm.get("display_ci")
-            if isinstance(dci, tuple | list) and len(dci) == 2:
-                try:
-                    lo, hi = float(dci[0]), float(dci[1])
-                    console.print(f"  CI: {lo:.3f}–{hi:.3f}")
-                except Exception:
-                    pass
-            ratio = pm.get("ratio_vs_baseline")
-            if isinstance(ratio, int | float):
-                console.print(f"  ratio vs baseline: {ratio:.3f}")
-
-        # Show certificate validation if generated
-        if "cert" in formats and baseline_report:
+        _event("PASS", "Reports generated successfully.", emoji="✅")
+
+        if "report" in formats and baseline_report:
             try:
-                certificate = certificate_lib.make_certificate(
+                evaluation_report = report_builder.make_report(
                     primary_report, baseline_report
                 )
-                certificate_lib.validate_certificate(certificate)
+                report_builder.validate_report(evaluation_report)
                 from invarlock.reporting.render import (
                     compute_console_validation_block as _console_block,
                 )
 
-                block = _console_block(certificate)
+                block = _console_block(evaluation_report)
                 overall_pass = bool(block.get("overall_pass"))
-
-                console.print("\n📜 Certificate Validation:")
-                status_emoji = "✅" if overall_pass else "❌"
-                console.print(
-                    f"  Overall Status: {status_emoji} {'PASS' if overall_pass else 'FAIL'}"
+                status_text = _format_status(overall_pass)
+
+                console.print("")
+                summary_suffix: str | None = None
+                if summary_report_start is not None:
+                    try:
+                        base = (
+                            float(summary_baseline_seconds)
+                            if summary_baseline_seconds is not None
+                            else 0.0
+                        )
+                        subject = (
+                            float(summary_subject_seconds)
+                            if summary_subject_seconds is not None
+                            else 0.0
+                        )
+                        report_elapsed = max(
+                            0.0, float(perf_counter() - float(summary_report_start))
+                        )
+                        summary_suffix = f"[{(base + subject + report_elapsed):.2f}s]"
+                    except Exception:
+                        summary_suffix = None
+                _print_section_header(
+                    console,
+                    "EVALUATION REPORT SUMMARY",
+                    suffix=summary_suffix,
                 )
+                console.print(_format_kv_line("Status", status_text))
+
+                schema_version = evaluation_report.get("schema_version")
+                if schema_version:
+                    console.print(
+                        _format_kv_line("Schema Version", str(schema_version))
+                    )
 
+                run_id = evaluation_report.get("run_id") or (
+                    (primary_report.get("meta", {}) or {}).get("run_id")
+                )
+                if run_id:
+                    console.print(_format_kv_line("Run ID", str(run_id)))
+
+                model_id = (primary_report.get("meta", {}) or {}).get("model_id")
+                edit_name = (primary_report.get("edit", {}) or {}).get("name")
+                if model_id:
+                    console.print(_format_kv_line("Model", str(model_id)))
+                if edit_name:
+                    console.print(_format_kv_line("Edit", str(edit_name)))
+
+                pm = (
+                    (evaluation_report.get("primary_metric") or {})
+                    if isinstance(evaluation_report, dict)
+                    else {}
+                )
+                if not pm:
+                    pm = (primary_report.get("metrics", {}) or {}).get(
+                        "primary_metric", {}
+                    )
+                console.print("  PRIMARY METRIC")
+                pm_entries: list[tuple[str, str]] = []
+                if isinstance(pm, dict) and pm:
+                    kind = str(pm.get("kind") or "primary")
+                    pm_entries.append(("Kind", kind))
+                    preview = pm.get("preview")
+                    if preview is not None:
+                        pm_entries.append(("Preview", _fmt_metric_value(preview)))
+                    final = pm.get("final")
+                    if final is not None:
+                        pm_entries.append(("Final", _fmt_metric_value(final)))
+                    ratio = pm.get("ratio_vs_baseline")
+                    if ratio is not None:
+                        pm_entries.append(("Ratio", _fmt_metric_value(ratio)))
+                    dci = pm.get("display_ci")
+                    ci_95 = _fmt_ci_95(dci)
+                    if ci_95 is not None:
+                        pm_entries.append(("CI (95%)", ci_95))
+                if not pm_entries:
+                    pm_entries.append(("Status", "Unavailable"))
+                for idx, (label, value) in enumerate(pm_entries):
+                    branch = "└─" if idx == len(pm_entries) - 1 else "├─"
+                    console.print(f"  {branch} {label:<14} {value}")
+
+                console.print("  VALIDATION GATES")
                 rows = block.get("rows", [])
                 if isinstance(rows, list) and rows:
-                    for row in rows:
-                        try:
-                            label = row.get("label")
-                            status = row.get("status")
-                            if label and status:
-                                console.print(f"  {label}: {status}")
-                        except Exception:
-                            continue
+                    for idx, row in enumerate(rows):
+                        label = str(row.get("label") or "Unknown")
+                        ok = bool(row.get("ok"))
+                        status = _format_status(ok)
+                        mark = "✓" if ok else "✗"
+                        branch = "└─" if idx == len(rows) - 1 else "├─"
+                        console.print(
+                            f"  {branch} {label:<{GATE_LABEL_WIDTH}} {mark} {status}"
+                        )
+                else:
+                    console.print(f"  └─ {'No validation rows':<{GATE_LABEL_WIDTH}} -")
+
+                console.print("  ARTIFACTS")
+                entries = _artifact_entries(saved_files, str(output_dir))
+                for idx, (label, value) in enumerate(entries):
+                    branch = "└─" if idx == len(entries) - 1 else "├─"
+                    console.print(f"  {branch} {label:<{ARTIFACT_LABEL_WIDTH}} {value}")
+                console.print("═" * SECTION_WIDTH)
 
                 # In CLI report flow, do not hard-exit on validation failure; just display status.
                 # CI gating should be handled by dedicated verify commands.
 
             except Exception as e:
+                _event("WARN", f"Evaluation report validation error: {e}", emoji="⚠️")
+                # Exit non-zero on evaluation report generation error
+                raise typer.Exit(1) from e
+        else:
+            console.print(_format_kv_line("Output", str(output_dir)))
+            for label, value in _artifact_entries(saved_files, str(output_dir))[1:]:
                 console.print(
-                    f"  [yellow]⚠️  Certificate validation error: {e}[/yellow]"
+                    _format_kv_line(label, str(value), width=ARTIFACT_LABEL_WIDTH)
                 )
-                # Exit non-zero on certificate generation error
-                raise typer.Exit(1) from e
 
     except Exception as e:
-        console.print(f"[red]❌ Report generation failed: {e}[/red]")
+        print_event(
+            console,
+            "FAIL",
+            f"Report generation failed: {e}",
+            style=resolve_output_style(
+                style="audit",
+                profile="ci",
+                progress=False,
+                timing=False,
+                no_color=False,
+            ),
+            emoji="❌",
+        )
         raise typer.Exit(1) from e
 
 
@@ -195,7 +380,7 @@ def report_callback(
         None, "--run", help="Path to run directory or RunReport JSON"
     ),
     format: str = typer.Option(
-        "json", "--format", help="Output format (json|md|html|cert|all)"
+        "json", "--format", help="Output format (json|md|html|report|all)"
     ),
     compare: str | None = typer.Option(
         None, "--compare", help="Path to second run for comparison"
@@ -203,18 +388,40 @@ def report_callback(
     baseline: str | None = typer.Option(
         None,
         "--baseline",
-        help="Path to baseline run for certificate generation (required for cert format)",
+        help="Path to baseline run for evaluation report generation (required for report format)",
     ),
     output: str | None = typer.Option(None, "--output", "-o", help="Output directory"),
+    style: str = typer.Option("audit", "--style", help="Output style (audit|friendly)"),
+    no_color: bool = typer.Option(
+        False, "--no-color", help="Disable ANSI colors (respects NO_COLOR=1)"
+    ),
 ):
     """Generate a report from a run (default callback)."""
     if getattr(ctx, "resilient_parsing", False) or ctx.invoked_subcommand is not None:
         return
     if not run:
-        console.print("[red]❌ --run is required when no subcommand is provided[/red]")
+        print_event(
+            console,
+            "FAIL",
+            "--run is required when no subcommand is provided",
+            style=resolve_output_style(
+                style=str(style),
+                profile="ci",
+                progress=False,
+                timing=False,
+                no_color=no_color,
+            ),
+            emoji="❌",
+        )
         raise typer.Exit(2)
     return _generate_reports(
-        run=run, format=format, compare=compare, baseline=baseline, output=output
+        run=run,
+        format=format,
+        compare=compare,
+        baseline=baseline,
+        output=output,
+        style=style,
+        no_color=no_color,
     )
 
 
@@ -225,9 +432,23 @@ def report_command(
     compare: str | None = None,
     baseline: str | None = None,
     output: str | None = None,
+    style: str = "audit",
+    no_color: bool = False,
+    summary_baseline_seconds: float | None = None,
+    summary_subject_seconds: float | None = None,
+    summary_report_start: float | None = None,
 ):
     return _generate_reports(
-        run=run, format=format, compare=compare, baseline=baseline, output=output
+        run=run,
+        format=format,
+        compare=compare,
+        baseline=baseline,
+        output=output,
+        style=style,
+        no_color=no_color,
+        summary_baseline_seconds=summary_baseline_seconds,
+        summary_subject_seconds=summary_subject_seconds,
+        summary_report_start=summary_report_start,
     )
 
 
@@ -254,16 +475,16 @@ def _load_run_report(path: str) -> dict:
 
 # Subcommands wired from existing modules
 @report_app.command(
-    name="verify", help="Recompute and verify metrics for a report/cert."
+    name="verify", help="Recompute and verify metrics for evaluation reports."
 )
 def report_verify_command(
-    certificates: list[str] = typer.Argument(
-        ..., help="One or more certificate JSON files to verify."
+    reports: list[str] = typer.Argument(
+        ..., help="One or more evaluation report JSON files to verify."
     ),
     baseline: str | None = typer.Option(
         None,
         "--baseline",
-        help="Optional baseline certificate/report JSON to enforce provider parity.",
+        help="Optional baseline evaluation report JSON to enforce provider parity.",
     ),
     tolerance: float = typer.Option(
         1e-9, "--tolerance", help="Tolerance for analysis-basis comparisons."
@@ -278,10 +499,10 @@ def report_verify_command(
 
     from .verify import verify_command as _verify_command
 
-    cert_paths = [_Path(c) for c in certificates]
+    report_paths = [_Path(p) for p in reports]
     baseline_path = _Path(baseline) if isinstance(baseline, str) else None
     return _verify_command(
-        certificates=cert_paths,
+        reports=report_paths,
         baseline=baseline_path,
         tolerance=tolerance,
         profile=profile,
@@ -289,7 +510,7 @@ def report_verify_command(
 
 
 @report_app.command(
-    name="explain", help="Explain certificate gates for report vs baseline."
+    name="explain", help="Explain evaluation report gates for report vs baseline."
 )
 def report_explain(
     report: str = typer.Option(..., "--report", help="Path to primary report.json"),
@@ -297,15 +518,17 @@ def report_explain(
         ..., "--baseline", help="Path to baseline report.json"
     ),
 ):  # pragma: no cover - thin wrapper
-    """Explain certificate gates for a report vs baseline."""
+    """Explain evaluation report gates for a report vs baseline."""
     from .explain_gates import explain_gates_command as _explain
 
     return _explain(report=report, baseline=baseline)
 
 
-@report_app.command(name="html", help="Render a certificate JSON to HTML.")
+@report_app.command(name="html", help="Render an evaluation report JSON to HTML.")
 def report_html(
-    input: str = typer.Option(..., "--input", "-i", help="Path to certificate JSON"),
+    input: str = typer.Option(
+        ..., "--input", "-i", help="Path to evaluation report JSON"
+    ),
     output: str = typer.Option(..., "--output", "-o", help="Path to output HTML file"),
     embed_css: bool = typer.Option(
         True, "--embed-css/--no-embed-css", help="Inline a minimal static stylesheet"
@@ -322,32 +545,43 @@ def report_html(
 @report_app.command("validate")
 def report_validate(
     report: str = typer.Argument(
-        ..., help="Path to certificate JSON to validate against schema v1"
+        ..., help="Path to evaluation report JSON to validate against schema v1"
     ),
 ):
-    """Validate a certificate JSON against the current schema (v1)."""
+    """Validate an evaluation report JSON against the current schema (v1)."""
+    output_style = resolve_output_style(
+        style="audit",
+        profile="ci",
+        progress=False,
+        timing=False,
+        no_color=False,
+    )
+
+    def _event(tag: str, message: str, *, emoji: str | None = None) -> None:
+        print_event(console, tag, message, style=output_style, emoji=emoji)
+
     p = Path(report)
     try:
         payload = json.loads(p.read_text(encoding="utf-8"))
     except Exception as exc:  # noqa: BLE001
-        console.print(f"[red]❌ Failed to read input JSON: {exc}[/red]")
+        _event("FAIL", f"Failed to read input JSON: {exc}", emoji="❌")
         raise typer.Exit(1) from exc
 
     try:
-        from invarlock.reporting.certificate import validate_certificate
+        from invarlock.reporting.report_builder import validate_report
 
-        ok = validate_certificate(payload)
+        ok = validate_report(payload)
         if not ok:
-            console.print("[red]❌ Certificate schema validation failed[/red]")
+            _event("FAIL", "Evaluation report schema validation failed", emoji="❌")
             raise typer.Exit(2)
-        console.print("✅ Certificate schema is valid")
+        _event("PASS", "Evaluation report schema is valid", emoji="✅")
     except ValueError as exc:
-        console.print(f"[red]❌ Certificate validation error: {exc}[/red]")
+        _event("FAIL", f"Evaluation report validation error: {exc}", emoji="❌")
         raise typer.Exit(2) from exc
     except typer.Exit:
         raise
     except Exception as exc:  # noqa: BLE001
-        console.print(f"[red]❌ Validation failed: {exc}[/red]")
+        _event("FAIL", f"Validation failed: {exc}", emoji="❌")
         raise typer.Exit(1) from exc