rlshiftlab 0.2.0__py3-none-any.whl

rlshiftlab/report.py

@@ -0,0 +1,273 @@
+"""Write self-contained HTML reports for evaluations and direct comparisons."""
+
+from __future__ import annotations
+
+import base64
+import html
+import io
+import json
+from collections.abc import Sequence
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from matplotlib.backends.backend_agg import FigureCanvasAgg
+from matplotlib.figure import Figure
+
+if TYPE_CHECKING:
+    from .comparisons import ComparisonResults
+    from .groups import GroupResults
+    from .results import Results
+
+
+NOTICE = (
+    "Each curve varies one shift at a time on a frozen policy. "
+    "Differences are measured; this report does not establish their causes."
+)
+
+STYLE = """
+    :root { color-scheme: light; font-family: system-ui, sans-serif; }
+    body { max-width: 1100px; margin: 2rem auto; padding: 0 1rem; color: #18212b; line-height: 1.5; }
+    h1, h2, h3 { line-height: 1.2; }
+    section { margin: 2rem 0; }
+    dl { display: grid; grid-template-columns: max-content minmax(0, 1fr); gap: .35rem 1rem; }
+    dt { font-weight: 650; }
+    dd { margin: 0; overflow-wrap: anywhere; }
+    img { display: block; max-width: 100%; height: auto; }
+    pre { padding: 1rem; overflow-x: auto; background: #f3f5f7; border-radius: .4rem; }
+    .table-wrap { overflow-x: auto; }
+    table { border-collapse: collapse; width: 100%; font-size: .88rem; }
+    th, td { padding: .45rem .65rem; text-align: left; border-bottom: 1px solid #d8dee5; white-space: nowrap; }
+    th { background: #f3f5f7; }
+    .notice { border-left: .3rem solid #405d7a; padding: .7rem 1rem; background: #f3f5f7; }
+"""
+
+
+def _display(value: object) -> str:
+    """Escape a value before inserting it into HTML text or an attribute."""
+    if value is None:
+        return ""
+    if isinstance(value, (list, tuple, dict)):
+        value = json.dumps(value, ensure_ascii=False)
+    return html.escape(str(value), quote=True)
+
+
+def _figure_data(figure: Figure) -> str:
+    """Encode a headless Matplotlib figure as an inline PNG."""
+    image = io.BytesIO()
+    FigureCanvasAgg(figure).print_png(image)
+    return base64.b64encode(image.getvalue()).decode("ascii")
+
+
+def _image(figure: Figure, alt: str) -> str:
+    return (
+        f'<img alt="{_display(alt)}" '
+        f'src="data:image/png;base64,{_figure_data(figure)}">'
+    )
+
+
+def _table(rows: Sequence[dict[str, Any]], columns: Sequence[str]) -> str:
+    headings = "".join(f'<th scope="col">{_display(name)}</th>' for name in columns)
+    body = "\n".join(
+        "<tr>" + "".join(f"<td>{_display(row.get(name))}</td>" for name in columns) + "</tr>"
+        for row in rows
+    )
+    return f'<div class="table-wrap"><table><thead><tr>{headings}</tr></thead><tbody>{body}</tbody></table></div>'
+
+
+def _setup(meta: dict[str, Any]) -> str:
+    rows = "\n".join(
+        f"<dt>{_display(key)}</dt><dd>{_display(value)}</dd>"
+        for key, value in meta.items()
+    )
+    return f"<section><h2>Setup</h2><dl>{rows}</dl></section>"
+
+
+def _section(title: str, body: str) -> str:
+    return f"<section><h2>{_display(title)}</h2>{body}</section>"
+
+
+def _comparison_sections(comparisons: Sequence[ComparisonResults]) -> str:
+    from .exports import COMPARISON_COLUMNS
+
+    sections = []
+    for comparison in comparisons:
+        left = comparison.meta.get("left", "left")
+        right = comparison.meta.get("right", "right")
+        metric = comparison.meta.get("metric", "return")
+        title = f"Direct comparison: {left} - {right} ({metric})"
+        explanation = (
+            "<p>Positive means the left side has a larger measured metric; "
+            "larger is not necessarily better. Intervals are pointwise 95% "
+            "confidence intervals, not simultaneous or adjusted for multiple "
+            "comparisons. Evaluation episodes are paired by their reset seeds.</p>"
+        )
+        if comparison.meta.get("groups") is not None:
+            explanation += (
+                "<p>Training members on the two sides are resampled independently; "
+                "matching member ID strings do not create training pairs.</p>"
+            )
+        body = (
+            explanation
+            + _image(comparison.plot(), f"{left} minus {right} for {metric}, with zero reference")
+            + f"<pre>{_display(comparison.summary())}</pre>"
+            + _table(comparison.rows, COMPARISON_COLUMNS)
+        )
+        sections.append(_section(title, body))
+    return "\n".join(sections)
+
+
+def _single_body(results: Results, comparisons: Sequence[ComparisonResults]) -> str:
+    from .exports import METRIC_COLUMNS
+    from .results import CSV_COLUMNS
+
+    sections = [
+        _setup(results.meta),
+        _section(
+            "Degradation curves",
+            _image(
+                results.plot(),
+                "Mean return and 95% confidence intervals by shift level and policy",
+            ),
+        ),
+        _section("Summary", f"<pre>{_display(results.summary())}</pre>"),
+        _section("Full results", _table(results.rows, CSV_COLUMNS)),
+    ]
+    if results.metric_rows:
+        sections.append(
+            _section(
+                "Custom per-episode metrics",
+                "<p>Each row summarizes a scalar metric measured per evaluation episode. "
+                "Drop is nominal minus shifted for the same episode seed.</p>"
+                + _table(results.metric_rows, METRIC_COLUMNS),
+            )
+        )
+    if results.records:
+        sections.append(
+            _section(
+                "Episode audit",
+                f"<p>{len(results.records)} per-episode records are available through "
+                "<code>to_episode_csv()</code>. Arbitrary observations and environment "
+                "<code>info</code> values are not placed in this report.</p>",
+            )
+        )
+    if comparisons:
+        sections.append(_comparison_sections(comparisons))
+    return "\n".join(sections)
+
+
+def _group_body(results: GroupResults, comparisons: Sequence[ComparisonResults]) -> str:
+    from .exports import GROUP_COLUMNS, GROUP_METRIC_COLUMNS, MEMBER_COLUMNS
+
+    n_members = results.meta.get("n_members", {})
+    counts = ", ".join(f"{name}: {count}" for name, count in n_members.items())
+    explanation = (
+        "<p>Each group mean gives equal weight to each caller-provided frozen "
+        "policy member after averaging its evaluation episodes. "
+        "Observed member SD, minimum and maximum describe the spread of those "
+        "member means; that spread includes finite-evaluation noise and is not "
+        "a confidence interval or pure training variance. "
+        "The pointwise 95% confidence intervals use a crossed bootstrap over "
+        "training members and evaluation episode keys. They are approximate, "
+        "not simultaneous curve bands or adjusted for multiple comparisons. "
+        "Intervals with only a few members can be unstable.</p>"
+        f"<p>Members per group: {_display(counts)}.</p>"
+    )
+    sections = [
+        _setup(results.meta),
+        _section("Group interpretation", explanation),
+        _section(
+            "Group degradation curves",
+            _image(
+                results.plot(metric="return", band="ci"),
+                "Group mean return with pointwise crossed confidence intervals",
+            ),
+        ),
+        _section("Summary", f"<pre>{_display(results.summary())}</pre>"),
+        _section("Full group results", _table(results.rows, GROUP_COLUMNS)),
+        _section("Member summaries", _table(results.member_rows, MEMBER_COLUMNS)),
+    ]
+    if results.metric_rows:
+        sections.append(
+            _section(
+                "Custom per-episode metrics",
+                "<p>Custom metric means and drops use the same member and evaluation "
+                "episode units as the return summaries.</p>"
+                + _table(results.metric_rows, GROUP_METRIC_COLUMNS),
+            )
+        )
+    sections.append(
+        _section(
+            "Episode audit",
+            f"<p>{len(results.records)} per-episode records are retained. Use "
+            "<code>to_episode_csv()</code> for the full audit; raw observations "
+            "and environment <code>info</code> are not embedded here.</p>",
+        )
+    )
+    if comparisons:
+        sections.append(_comparison_sections(comparisons))
+    return "\n".join(sections)
+
+
+def _comparison_body(results: ComparisonResults) -> str:
+    return "\n".join(
+        [
+            _setup(results.meta),
+            _comparison_sections([results]),
+            _section(
+                "Interpretation",
+                "<p>Positive left-minus-right values mean a larger measured metric "
+                "for the left side, which is not always better. Intervals are "
+                "pointwise 95% confidence intervals. They do not establish "
+                "causes or account for simultaneous curve coverage or multiple "
+                "comparisons.</p>",
+            ),
+        ]
+    )
+
+
+def write_html(
+    results: Results | GroupResults | ComparisonResults,
+    path: str | Path,
+    comparisons: Sequence[ComparisonResults] | None = None,
+) -> None:
+    """Write one standalone report for the supplied result and comparisons."""
+    from .comparisons import ComparisonResults
+    from .groups import GroupResults
+    from .results import Results
+
+    if isinstance(results, ComparisonResults):
+        if comparisons is not None:
+            raise ValueError("standalone comparison report does not accept nested comparisons")
+        title = "RLShiftLab direct comparison report"
+        body = _comparison_body(results)
+    elif isinstance(results, GroupResults):
+        title = "RLShiftLab group degradation report"
+        selected = [] if comparisons is None else list(comparisons)
+        if any(not isinstance(item, ComparisonResults) for item in selected):
+            raise TypeError("comparisons must contain ComparisonResults objects")
+        body = _group_body(results, selected)
+    elif isinstance(results, Results):
+        title = "RLShiftLab degradation report"
+        selected = [] if comparisons is None else list(comparisons)
+        if any(not isinstance(item, ComparisonResults) for item in selected):
+            raise TypeError("comparisons must contain ComparisonResults objects")
+        body = _single_body(results, selected)
+    else:
+        raise TypeError("results must be Results, GroupResults, or ComparisonResults")
+
+    document = f"""<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>{_display(title)}</title>
+  <style>{STYLE}</style>
+</head>
+<body>
+  <h1>{_display(title)}</h1>
+  <p class="notice">{_display(NOTICE)}</p>
+  {body}
+</body>
+</html>
+"""
+    Path(path).write_text(document, encoding="utf-8")

rlshiftlab/results.py

@@ -0,0 +1,200 @@
+"""Evaluation records, summaries, CSV output, and degradation plots."""
+
+from __future__ import annotations
+
+import csv
+from dataclasses import dataclass, field
+from math import ceil
+from os import PathLike
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from matplotlib.backends.backend_agg import FigureCanvasAgg
+from matplotlib.figure import Figure
+
+from rlshiftlab.records import EpisodeRecord
+
+if TYPE_CHECKING:
+    from rlshiftlab.comparisons import ComparisonResults
+
+CSV_COLUMNS = (
+    "policy",
+    "shift",
+    "level",
+    "n_episodes",
+    "mean_return",
+    "return_ci_low",
+    "return_ci_high",
+    "drop",
+    "drop_ci_low",
+    "drop_ci_high",
+    "mean_length",
+    "success_rate",
+    "success_ci_low",
+    "success_ci_high",
+)
+
+
+@dataclass
+class Results:
+    """Rows of evaluated conditions and metadata for later presentation."""
+
+    rows: list[dict[str, Any]]
+    meta: dict[str, Any]
+    records: list[EpisodeRecord] = field(default_factory=list)
+    metric_rows: list[dict[str, Any]] = field(default_factory=list)
+
+    def to_csv(self, path: str | PathLike[str]) -> None:
+        """Write tidy, deterministic results with the public column order."""
+        with Path(path).open("w", encoding="utf-8", newline="") as output:
+            writer = csv.DictWriter(output, fieldnames=CSV_COLUMNS, lineterminator="\n")
+            writer.writeheader()
+            for row in self.rows:
+                writer.writerow(
+                    {
+                        column: (
+                            "" if row.get(column) is None else repr(row[column])
+                            if isinstance(row[column], float)
+                            else row[column]
+                        )
+                        for column in CSV_COLUMNS
+                    }
+                )
+
+    def to_episode_csv(self, path: str | PathLike[str]) -> None:
+        """Write one scalar audit row per evaluation episode."""
+        from rlshiftlab.exports import episode_columns, episode_rows, write_rows
+
+        write_rows(path, episode_rows(self.records, self.meta), episode_columns(self.meta))
+
+    def to_metrics_csv(self, path: str | PathLike[str]) -> None:
+        """Write custom metric summaries in a separate long-form CSV."""
+        from rlshiftlab.exports import METRIC_COLUMNS, write_rows
+
+        write_rows(path, self.metric_rows, METRIC_COLUMNS)
+
+    def summary(self) -> str:
+        """Summarize nominal returns and the strongest level of each shift."""
+        headings = ("policy", "shift", "level", "mean return", "drop (95% CI)", "nominal mean")
+        lines = []
+        policies = self.meta.get("policies") or list(dict.fromkeys(row["policy"] for row in self.rows))
+        shifts = self.meta.get("shifts") or {
+            name: [] for name in dict.fromkeys(row["shift"] for row in self.rows if row["shift"] != "nominal")
+        }
+        for policy in policies:
+            nominal = next(
+                (row for row in self.rows if row["policy"] == policy and row["shift"] == "nominal"),
+                None,
+            )
+            if nominal is None:
+                continue
+            nominal_mean = float(nominal["mean_return"])
+            for shift in shifts:
+                matching = [
+                    row for row in self.rows if row["policy"] == policy and row["shift"] == shift
+                ]
+                if not matching:
+                    continue
+                strongest = max(matching, key=lambda row: float(row["level"]))
+                drop_text = (
+                    f"{float(strongest['drop']):.3f} "
+                    f"[{float(strongest['drop_ci_low']):.3f}, "
+                    f"{float(strongest['drop_ci_high']):.3f}]"
+                )
+                lines.append(
+                    (
+                        str(policy),
+                        str(shift),
+                        f"{float(strongest['level']):g}",
+                        f"{float(strongest['mean_return']):.3f}",
+                        drop_text,
+                        f"{nominal_mean:.3f}",
+                    )
+                )
+        widths = [
+            max([len(heading), *(len(line[index]) for line in lines)])
+            for index, heading in enumerate(headings)
+        ]
+        header = "  ".join(heading.ljust(widths[index]) for index, heading in enumerate(headings))
+        rule = "  ".join("-" * width for width in widths)
+        body = ["  ".join(value.ljust(widths[index]) for index, value in enumerate(line)) for line in lines]
+        return "\n".join([header, rule, *body])
+
+    def plot(
+        self, path: str | PathLike[str] | None = None, *, metric: str = "return"
+    ) -> Figure:
+        """Plot mean returns or a custom metric against each shift level."""
+        if metric == "return":
+            plot_rows = self.rows
+            mean_key, low_key, high_key = "mean_return", "return_ci_low", "return_ci_high"
+            y_label = "mean return (95% CI)"
+        else:
+            configured = [
+                *self.meta.get("step_metrics", ()), *self.meta.get("episode_metrics", ())
+            ]
+            if metric not in configured:
+                raise ValueError(f"custom metric {metric!r} is not configured")
+            plot_rows = [row for row in self.metric_rows if row["metric"] == metric]
+            mean_key, low_key, high_key = "mean", "ci_low", "ci_high"
+            y_label = f"mean {metric} (95% CI)"
+        shifts = list(
+            self.meta.get("shifts")
+            or dict.fromkeys(row["shift"] for row in self.rows if row["shift"] != "nominal")
+        )
+        if not shifts:
+            raise ValueError("results contain no shifted conditions to plot")
+        policies = self.meta.get("policies") or list(dict.fromkeys(row["policy"] for row in self.rows))
+        columns = min(3, len(shifts))
+        rows = ceil(len(shifts) / columns)
+        figure = Figure(figsize=(5.5 * columns, 4 * rows))
+        FigureCanvasAgg(figure)
+        for index, shift in enumerate(shifts, start=1):
+            axis = figure.add_subplot(rows, columns, index)
+            for policy in policies:
+                nominal = next(
+                    (
+                        row for row in plot_rows if row["policy"] == policy and row["shift"] == "nominal"
+                    ),
+                    None,
+                )
+                shifted = sorted(
+                    (row for row in plot_rows if row["policy"] == policy and row["shift"] == shift),
+                    key=lambda row: float(row["level"]),
+                )
+                if nominal is None or not shifted:
+                    continue
+                points = [nominal, *shifted]
+                x = [float(point["level"]) for point in points]
+                means = [float(point[mean_key]) for point in points]
+                lows = [float(point[low_key]) for point in points]
+                highs = [float(point[high_key]) for point in points]
+                line = axis.plot(x, means, marker="o", label=str(policy))[0]
+                axis.fill_between(x, lows, highs, color=line.get_color(), alpha=0.18)
+            axis.set_title(shift.replace("_", " "))
+            axis.set_xlabel("shift strength")
+            axis.set_ylabel(y_label)
+            axis.grid(alpha=0.2)
+            axis.legend()
+        figure.tight_layout()
+        if path is not None:
+            figure.savefig(path, dpi=150)
+        return figure
+
+    def to_html(
+        self, path: str | PathLike[str], *, comparisons: Any = None
+    ) -> None:
+        """Write a standalone HTML report containing this result set."""
+        from rlshiftlab.report import write_html
+
+        if comparisons is None:
+            write_html(self, path)
+        else:
+            write_html(self, path, comparisons=comparisons)
+
+    def compare(
+        self, left: str, right: str, metric: str = "return"
+    ) -> ComparisonResults:
+        """Compare two policies on matched evaluation episode keys."""
+        from rlshiftlab.comparisons import compare_results
+
+        return compare_results(self, left, right, metric=metric, grouped=False)