dataeval-plots 0.0.14__tar.gz → 0.0.16__tar.gz

{dataeval_plots-0.0.14 → dataeval_plots-0.0.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: dataeval-plots
-Version: 0.0.14
+Version: 0.0.16
 Summary: DataEval companion package for plotting utilities
 Project-URL: Homepage, https://dataeval.ai/
 Project-URL: Repository, https://github.com/aria-ml/dataeval/
@@ -238,7 +238,13 @@ up, so rounding never overstates what the evidence shows. Requirements and confi
 "95.5%" or "at 97.5% joint confidence".
 
 When a ceiling bounds the band, the caption ends with the footnote "\* Ruled out under the recorded assumptions, at
-the stated confidence." A group-based ceiling adds "(group-based ceiling; validity pending lead-scientist sign-off)".
+the stated confidence."
+
+A recall requirement is titled "Required recall for Tank (at most 5% false positives)". Only a witness whose own upper
+confidence bound on false positives is within that limit counts; the caption adds its bound after the observed
+sentence, for example "Its upper confidence bound on false positives is 4.1% (observed 2.6% of 651 other images)." In
+the evidence view, a witness over the limit keeps its recall bound and observed score but is drawn faint, its label
+saying why it is not counted, for example "66.0% lower bound · not counted: false positives ≤8.3% (limit 5%)".
 
 A ceiling that applies only to narrower models than the band judges never draws a zone. It appears as a note instead,
 for example "Group-based ceiling (72.1%) applies only to models that give one answer per group."

{dataeval_plots-0.0.14 → dataeval_plots-0.0.16}/README.md

@@ -192,7 +192,13 @@ up, so rounding never overstates what the evidence shows. Requirements and confi
 "95.5%" or "at 97.5% joint confidence".
 
 When a ceiling bounds the band, the caption ends with the footnote "\* Ruled out under the recorded assumptions, at
-the stated confidence." A group-based ceiling adds "(group-based ceiling; validity pending lead-scientist sign-off)".
+the stated confidence."
+
+A recall requirement is titled "Required recall for Tank (at most 5% false positives)". Only a witness whose own upper
+confidence bound on false positives is within that limit counts; the caption adds its bound after the observed
+sentence, for example "Its upper confidence bound on false positives is 4.1% (observed 2.6% of 651 other images)." In
+the evidence view, a witness over the limit keeps its recall bound and observed score but is drawn faint, its label
+saying why it is not counted, for example "66.0% lower bound · not counted: false positives ≤8.3% (limit 5%)".
 
 A ceiling that applies only to narrower models than the band judges never draws a zone. It appears as a note instead,
 for example "Group-based ceiling (72.1%) applies only to models that give one answer per group."

{dataeval_plots-0.0.14 → dataeval_plots-0.0.16}/src/dataeval_plots/__init__.py

@@ -19,6 +19,7 @@ from dataeval_plots.protocols import (
     Dataset,
     PlottableBalance,
     PlottableDiversity,
+    PlottableDrift,
     PlottableDriftMVDC,
     PlottableFeasibility,
     PlottableStats,
@@ -98,6 +99,15 @@ def plot(
 ) -> Any: ...
 
 
+@overload
+def plot(
+    output: PlottableDrift,
+    /,
+    figsize: tuple[float, float] | None = None,
+    backend: str | None = None,
+) -> Any: ...
+
+
 @overload
 def plot(
     output: PlottableFeasibility,

{dataeval_plots-0.0.14 → dataeval_plots-0.0.16}/src/dataeval_plots/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.0.14'
-__version_tuple__ = version_tuple = (0, 0, 14)
+__version__ = version = '0.0.16'
+__version_tuple__ = version_tuple = (0, 0, 16)
 
 __commit_id__ = commit_id = None

{dataeval_plots-0.0.14 → dataeval_plots-0.0.16}/src/dataeval_plots/backends/_altair.py

@@ -29,6 +29,7 @@ from dataeval_plots.backends._shared import (
     normalize_reference_outputs,
     parse_dataset_item,
     prepare_balance_data,
+    prepare_chunked_drift,
     prepare_diversity_data,
     prepare_drift_data,
     prepare_feasibility_data,
@@ -39,6 +40,7 @@ from dataeval_plots.protocols import (
     Dataset,
     PlottableBalance,
     PlottableDiversity,
+    PlottableDrift,
     PlottableDriftMVDC,
     PlottableFeasibility,
     PlottableStats,
@@ -819,6 +821,82 @@ class AltairBackend(BasePlottingBackend):
             width=width, height=height, title="Domain Classifier, Drift Detection"
         )
 
+    def _plot_drift(
+        self,
+        output: PlottableDrift,
+        figsize: tuple[float, float] | None = None,
+    ) -> Any:  # alt.LayerChart
+        """
+        Render the metric of each chunk in relation to the thresholds of the detector.
+
+        When the output carries the metric of the reference chunks, they are drawn in gray to the left of the
+        analysis chunks. The chunks are numbered from -n for the reference and from 0 for the analysis.
+
+        Parameters
+        ----------
+        output : PlottableDrift
+            The chunked drift output object to plot
+        figsize : tuple[float, float] or None, default None
+            Figure size in inches (width, height), drawn at 100 pixels per inch. If None, defaults to (6, 3).
+
+        Returns
+        -------
+        alt.LayerChart
+        """
+        import altair as alt
+        import pandas as pd
+
+        data = prepare_chunked_drift(output)
+        width_inches, height_inches = figsize or (6, 3)
+
+        # Convert via dict to avoid pyarrow dependency
+        plot_df = pd.DataFrame(
+            data.frame.with_columns(
+                pl.when(pl.col("period") == "reference")
+                .then(pl.lit("Reference chunks"))
+                .otherwise(pl.lit("Analysis chunks"))
+                .alias("period")
+            ).to_dict(as_series=False)
+        )
+
+        x = alt.X("index:Q", title=data.chunk_label, axis=alt.Axis(tickMinStep=1))
+        scale = alt.Scale(zero=False)
+        y = alt.Y("value:Q", title=data.metric_name, scale=scale)
+        tooltip = ["index:Q", alt.Tooltip("value:Q", title=data.metric_name, format=".4f")]
+
+        # The no-drift range is the band between the two thresholds
+        layers = [
+            alt.Chart(plot_df)
+            .mark_area(color="green", opacity=0.15, interpolate="step")
+            .encode(x=x, y=alt.Y("lower_threshold:Q", title=data.metric_name, scale=scale), y2="upper_threshold:Q")
+        ]
+
+        # The metric of each chunk, with the reference chunks in gray
+        has_reference = bool((plot_df["period"] == "Reference chunks").any())
+        color = alt.Color(
+            "period:N",
+            scale=alt.Scale(domain=["Reference chunks", "Analysis chunks"], range=["gray", "royalblue"]),
+            legend=alt.Legend(title=None),
+        )
+        layers.append(alt.Chart(plot_df).mark_line(point=True).encode(x=x, y=y, color=color, tooltip=tooltip))
+        if has_reference:
+            layers.append(
+                alt.Chart(pd.DataFrame({"index": [-0.5]}))
+                .mark_rule(color="gray", strokeDash=[2, 2])
+                .encode(x="index:Q")
+            )
+
+        # The chunks that the detector flagged
+        layers.append(
+            alt.Chart(plot_df[plot_df["drifted"]])  # type: ignore[arg-type]
+            .mark_point(shape="diamond", size=120, color="crimson", filled=True)
+            .encode(x=x, y=y, tooltip=tooltip)
+        )
+
+        return alt.layer(*layers).properties(
+            width=int(width_inches * 100), height=int(height_inches * 100), title="Drift by chunk"
+        )
+
     def _plot_image_grid(
         self,
         dataset: Dataset,

{dataeval_plots-0.0.14 → dataeval_plots-0.0.16}/src/dataeval_plots/backends/_base.py

@@ -14,6 +14,7 @@ from dataeval_plots.protocols import (
     Dataset,
     PlottableBalance,
     PlottableDiversity,
+    PlottableDrift,
     PlottableDriftMVDC,
     PlottableFeasibility,
     PlottableStats,
@@ -81,6 +82,14 @@ class PlottingBackend(Protocol):
         figsize: tuple[float, float] | None = None,
     ) -> Any: ...
 
+    @overload
+    def plot(
+        self,
+        output: PlottableDrift,
+        *,
+        figsize: tuple[float, float] | None = None,
+    ) -> Any: ...
+
     @overload
     def plot(
         self,
@@ -245,6 +254,8 @@ class BasePlottingBackend(PlottingBackend, ABC):
             return self._plot_sufficiency(cast(PlottableSufficiency, output), figsize=figsize, **kwargs)
         if plot_type == "drift_mvdc":
             return self._plot_drift_mvdc(cast(PlottableDriftMVDC, output), figsize=figsize, **kwargs)
+        if plot_type == "drift":
+            return self._plot_drift(cast(PlottableDrift, output), figsize=figsize, **kwargs)
         if plot_type == "stats":
             return self._plot_stats(cast(PlottableStats, output), figsize=figsize, **kwargs)
         if plot_type == "feasibility":
@@ -310,6 +321,15 @@ class BasePlottingBackend(PlottingBackend, ABC):
         """Plot drift MVDC output."""
         ...
 
+    @abstractmethod
+    def _plot_drift(
+        self,
+        output: PlottableDrift,
+        figsize: tuple[float, float] | None = None,
+    ) -> Any:
+        """Plot the per-chunk results of a chunked drift detector."""
+        ...
+
     @abstractmethod
     def _plot_feasibility(
         self,

{dataeval_plots-0.0.14 → dataeval_plots-0.0.16}/src/dataeval_plots/backends/_matplotlib.py

@@ -21,8 +21,10 @@ from dataeval_plots.backends._shared import (
     calculate_subplot_grid,
     feasibility_zones,
     normalize_reference_outputs,
+    plot_chunked_drift_on_axis,
     plot_drift_on_axis,
     prepare_balance_data,
+    prepare_chunked_drift,
     prepare_diversity_data,
     prepare_drift_data,
     prepare_feasibility_data,
@@ -32,6 +34,7 @@ from dataeval_plots.backends._shared import (
 from dataeval_plots.protocols import (
     PlottableBalance,
     PlottableDiversity,
+    PlottableDrift,
     PlottableDriftMVDC,
     PlottableFeasibility,
     PlottableStats,
@@ -833,6 +836,35 @@ class MatplotlibBackend(BasePlottingBackend):
             )
         return fig
 
+    def _plot_drift(
+        self,
+        output: PlottableDrift,
+        figsize: tuple[float, float] | None = None,
+    ) -> Figure:
+        """
+        Render the metric of each chunk in relation to the thresholds of the detector.
+
+        Parameters
+        ----------
+        output : PlottableDrift
+            The chunked drift output object to plot
+        figsize : tuple[float, float] or None, default None
+            Figure size in inches (width, height). If None, defaults to (8, 3.5).
+
+        Returns
+        -------
+        matplotlib.figure.Figure
+        """
+        import matplotlib.pyplot as plt
+
+        data = prepare_chunked_drift(output)
+
+        fig, ax = plt.subplots(figsize=figsize or (8, 3.5))
+        plot_chunked_drift_on_axis(ax, data)
+        ax.set_title("Drift by chunk")
+        fig.tight_layout()
+        return fig
+
     def _plot_feasibility(
         self,
         output: PlottableFeasibility,

{dataeval_plots-0.0.14 → dataeval_plots-0.0.16}/src/dataeval_plots/backends/_plotly.py

@@ -22,6 +22,7 @@ from dataeval_plots.backends._shared import (
     feasibility_zones,
     format_label_from_target,
     prepare_balance_data,
+    prepare_chunked_drift,
     prepare_diversity_data,
     prepare_drift_data,
     prepare_feasibility_data,
@@ -33,6 +34,7 @@ from dataeval_plots.protocols import (
     Dataset,
     PlottableBalance,
     PlottableDiversity,
+    PlottableDrift,
     PlottableDriftMVDC,
     PlottableFeasibility,
     PlottableStats,
@@ -849,6 +851,113 @@ class PlotlyBackend(BasePlottingBackend):
 
         return fig
 
+    def _plot_drift(
+        self,
+        output: PlottableDrift,
+        figsize: tuple[float, float] | None = None,
+    ) -> Any:  # go.Figure
+        """
+        Render the metric of each chunk in relation to the thresholds of the detector.
+
+        When the output carries the metric of the reference chunks, they are drawn in gray to the left of the
+        analysis chunks. The chunks are numbered from -n for the reference and from 0 for the analysis.
+
+        Parameters
+        ----------
+        output : PlottableDrift
+            The chunked drift output object to plot
+        figsize : tuple[float, float] or None, default None
+            Figure size in inches (width, height), drawn at 100 pixels per inch. If None, defaults to (8, 3.5).
+
+        Returns
+        -------
+        plotly.graph_objects.Figure
+        """
+        import plotly.graph_objects as go
+
+        data = prepare_chunked_drift(output)
+        frame, reference, analysis = data.frame, data.reference, data.analysis
+        drifted = analysis.filter(pl.col("drifted"))
+        hover = f"Chunk: %{{x}}<br>{data.metric_name}: %{{y:.4f}}<extra></extra>"
+
+        fig = go.Figure()
+
+        # The no-drift range is the band between the two thresholds
+        fig.add_trace(
+            go.Scatter(
+                x=frame["index"].to_numpy(),
+                y=frame["upper_threshold"].to_numpy(),
+                mode="lines",
+                line={"width": 0, "shape": "hvh"},
+                showlegend=False,
+                hoverinfo="skip",
+            )
+        )
+        fig.add_trace(
+            go.Scatter(
+                x=frame["index"].to_numpy(),
+                y=frame["lower_threshold"].to_numpy(),
+                mode="lines",
+                line={"width": 0, "shape": "hvh"},
+                fill="tonexty",
+                fillcolor="rgba(44, 160, 44, 0.15)",
+                name="No-drift range",
+                hoverinfo="skip",
+            )
+        )
+
+        # The reference chunks, which the thresholds derive from
+        if len(reference):
+            fig.add_trace(
+                go.Scatter(
+                    x=reference["index"].to_numpy(),
+                    y=reference["value"].to_numpy(),
+                    mode="lines+markers",
+                    name="Reference chunks",
+                    line={"color": "gray", "width": 2},
+                    hovertemplate=hover,
+                )
+            )
+            fig.add_vline(x=-0.5, line_dash="dot", line_color="gray", line_width=1)
+
+        # The metric of each analysis chunk
+        fig.add_trace(
+            go.Scatter(
+                x=analysis["index"].to_numpy(),
+                y=analysis["value"].to_numpy(),
+                mode="lines+markers",
+                name="Analysis chunks",
+                line={"color": "royalblue", "width": 2},
+                hovertemplate=hover,
+            )
+        )
+
+        # The chunks that the detector flagged
+        if len(drifted):
+            fig.add_trace(
+                go.Scatter(
+                    x=drifted["index"].to_numpy(),
+                    y=drifted["value"].to_numpy(),
+                    mode="markers",
+                    name="Drift",
+                    marker={"symbol": "diamond", "size": 11, "color": "crimson"},
+                    hovertemplate=hover,
+                )
+            )
+
+        width_inches, height_inches = figsize or (8, 3.5)
+        fig.update_layout(
+            title="Drift by chunk",
+            xaxis_title=data.chunk_label,
+            yaxis_title=data.metric_name,
+            xaxis={"dtick": 1} if len(frame) <= 20 else {},
+            width=int(width_inches * 100),
+            height=int(height_inches * 100),
+            hovermode="closest",
+        )
+
+        return fig
+
     def _plot_image_grid(
         self,
         dataset: Dataset,

{dataeval_plots-0.0.14 → dataeval_plots-0.0.16}/src/dataeval_plots/backends/_seaborn.py

@@ -10,14 +10,17 @@ from numpy.typing import NDArray
 from dataeval_plots.backends._base import BasePlottingBackend
 from dataeval_plots.backends._shared import (
     CHANNELWISE_METRICS,
+    plot_chunked_drift_on_axis,
     plot_drift_on_axis,
     prepare_balance_data,
+    prepare_chunked_drift,
     prepare_diversity_data,
     prepare_drift_data,
 )
 from dataeval_plots.protocols import (
     PlottableBalance,
     PlottableDiversity,
+    PlottableDrift,
     PlottableDriftMVDC,
     PlottableFeasibility,
     PlottableStats,
@@ -493,6 +496,47 @@ class SeabornBackend(BasePlottingBackend):
         fig.tight_layout()
         return fig
 
+    def _plot_drift(
+        self,
+        output: PlottableDrift,
+        figsize: tuple[float, float] | None = None,
+    ) -> Figure:
+        """
+        Render the metric of each chunk in relation to the thresholds of the detector.
+
+        Parameters
+        ----------
+        output : PlottableDrift
+            The chunked drift output object to plot
+        figsize : tuple[float, float] or None, default None
+            Figure size in inches (width, height). If None, defaults to (10, 4).
+
+        Returns
+        -------
+        matplotlib.figure.Figure
+        """
+        import matplotlib.pyplot as plt
+        import seaborn as sns
+
+        sns.set_style("whitegrid")
+        palette = sns.color_palette("deep")  # not the global palette, which a user can shorten
+
+        data = prepare_chunked_drift(output)
+
+        fig, ax = plt.subplots(figsize=figsize or (10, 4))
+        plot_chunked_drift_on_axis(
+            ax,
+            data,
+            line_color=palette[0],
+            band_color=palette[2],
+            drift_color=palette[3],
+            reference_color=palette[7],
+        )
+        ax.set_title("Drift by chunk")
+        sns.despine(ax=ax)
+        fig.tight_layout()
+        return fig
+
     def _plot_feasibility(
         self,
         output: PlottableFeasibility,

{dataeval_plots-0.0.14 → dataeval_plots-0.0.16}/src/dataeval_plots/backends/_shared.py

@@ -19,6 +19,7 @@ if TYPE_CHECKING:
     from dataeval_plots.protocols import (
         PlottableBalance,
         PlottableDiversity,
+        PlottableDrift,
         PlottableDriftMVDC,
         PlottableFeasibility,
     )
@@ -46,6 +47,10 @@ __all__ = [
     "prepare_diversity_data",
     "prepare_drift_data",
     "plot_drift_on_axis",
+    "CHUNKED_DRIFT_COLUMNS",
+    "ChunkedDriftData",
+    "prepare_chunked_drift",
+    "plot_chunked_drift_on_axis",
     "FEASIBILITY_COLORS",
     "FEASIBILITY_CONFLICT_TEXT",
     "FeasibilityBand",
@@ -498,6 +503,200 @@ def prepare_drift_data(
     return resdf, trndf, tstdf, driftx, True
 
 
+CHUNKED_DRIFT_COLUMNS = ("index", "value", "lower_threshold", "upper_threshold", "drifted")
+
+
+@dataclass(frozen=True)
+class ChunkedDriftData:
+    """
+    The per-chunk results of a chunked drift detector, ready for plotting.
+
+    Attributes
+    ----------
+    frame : pl.DataFrame
+        One row per chunk with the columns ``index``, ``value``, ``lower_threshold``, ``upper_threshold``,
+        ``drifted`` and ``period``. The rows of the reference chunks come first, numbered from ``-n`` to ``-1``
+        so that the analysis chunks keep their own indexes, and have ``period`` set to ``"reference"``. The
+        analysis chunks have ``period`` set to ``"analysis"``.
+    metric_name : str
+        Name of the metric that ``value`` reports
+    chunk_label : str
+        Label for the axis of chunks, which includes the size of a chunk when every analysis chunk has the same size
+    """
+
+    frame: pl.DataFrame
+    metric_name: str
+    chunk_label: str
+
+    @property
+    def reference(self) -> pl.DataFrame:
+        """The rows of the reference chunks, which is empty when the output carries no baseline."""
+        return self.frame.filter(pl.col("period") == "reference")
+
+    @property
+    def analysis(self) -> pl.DataFrame:
+        """The rows of the analysis chunks."""
+        return self.frame.filter(pl.col("period") == "analysis")
+
+
+def prepare_chunked_drift(output: PlottableDrift) -> ChunkedDriftData:
+    """
+    Read the per-chunk table, the reference chunks and the metric name from a chunked drift output.
+
+    The reference chunks are the optional ``baseline`` attribute of the output, which holds the metric of each
+    reference chunk. An output without it, such as one from an earlier version of DataEval, has no reference rows
+    and the plot leaves them out.
+
+    Parameters
+    ----------
+    output : PlottableDrift
+        The drift output object to plot
+
+    Returns
+    -------
+    ChunkedDriftData
+        The per-chunk results. A threshold that is missing, as for a one-sided detector, is replaced by a bound just
+        beyond the data, so that the range of no drift has no limit on that side.
+
+    Raises
+    ------
+    ValueError
+        If the output holds the statistics of a single test and not a per-chunk table, or if the table lacks a column
+        that the plot needs
+    """
+    details = output.details
+    if not isinstance(details, pl.DataFrame):
+        raise ValueError("Plotting drift needs chunked results. Call .chunked() on the detector before predict().")
+    missing = [name for name in CHUNKED_DRIFT_COLUMNS if name not in details.columns]
+    if missing:
+        raise ValueError(f"The per-chunk drift results lack the columns {missing}.")
+
+    analysis = details.select(
+        pl.col("index").cast(pl.Int64),
+        pl.col("value", "lower_threshold", "upper_threshold").cast(pl.Float64),
+        pl.col("drifted").cast(pl.Boolean),
+        pl.lit("analysis").alias("period"),
+    )
+
+    # The thresholds come from the reference chunks, so the band spans them too
+    baseline = np.asarray(getattr(output, "baseline", None) or (), dtype=np.float64)
+    if len(baseline) and len(analysis):
+        reference = pl.DataFrame(
+            {
+                "index": np.arange(-len(baseline), 0, dtype=np.int64),
+                "value": baseline,
+                "lower_threshold": [analysis["lower_threshold"][0]] * len(baseline),
+                "upper_threshold": [analysis["upper_threshold"][0]] * len(baseline),
+                "drifted": [False] * len(baseline),
+                "period": ["reference"] * len(baseline),
+            },
+            schema=analysis.schema,
+        )
+        frame = pl.concat([reference, analysis])
+    else:
+        frame = analysis
+
+    if frame["lower_threshold"].null_count() or frame["upper_threshold"].null_count():
+        extent = pl.concat([frame["value"], frame["lower_threshold"], frame["upper_threshold"]]).drop_nulls()
+        low, high = extent.min(), extent.max()
+        pad = 0.1 * (high - low) if low != high else 1.0  # type: ignore[operator]
+        frame = frame.with_columns(
+            pl.col("lower_threshold").fill_null(low - pad),  # type: ignore[operator]
+            pl.col("upper_threshold").fill_null(high + pad),  # type: ignore[operator]
+        )
+
+    return ChunkedDriftData(frame, output.metric_name, _chunk_label(details))
+
+
+def _chunk_label(details: pl.DataFrame) -> str:
+    """Label the axis of chunks with the size of a chunk, when every chunk has the same size."""
+    if "start_index" in details.columns and "end_index" in details.columns and len(details):
+        sizes = (details["end_index"] - details["start_index"] + 1).unique()
+        if len(sizes) == 1 and (details["start_index"] >= 0).all():
+            return f"Chunk ({sizes[0]} samples each)"
+    return "Chunk"
+
+
+def plot_chunked_drift_on_axis(
+    ax: Any,
+    data: ChunkedDriftData,
+    line_color: Any = "tab:blue",
+    band_color: Any = "tab:green",
+    drift_color: Any = "tab:red",
+    reference_color: Any = "tab:gray",
+    linewidth: float = 1.5,
+) -> None:
+    """
+    Plot the metric of each chunk against the threshold range of the detector on a matplotlib axis.
+
+    This is a shared helper for the matplotlib and seaborn backends. The metric is a line with a marker per chunk,
+    the band is the range between the lower and upper thresholds in which a chunk counts as no drift, and a diamond
+    marks each chunk that the detector flagged. When the output carries the metric of the reference chunks, they are
+    drawn in a second color to the left of the analysis chunks, with a dotted line between them.
+
+    Parameters
+    ----------
+    ax : matplotlib.axes.Axes
+        Axis to plot on
+    data : ChunkedDriftData
+        Per-chunk results from :func:`prepare_chunked_drift`
+    line_color : Any, default "tab:blue"
+        Color of the metric line of the analysis chunks
+    band_color : Any, default "tab:green"
+        Color of the band between the thresholds
+    drift_color : Any, default "tab:red"
+        Color of the markers on the chunks with drift
+    reference_color : Any, default "tab:gray"
+        Color of the metric line of the reference chunks and of the line that separates them
+    linewidth : float, default 1.5
+        Width of the metric lines
+    """
+    from matplotlib.ticker import MaxNLocator
+
+    frame, reference, analysis = data.frame, data.reference, data.analysis
+    ax.fill_between(
+        frame["index"].to_numpy(),
+        frame["lower_threshold"].to_numpy(),
+        frame["upper_threshold"].to_numpy(),
+        step="mid",
+        color=band_color,
+        alpha=0.15,
+        label="No-drift range",
+    )
+    if len(reference):
+        ax.plot(
+            reference["index"].to_numpy(),
+            reference["value"].to_numpy(),
+            marker="o",
+            color=reference_color,
+            linewidth=linewidth,
+            label="Reference chunks",
+        )
+        ax.axvline(-0.5, color=reference_color, linestyle=":", linewidth=1)
+    ax.plot(
+        analysis["index"].to_numpy(),
+        analysis["value"].to_numpy(),
+        marker="o",
+        color=line_color,
+        linewidth=linewidth,
+        label="Analysis chunks",
+    )
+    drifted = analysis.filter(pl.col("drifted"))
+    if len(drifted):
+        ax.scatter(
+            drifted["index"].to_numpy(),
+            drifted["value"].to_numpy(),
+            marker="D",
+            color=drift_color,
+            zorder=3,
+            label="Drift",
+        )
+    ax.xaxis.set_major_locator(MaxNLocator(integer=True))
+    ax.set_xlabel(data.chunk_label)
+    ax.set_ylabel(data.metric_name)
+    ax.legend(loc="best")
+
+
 def plot_drift_on_axis(
     ax: Any,
     resdf: Any,
@@ -697,6 +896,7 @@ _RULED_OUT = "ruled out*"
 _AGGREGATION_WORDS = {None: "", "sequence_mean": "per-sequence mean ", "frame_pooled": "frame-pooled "}
 _GROUP_CEILING = "group-based ceiling"
 _NO_BOUND = "no bound available"
+_FALSE_POSITIVE = "false_positive_rate"
 
 
 def _pct(value: float, direction: str | None = None) -> str:
@@ -712,6 +912,11 @@ def _pct(value: float, direction: str | None = None) -> str:
     return f"{tenths / 10:.1f}%"
 
 
+def _confidence(value: float, direction: str) -> str:
+    """Round a confidence down or an error share up to 0.1%, dropping a trailing ".0", as the printout does."""
+    return _pct(value, direction).replace(".0%", "%")
+
+
 def _percent(value: float) -> str:
     """A threshold, confidence or alpha as DataEval prints it: no rounding to whole percents, no trailing zeros."""
     return f"{value * 100:g}%"
@@ -769,6 +974,16 @@ def _feasibility_range(values: Iterable[float]) -> tuple[float, float]:
     return round(max(low, 0.0), 2), 1.0
 
 
+def _not_counted(e: dict[str, Any]) -> str:
+    """The label suffix of a recall witness that does not count: its false-positive bound is over the limit or none."""
+    if not e["uncertified"]:
+        return ""
+    fp = e["false_positive_bound"]
+    seen = "no false-positive bound" if fp is None else f"false positives ≤{_pct(fp, 'upper')}"
+    limit = "" if fp is None else f" (limit {_percent(e['false_positive_limit'])})"
+    return f" · not counted: {seen}{limit}"
+
+
 def _feasibility_evidence(
     evidence: list[dict[str, Any]], scope: str, unit: str
 ) -> tuple[list[FeasibilityEvidenceRow], list[str]]:
@@ -802,10 +1017,10 @@ def _feasibility_evidence(
                 observed=observed,
                 restricted=restricted,
                 label=label,
-                bound_label=_NO_BOUND if bound is None else f"{_pct(bound, kind)} {kind} bound",
+                bound_label=_NO_BOUND if bound is None else f"{_pct(bound, kind)} {kind} bound" + _not_counted(e),
                 observed_label=None if observed is None else f"{_pct(observed)} observed",
                 bound_left=kind == "lower",
-                faint=restricted or bound is None,
+                faint=restricted or bound is None or e["uncertified"],
             )
         )
         if restricted and kind == "upper" and bound is not None:
@@ -849,18 +1064,29 @@ def _unit(rows: list[dict[str, Any]], evidence: list[dict[str, Any]]) -> str | N
     return None
 
 
+def _other(fp: dict[str, Any]) -> str:
+    """The audit negatives a false-positive bound was measured on, such as ``651 other images``."""
+    return ("" if fp["n"] is None else f"{fp['n']:,} ") + f"other {fp['unit']}s"
+
+
 def _feasibility_title(spec: dict[str, Any]) -> str:
     """The axis title, naming the AP class (or macro) and any aggregation so per-class bands differ."""
     metric = spec["metric"]
     if metric == "ap":
         metric = "macro AP" if spec.get("class_name") is None else f"AP for class {spec['class_name']}"
+    elif metric == "recall":
+        f0 = _percent(spec["max_false_positive_rate"])
+        metric = f"recall for {spec['class_name']} (at most {f0} false positives)"
     return "Required " + _AGGREGATION_WORDS[spec.get("aggregation")] + metric
 
 
 def _feasibility_band(
-    rows: list[dict[str, Any]], evidence: list[dict[str, Any]], alpha: float, noun: str
+    rows: list[dict[str, Any]], all_evidence: list[dict[str, Any]], alpha: float, noun: str
 ) -> FeasibilityBand:
     """One band; ``noun`` is the unit the printout's footer names for the whole call."""
+    # Bands and evidence rows draw the metric; a recall witness's false-positive bound only joins the caption
+    evidence = [e for e in all_evidence if e["quantity"] == "metric"]
+    false_positives = [e for e in all_evidence if e["quantity"] == _FALSE_POSITIVE]
     scope = rows[0]["judged_scope"]
     unit = _unit(rows, evidence) or "image"
     # A lower bound of 0% demonstrates nothing and an upper bound of 100% rules nothing out
@@ -887,7 +1113,7 @@ def _feasibility_band(
 
     conflict_caption = (
         f"Bounds disagree: check the assumptions, such as evaluation {unit}s entering training. "
-        f"Even when they hold, the chance that any bound fails is at most {_percent(alpha)}."
+        f"Even when they hold, the chance that any bound fails is at most {_confidence(alpha, 'upper')}."
     )
     if conflict:
         caption = [conflict_caption]
@@ -896,25 +1122,38 @@ def _feasibility_band(
         witness = None if best is None else _deciding(best, evidence)[0]
         if witness and witness["observed"] is not None and w is not None:
             on = "the " if witness["n"] is None else f"{witness['n']:,} "
+            # A recall witness's count is the class's positives, which the printout names by class
+            spec = json.loads(rows[0]["target_spec"])
+            scored = f"{_pct(witness['observed'])} on {on}evaluation {unit}s"
+            if spec["metric"] == "recall":
+                scored = f"{_pct(witness['observed'])} recall on {on}{spec['class_name']} {unit}s"
             caption.append(
-                f"{_sentence_case(witness['source'])} scored {_pct(witness['observed'])} on {on}evaluation "
-                f"{unit}s; given sampling uncertainty, its lower confidence bound is {_pct(w, 'lower')}."
+                f"{_sentence_case(witness['source'])} scored {scored}; given sampling uncertainty, its lower "
+                f"confidence bound is {_pct(w, 'lower')}."
             )
-        caption.append(
-            f"Bounds apply to new {noun}s from the same population, at {_percent(1 - alpha)} joint confidence."
-        )
+        fp = None
+        if witness and w is not None:
+            fp = next(
+                (
+                    e
+                    for e in false_positives
+                    if e["evidence_id"] == witness["evidence_id"]
+                    and e["target_spec"] == witness["target_spec"]
+                    and e["bound"] is not None
+                ),
+                None,
+            )
+        if fp is not None:
+            seen = "" if fp["observed"] is None else f" (observed {_pct(fp['observed'])} of {_other(fp)})"
+            caption.append(f"Its upper confidence bound on false positives is {_pct(fp['bound'], 'upper')}{seen}.")
+        joint = _confidence(1 - alpha, "lower")
+        caption.append(f"Bounds apply to new {noun}s from the same population, at {joint} joint confidence.")
         # The "*" footnote to every "ruled out*" goes last, as a footnote does
         if deciding is not None and c is not None:
-            ceiling = _deciding(deciding, evidence)[1]
-            sign_off = ceiling is not None and ceiling["source"] == _GROUP_CEILING
-            caption.append(
-                "* Ruled out under the recorded assumptions, at the stated confidence"
-                + (" (group-based ceiling; validity pending lead-scientist sign-off)" if sign_off else "")
-                + "."
-            )
+            caption.append("* Ruled out under the recorded assumptions, at the stated confidence.")
     evidence_caption = [
         f"Observed on the evaluation {unit}s; bounds apply to new {unit}s from the same population, "
-        f"at {_percent(1 - alpha)} joint confidence."
+        f"at {_confidence(1 - alpha, 'lower')} joint confidence."
     ]
     if conflict:
         evidence_caption.append(conflict_caption)
@@ -962,6 +1201,10 @@ _EVIDENCE_SCHEMA = {
     "bound": pl.Float64,
     "n": pl.Int64,
     "state": pl.String,
+    "quantity": pl.String,
+    "uncertified": pl.Boolean,
+    "false_positive_bound": pl.Float64,
+    "false_positive_limit": pl.Float64,
 }
 
 
@@ -971,10 +1214,14 @@ def _evidence_rows(output: PlottableFeasibility) -> pl.DataFrame:
     rows = []
     for spec, target in targets.items():
         for eid, ev in output.evidence.items():
-            for e in ev.endpoints:
-                if e.target_spec != spec:
-                    continue
-                stats = ev.details["endpoints"][spec] or {}
+            # An older DataEval has no quantity; its endpoints all bound the metric
+            mine = [(e, getattr(e, "quantity", "metric")) for e in ev.endpoints if e.target_spec == spec]
+            fp = next((e.value for e, q in mine if q == _FALSE_POSITIVE), None)
+            f0 = json.loads(spec).get("max_false_positive_rate")
+            for e, q in mine:
+                stats = ev.details["endpoints"].get(spec if q == "metric" else f"{spec}#{q}") or {}
+                # A recall witness counts only when its own false-positive upper bound is within the limit
+                uncertified = q == "metric" and e.direction == "lower" and f0 is not None and (fp is None or fp > f0)
                 rows.append(
                     {
                         "target_spec": spec,
@@ -988,10 +1235,14 @@ def _evidence_rows(output: PlottableFeasibility) -> pl.DataFrame:
                         "unit": "sequence" if ev.unit == "group" else "image",
                         "bound_type": e.direction,
                         # A ceiling's mean is panel agreement, not an observed score
-                        "observed": stats.get("mean") if e.direction == "lower" else None,
+                        "observed": stats.get("mean") if e.direction == "lower" or q == _FALSE_POSITIVE else None,
                         "bound": e.value,
                         "n": stats.get("n"),
                         "state": e.state,
+                        "quantity": q,
+                        "uncertified": uncertified,
+                        "false_positive_bound": fp,
+                        "false_positive_limit": f0,
                     }
                 )
     return pl.DataFrame(rows, schema=_EVIDENCE_SCHEMA)
@@ -1022,7 +1273,7 @@ def prepare_feasibility_data(output: PlottableFeasibility) -> list[FeasibilityBa
         groups.setdefault(json.dumps(spec, sort_keys=True), []).append(row)
     evidence = _evidence_rows(output).to_dicts()
     alpha = output.meta().state["alpha"]
-    noun = _unit(rows, evidence) or "image"  # the printout's footer noun, for the whole call
+    noun = _unit(rows, [e for e in evidence if e["quantity"] == "metric"]) or "image"  # the printout's footer noun
     bands = []
     for band in groups.values():
         specs = {r["target_spec"] for r in band}

{dataeval_plots-0.0.14 → dataeval_plots-0.0.16}/src/dataeval_plots/protocols.py

@@ -12,6 +12,7 @@ The protocol hierarchy:
   - PlottableSufficiency
   - PlottableStats
   - PlottableDriftMVDC
+  - PlottableDrift
   - PlottableFeasibility
 """
 
@@ -122,13 +123,15 @@ class Plottable(Protocol):
     """
 
     @property
-    def plot_type(self) -> Literal["balance", "diversity", "sufficiency", "stats", "drift_mvdc", "feasibility"]:
+    def plot_type(
+        self,
+    ) -> Literal["balance", "diversity", "sufficiency", "stats", "drift_mvdc", "drift", "feasibility"]:
         """Return the plot type identifier for routing to appropriate plot function.
 
         Returns
         -------
         str
-            One of: 'balance', 'diversity', 'sufficiency', 'stats', 'drift_mvdc', 'feasibility'
+            One of: 'balance', 'diversity', 'sufficiency', 'stats', 'drift_mvdc', 'drift', 'feasibility'
         """
         ...
 
@@ -289,8 +292,45 @@ class PlottableDriftMVDC(Plottable, Protocol):
     def plot_type(self) -> Literal["drift_mvdc"]: ...
 
 
+@runtime_checkable
+class PlottableDrift(Plottable, Protocol):
+    """Protocol for chunked drift detector outputs.
+
+    Required attributes:
+    - details: Per-chunk results as a polars DataFrame
+    - metric_name: Name of the drift metric that each chunk reports
+    - plot_type -> 'drift'
+    """
+
+    @property
+    def details(self) -> Any:
+        """Per-chunk drift results.
+
+        Returns
+        -------
+        pl.DataFrame
+            One row per chunk with the columns ``index``, ``value``, ``lower_threshold``, ``upper_threshold`` and
+            ``drifted``. A detector that was not chunked holds a mapping of statistics instead, which cannot be
+            plotted.
+        """
+        ...
+
+    @property
+    def metric_name(self) -> str:
+        """Name of the metric that ``value`` reports, such as ``auroc`` or ``mmd2``."""
+        ...
+
+    @property
+    def plot_type(self) -> Literal["drift"]: ...
+
+
 class FeasibilityEndpoint(Protocol):
-    """One bound on one target, as DataEval's ``Endpoint`` records it; only the fields the plot reads."""
+    """
+    One bound on one target, as DataEval's ``Endpoint`` records it; only the fields the plot reads.
+
+    Newer outputs also give ``quantity``, ``"metric"`` or ``"false_positive_rate"``; the plot reads it with a default
+    of ``"metric"`` so older outputs still plot.
+    """
 
     @property
     def target_spec(self) -> str: ...
@@ -350,5 +390,6 @@ PlottableType = (
     | PlottableSufficiency
     | PlottableStats
     | PlottableDriftMVDC
+    | PlottableDrift
     | PlottableFeasibility
 )