dataeval-plots 0.0.15__tar.gz → 0.0.16__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dataeval_plots-0.0.15 → dataeval_plots-0.0.16}/PKG-INFO +1 -1
- {dataeval_plots-0.0.15 → dataeval_plots-0.0.16}/src/dataeval_plots/__init__.py +10 -0
- {dataeval_plots-0.0.15 → dataeval_plots-0.0.16}/src/dataeval_plots/_version.py +2 -2
- {dataeval_plots-0.0.15 → dataeval_plots-0.0.16}/src/dataeval_plots/backends/_altair.py +78 -0
- {dataeval_plots-0.0.15 → dataeval_plots-0.0.16}/src/dataeval_plots/backends/_base.py +20 -0
- {dataeval_plots-0.0.15 → dataeval_plots-0.0.16}/src/dataeval_plots/backends/_matplotlib.py +32 -0
- {dataeval_plots-0.0.15 → dataeval_plots-0.0.16}/src/dataeval_plots/backends/_plotly.py +109 -0
- {dataeval_plots-0.0.15 → dataeval_plots-0.0.16}/src/dataeval_plots/backends/_seaborn.py +44 -0
- {dataeval_plots-0.0.15 → dataeval_plots-0.0.16}/src/dataeval_plots/backends/_shared.py +199 -0
- {dataeval_plots-0.0.15 → dataeval_plots-0.0.16}/src/dataeval_plots/protocols.py +38 -2
- {dataeval_plots-0.0.15 → dataeval_plots-0.0.16}/.gitignore +0 -0
- {dataeval_plots-0.0.15 → dataeval_plots-0.0.16}/README.md +0 -0
- {dataeval_plots-0.0.15 → dataeval_plots-0.0.16}/pyproject.toml +0 -0
- {dataeval_plots-0.0.15 → dataeval_plots-0.0.16}/src/dataeval_plots/_registry.py +0 -0
- {dataeval_plots-0.0.15 → dataeval_plots-0.0.16}/src/dataeval_plots/backends/__init__.py +0 -0
- {dataeval_plots-0.0.15 → dataeval_plots-0.0.16}/src/dataeval_plots/backends/_feasibility_layout.py +0 -0
- {dataeval_plots-0.0.15 → dataeval_plots-0.0.16}/src/dataeval_plots/py.typed +0 -0
|
@@ -19,6 +19,7 @@ from dataeval_plots.protocols import (
|
|
|
19
19
|
Dataset,
|
|
20
20
|
PlottableBalance,
|
|
21
21
|
PlottableDiversity,
|
|
22
|
+
PlottableDrift,
|
|
22
23
|
PlottableDriftMVDC,
|
|
23
24
|
PlottableFeasibility,
|
|
24
25
|
PlottableStats,
|
|
@@ -98,6 +99,15 @@ def plot(
|
|
|
98
99
|
) -> Any: ...
|
|
99
100
|
|
|
100
101
|
|
|
102
|
+
@overload
|
|
103
|
+
def plot(
|
|
104
|
+
output: PlottableDrift,
|
|
105
|
+
/,
|
|
106
|
+
figsize: tuple[float, float] | None = None,
|
|
107
|
+
backend: str | None = None,
|
|
108
|
+
) -> Any: ...
|
|
109
|
+
|
|
110
|
+
|
|
101
111
|
@overload
|
|
102
112
|
def plot(
|
|
103
113
|
output: PlottableFeasibility,
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '0.0.
|
|
22
|
-
__version_tuple__ = version_tuple = (0, 0,
|
|
21
|
+
__version__ = version = '0.0.16'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 0, 16)
|
|
23
23
|
|
|
24
24
|
__commit_id__ = commit_id = None
|
|
@@ -29,6 +29,7 @@ from dataeval_plots.backends._shared import (
|
|
|
29
29
|
normalize_reference_outputs,
|
|
30
30
|
parse_dataset_item,
|
|
31
31
|
prepare_balance_data,
|
|
32
|
+
prepare_chunked_drift,
|
|
32
33
|
prepare_diversity_data,
|
|
33
34
|
prepare_drift_data,
|
|
34
35
|
prepare_feasibility_data,
|
|
@@ -39,6 +40,7 @@ from dataeval_plots.protocols import (
|
|
|
39
40
|
Dataset,
|
|
40
41
|
PlottableBalance,
|
|
41
42
|
PlottableDiversity,
|
|
43
|
+
PlottableDrift,
|
|
42
44
|
PlottableDriftMVDC,
|
|
43
45
|
PlottableFeasibility,
|
|
44
46
|
PlottableStats,
|
|
@@ -819,6 +821,82 @@ class AltairBackend(BasePlottingBackend):
|
|
|
819
821
|
width=width, height=height, title="Domain Classifier, Drift Detection"
|
|
820
822
|
)
|
|
821
823
|
|
|
824
|
+
def _plot_drift(
|
|
825
|
+
self,
|
|
826
|
+
output: PlottableDrift,
|
|
827
|
+
figsize: tuple[float, float] | None = None,
|
|
828
|
+
) -> Any: # alt.LayerChart
|
|
829
|
+
"""
|
|
830
|
+
Render the metric of each chunk in relation to the thresholds of the detector.
|
|
831
|
+
|
|
832
|
+
When the output carries the metric of the reference chunks, they are drawn in gray to the left of the
|
|
833
|
+
analysis chunks. The chunks are numbered from -n for the reference and from 0 for the analysis.
|
|
834
|
+
|
|
835
|
+
Parameters
|
|
836
|
+
----------
|
|
837
|
+
output : PlottableDrift
|
|
838
|
+
The chunked drift output object to plot
|
|
839
|
+
figsize : tuple[float, float] or None, default None
|
|
840
|
+
Figure size in inches (width, height), drawn at 100 pixels per inch. If None, defaults to (6, 3).
|
|
841
|
+
|
|
842
|
+
Returns
|
|
843
|
+
-------
|
|
844
|
+
alt.LayerChart
|
|
845
|
+
"""
|
|
846
|
+
import altair as alt
|
|
847
|
+
import pandas as pd
|
|
848
|
+
|
|
849
|
+
data = prepare_chunked_drift(output)
|
|
850
|
+
width_inches, height_inches = figsize or (6, 3)
|
|
851
|
+
|
|
852
|
+
# Convert via dict to avoid pyarrow dependency
|
|
853
|
+
plot_df = pd.DataFrame(
|
|
854
|
+
data.frame.with_columns(
|
|
855
|
+
pl.when(pl.col("period") == "reference")
|
|
856
|
+
.then(pl.lit("Reference chunks"))
|
|
857
|
+
.otherwise(pl.lit("Analysis chunks"))
|
|
858
|
+
.alias("period")
|
|
859
|
+
).to_dict(as_series=False)
|
|
860
|
+
)
|
|
861
|
+
|
|
862
|
+
x = alt.X("index:Q", title=data.chunk_label, axis=alt.Axis(tickMinStep=1))
|
|
863
|
+
scale = alt.Scale(zero=False)
|
|
864
|
+
y = alt.Y("value:Q", title=data.metric_name, scale=scale)
|
|
865
|
+
tooltip = ["index:Q", alt.Tooltip("value:Q", title=data.metric_name, format=".4f")]
|
|
866
|
+
|
|
867
|
+
# The no-drift range is the band between the two thresholds
|
|
868
|
+
layers = [
|
|
869
|
+
alt.Chart(plot_df)
|
|
870
|
+
.mark_area(color="green", opacity=0.15, interpolate="step")
|
|
871
|
+
.encode(x=x, y=alt.Y("lower_threshold:Q", title=data.metric_name, scale=scale), y2="upper_threshold:Q")
|
|
872
|
+
]
|
|
873
|
+
|
|
874
|
+
# The metric of each chunk, with the reference chunks in gray
|
|
875
|
+
has_reference = bool((plot_df["period"] == "Reference chunks").any())
|
|
876
|
+
color = alt.Color(
|
|
877
|
+
"period:N",
|
|
878
|
+
scale=alt.Scale(domain=["Reference chunks", "Analysis chunks"], range=["gray", "royalblue"]),
|
|
879
|
+
legend=alt.Legend(title=None),
|
|
880
|
+
)
|
|
881
|
+
layers.append(alt.Chart(plot_df).mark_line(point=True).encode(x=x, y=y, color=color, tooltip=tooltip))
|
|
882
|
+
if has_reference:
|
|
883
|
+
layers.append(
|
|
884
|
+
alt.Chart(pd.DataFrame({"index": [-0.5]}))
|
|
885
|
+
.mark_rule(color="gray", strokeDash=[2, 2])
|
|
886
|
+
.encode(x="index:Q")
|
|
887
|
+
)
|
|
888
|
+
|
|
889
|
+
# The chunks that the detector flagged
|
|
890
|
+
layers.append(
|
|
891
|
+
alt.Chart(plot_df[plot_df["drifted"]]) # type: ignore[arg-type]
|
|
892
|
+
.mark_point(shape="diamond", size=120, color="crimson", filled=True)
|
|
893
|
+
.encode(x=x, y=y, tooltip=tooltip)
|
|
894
|
+
)
|
|
895
|
+
|
|
896
|
+
return alt.layer(*layers).properties(
|
|
897
|
+
width=int(width_inches * 100), height=int(height_inches * 100), title="Drift by chunk"
|
|
898
|
+
)
|
|
899
|
+
|
|
822
900
|
def _plot_image_grid(
|
|
823
901
|
self,
|
|
824
902
|
dataset: Dataset,
|
|
@@ -14,6 +14,7 @@ from dataeval_plots.protocols import (
|
|
|
14
14
|
Dataset,
|
|
15
15
|
PlottableBalance,
|
|
16
16
|
PlottableDiversity,
|
|
17
|
+
PlottableDrift,
|
|
17
18
|
PlottableDriftMVDC,
|
|
18
19
|
PlottableFeasibility,
|
|
19
20
|
PlottableStats,
|
|
@@ -81,6 +82,14 @@ class PlottingBackend(Protocol):
|
|
|
81
82
|
figsize: tuple[float, float] | None = None,
|
|
82
83
|
) -> Any: ...
|
|
83
84
|
|
|
85
|
+
@overload
|
|
86
|
+
def plot(
|
|
87
|
+
self,
|
|
88
|
+
output: PlottableDrift,
|
|
89
|
+
*,
|
|
90
|
+
figsize: tuple[float, float] | None = None,
|
|
91
|
+
) -> Any: ...
|
|
92
|
+
|
|
84
93
|
@overload
|
|
85
94
|
def plot(
|
|
86
95
|
self,
|
|
@@ -245,6 +254,8 @@ class BasePlottingBackend(PlottingBackend, ABC):
|
|
|
245
254
|
return self._plot_sufficiency(cast(PlottableSufficiency, output), figsize=figsize, **kwargs)
|
|
246
255
|
if plot_type == "drift_mvdc":
|
|
247
256
|
return self._plot_drift_mvdc(cast(PlottableDriftMVDC, output), figsize=figsize, **kwargs)
|
|
257
|
+
if plot_type == "drift":
|
|
258
|
+
return self._plot_drift(cast(PlottableDrift, output), figsize=figsize, **kwargs)
|
|
248
259
|
if plot_type == "stats":
|
|
249
260
|
return self._plot_stats(cast(PlottableStats, output), figsize=figsize, **kwargs)
|
|
250
261
|
if plot_type == "feasibility":
|
|
@@ -310,6 +321,15 @@ class BasePlottingBackend(PlottingBackend, ABC):
|
|
|
310
321
|
"""Plot drift MVDC output."""
|
|
311
322
|
...
|
|
312
323
|
|
|
324
|
+
@abstractmethod
|
|
325
|
+
def _plot_drift(
|
|
326
|
+
self,
|
|
327
|
+
output: PlottableDrift,
|
|
328
|
+
figsize: tuple[float, float] | None = None,
|
|
329
|
+
) -> Any:
|
|
330
|
+
"""Plot the per-chunk results of a chunked drift detector."""
|
|
331
|
+
...
|
|
332
|
+
|
|
313
333
|
@abstractmethod
|
|
314
334
|
def _plot_feasibility(
|
|
315
335
|
self,
|
|
@@ -21,8 +21,10 @@ from dataeval_plots.backends._shared import (
|
|
|
21
21
|
calculate_subplot_grid,
|
|
22
22
|
feasibility_zones,
|
|
23
23
|
normalize_reference_outputs,
|
|
24
|
+
plot_chunked_drift_on_axis,
|
|
24
25
|
plot_drift_on_axis,
|
|
25
26
|
prepare_balance_data,
|
|
27
|
+
prepare_chunked_drift,
|
|
26
28
|
prepare_diversity_data,
|
|
27
29
|
prepare_drift_data,
|
|
28
30
|
prepare_feasibility_data,
|
|
@@ -32,6 +34,7 @@ from dataeval_plots.backends._shared import (
|
|
|
32
34
|
from dataeval_plots.protocols import (
|
|
33
35
|
PlottableBalance,
|
|
34
36
|
PlottableDiversity,
|
|
37
|
+
PlottableDrift,
|
|
35
38
|
PlottableDriftMVDC,
|
|
36
39
|
PlottableFeasibility,
|
|
37
40
|
PlottableStats,
|
|
@@ -833,6 +836,35 @@ class MatplotlibBackend(BasePlottingBackend):
|
|
|
833
836
|
)
|
|
834
837
|
return fig
|
|
835
838
|
|
|
839
|
+
def _plot_drift(
|
|
840
|
+
self,
|
|
841
|
+
output: PlottableDrift,
|
|
842
|
+
figsize: tuple[float, float] | None = None,
|
|
843
|
+
) -> Figure:
|
|
844
|
+
"""
|
|
845
|
+
Render the metric of each chunk in relation to the thresholds of the detector.
|
|
846
|
+
|
|
847
|
+
Parameters
|
|
848
|
+
----------
|
|
849
|
+
output : PlottableDrift
|
|
850
|
+
The chunked drift output object to plot
|
|
851
|
+
figsize : tuple[float, float] or None, default None
|
|
852
|
+
Figure size in inches (width, height). If None, defaults to (8, 3.5).
|
|
853
|
+
|
|
854
|
+
Returns
|
|
855
|
+
-------
|
|
856
|
+
matplotlib.figure.Figure
|
|
857
|
+
"""
|
|
858
|
+
import matplotlib.pyplot as plt
|
|
859
|
+
|
|
860
|
+
data = prepare_chunked_drift(output)
|
|
861
|
+
|
|
862
|
+
fig, ax = plt.subplots(figsize=figsize or (8, 3.5))
|
|
863
|
+
plot_chunked_drift_on_axis(ax, data)
|
|
864
|
+
ax.set_title("Drift by chunk")
|
|
865
|
+
fig.tight_layout()
|
|
866
|
+
return fig
|
|
867
|
+
|
|
836
868
|
def _plot_feasibility(
|
|
837
869
|
self,
|
|
838
870
|
output: PlottableFeasibility,
|
|
@@ -22,6 +22,7 @@ from dataeval_plots.backends._shared import (
|
|
|
22
22
|
feasibility_zones,
|
|
23
23
|
format_label_from_target,
|
|
24
24
|
prepare_balance_data,
|
|
25
|
+
prepare_chunked_drift,
|
|
25
26
|
prepare_diversity_data,
|
|
26
27
|
prepare_drift_data,
|
|
27
28
|
prepare_feasibility_data,
|
|
@@ -33,6 +34,7 @@ from dataeval_plots.protocols import (
|
|
|
33
34
|
Dataset,
|
|
34
35
|
PlottableBalance,
|
|
35
36
|
PlottableDiversity,
|
|
37
|
+
PlottableDrift,
|
|
36
38
|
PlottableDriftMVDC,
|
|
37
39
|
PlottableFeasibility,
|
|
38
40
|
PlottableStats,
|
|
@@ -849,6 +851,113 @@ class PlotlyBackend(BasePlottingBackend):
|
|
|
849
851
|
|
|
850
852
|
return fig
|
|
851
853
|
|
|
854
|
+
def _plot_drift(
|
|
855
|
+
self,
|
|
856
|
+
output: PlottableDrift,
|
|
857
|
+
figsize: tuple[float, float] | None = None,
|
|
858
|
+
) -> Any: # go.Figure
|
|
859
|
+
"""
|
|
860
|
+
Render the metric of each chunk in relation to the thresholds of the detector.
|
|
861
|
+
|
|
862
|
+
When the output carries the metric of the reference chunks, they are drawn in gray to the left of the
|
|
863
|
+
analysis chunks. The chunks are numbered from -n for the reference and from 0 for the analysis.
|
|
864
|
+
|
|
865
|
+
Parameters
|
|
866
|
+
----------
|
|
867
|
+
output : PlottableDrift
|
|
868
|
+
The chunked drift output object to plot
|
|
869
|
+
figsize : tuple[float, float] or None, default None
|
|
870
|
+
Figure size in inches (width, height), drawn at 100 pixels per inch. If None, defaults to (8, 3.5).
|
|
871
|
+
|
|
872
|
+
Returns
|
|
873
|
+
-------
|
|
874
|
+
plotly.graph_objects.Figure
|
|
875
|
+
"""
|
|
876
|
+
import plotly.graph_objects as go
|
|
877
|
+
|
|
878
|
+
data = prepare_chunked_drift(output)
|
|
879
|
+
frame, reference, analysis = data.frame, data.reference, data.analysis
|
|
880
|
+
drifted = analysis.filter(pl.col("drifted"))
|
|
881
|
+
hover = f"Chunk: %{{x}}<br>{data.metric_name}: %{{y:.4f}}<extra></extra>"
|
|
882
|
+
|
|
883
|
+
fig = go.Figure()
|
|
884
|
+
|
|
885
|
+
# The no-drift range is the band between the two thresholds
|
|
886
|
+
fig.add_trace(
|
|
887
|
+
go.Scatter(
|
|
888
|
+
x=frame["index"].to_numpy(),
|
|
889
|
+
y=frame["upper_threshold"].to_numpy(),
|
|
890
|
+
mode="lines",
|
|
891
|
+
line={"width": 0, "shape": "hvh"},
|
|
892
|
+
showlegend=False,
|
|
893
|
+
hoverinfo="skip",
|
|
894
|
+
)
|
|
895
|
+
)
|
|
896
|
+
fig.add_trace(
|
|
897
|
+
go.Scatter(
|
|
898
|
+
x=frame["index"].to_numpy(),
|
|
899
|
+
y=frame["lower_threshold"].to_numpy(),
|
|
900
|
+
mode="lines",
|
|
901
|
+
line={"width": 0, "shape": "hvh"},
|
|
902
|
+
fill="tonexty",
|
|
903
|
+
fillcolor="rgba(44, 160, 44, 0.15)",
|
|
904
|
+
name="No-drift range",
|
|
905
|
+
hoverinfo="skip",
|
|
906
|
+
)
|
|
907
|
+
)
|
|
908
|
+
|
|
909
|
+
# The reference chunks, which the thresholds derive from
|
|
910
|
+
if len(reference):
|
|
911
|
+
fig.add_trace(
|
|
912
|
+
go.Scatter(
|
|
913
|
+
x=reference["index"].to_numpy(),
|
|
914
|
+
y=reference["value"].to_numpy(),
|
|
915
|
+
mode="lines+markers",
|
|
916
|
+
name="Reference chunks",
|
|
917
|
+
line={"color": "gray", "width": 2},
|
|
918
|
+
hovertemplate=hover,
|
|
919
|
+
)
|
|
920
|
+
)
|
|
921
|
+
fig.add_vline(x=-0.5, line_dash="dot", line_color="gray", line_width=1)
|
|
922
|
+
|
|
923
|
+
# The metric of each analysis chunk
|
|
924
|
+
fig.add_trace(
|
|
925
|
+
go.Scatter(
|
|
926
|
+
x=analysis["index"].to_numpy(),
|
|
927
|
+
y=analysis["value"].to_numpy(),
|
|
928
|
+
mode="lines+markers",
|
|
929
|
+
name="Analysis chunks",
|
|
930
|
+
line={"color": "royalblue", "width": 2},
|
|
931
|
+
hovertemplate=hover,
|
|
932
|
+
)
|
|
933
|
+
)
|
|
934
|
+
|
|
935
|
+
# The chunks that the detector flagged
|
|
936
|
+
if len(drifted):
|
|
937
|
+
fig.add_trace(
|
|
938
|
+
go.Scatter(
|
|
939
|
+
x=drifted["index"].to_numpy(),
|
|
940
|
+
y=drifted["value"].to_numpy(),
|
|
941
|
+
mode="markers",
|
|
942
|
+
name="Drift",
|
|
943
|
+
marker={"symbol": "diamond", "size": 11, "color": "crimson"},
|
|
944
|
+
hovertemplate=hover,
|
|
945
|
+
)
|
|
946
|
+
)
|
|
947
|
+
|
|
948
|
+
width_inches, height_inches = figsize or (8, 3.5)
|
|
949
|
+
fig.update_layout(
|
|
950
|
+
title="Drift by chunk",
|
|
951
|
+
xaxis_title=data.chunk_label,
|
|
952
|
+
yaxis_title=data.metric_name,
|
|
953
|
+
xaxis={"dtick": 1} if len(frame) <= 20 else {},
|
|
954
|
+
width=int(width_inches * 100),
|
|
955
|
+
height=int(height_inches * 100),
|
|
956
|
+
hovermode="closest",
|
|
957
|
+
)
|
|
958
|
+
|
|
959
|
+
return fig
|
|
960
|
+
|
|
852
961
|
def _plot_image_grid(
|
|
853
962
|
self,
|
|
854
963
|
dataset: Dataset,
|
|
@@ -10,14 +10,17 @@ from numpy.typing import NDArray
|
|
|
10
10
|
from dataeval_plots.backends._base import BasePlottingBackend
|
|
11
11
|
from dataeval_plots.backends._shared import (
|
|
12
12
|
CHANNELWISE_METRICS,
|
|
13
|
+
plot_chunked_drift_on_axis,
|
|
13
14
|
plot_drift_on_axis,
|
|
14
15
|
prepare_balance_data,
|
|
16
|
+
prepare_chunked_drift,
|
|
15
17
|
prepare_diversity_data,
|
|
16
18
|
prepare_drift_data,
|
|
17
19
|
)
|
|
18
20
|
from dataeval_plots.protocols import (
|
|
19
21
|
PlottableBalance,
|
|
20
22
|
PlottableDiversity,
|
|
23
|
+
PlottableDrift,
|
|
21
24
|
PlottableDriftMVDC,
|
|
22
25
|
PlottableFeasibility,
|
|
23
26
|
PlottableStats,
|
|
@@ -493,6 +496,47 @@ class SeabornBackend(BasePlottingBackend):
|
|
|
493
496
|
fig.tight_layout()
|
|
494
497
|
return fig
|
|
495
498
|
|
|
499
|
+
def _plot_drift(
|
|
500
|
+
self,
|
|
501
|
+
output: PlottableDrift,
|
|
502
|
+
figsize: tuple[float, float] | None = None,
|
|
503
|
+
) -> Figure:
|
|
504
|
+
"""
|
|
505
|
+
Render the metric of each chunk in relation to the thresholds of the detector.
|
|
506
|
+
|
|
507
|
+
Parameters
|
|
508
|
+
----------
|
|
509
|
+
output : PlottableDrift
|
|
510
|
+
The chunked drift output object to plot
|
|
511
|
+
figsize : tuple[float, float] or None, default None
|
|
512
|
+
Figure size in inches (width, height). If None, defaults to (10, 4).
|
|
513
|
+
|
|
514
|
+
Returns
|
|
515
|
+
-------
|
|
516
|
+
matplotlib.figure.Figure
|
|
517
|
+
"""
|
|
518
|
+
import matplotlib.pyplot as plt
|
|
519
|
+
import seaborn as sns
|
|
520
|
+
|
|
521
|
+
sns.set_style("whitegrid")
|
|
522
|
+
palette = sns.color_palette("deep") # not the global palette, which a user can shorten
|
|
523
|
+
|
|
524
|
+
data = prepare_chunked_drift(output)
|
|
525
|
+
|
|
526
|
+
fig, ax = plt.subplots(figsize=figsize or (10, 4))
|
|
527
|
+
plot_chunked_drift_on_axis(
|
|
528
|
+
ax,
|
|
529
|
+
data,
|
|
530
|
+
line_color=palette[0],
|
|
531
|
+
band_color=palette[2],
|
|
532
|
+
drift_color=palette[3],
|
|
533
|
+
reference_color=palette[7],
|
|
534
|
+
)
|
|
535
|
+
ax.set_title("Drift by chunk")
|
|
536
|
+
sns.despine(ax=ax)
|
|
537
|
+
fig.tight_layout()
|
|
538
|
+
return fig
|
|
539
|
+
|
|
496
540
|
def _plot_feasibility(
|
|
497
541
|
self,
|
|
498
542
|
output: PlottableFeasibility,
|
|
@@ -19,6 +19,7 @@ if TYPE_CHECKING:
|
|
|
19
19
|
from dataeval_plots.protocols import (
|
|
20
20
|
PlottableBalance,
|
|
21
21
|
PlottableDiversity,
|
|
22
|
+
PlottableDrift,
|
|
22
23
|
PlottableDriftMVDC,
|
|
23
24
|
PlottableFeasibility,
|
|
24
25
|
)
|
|
@@ -46,6 +47,10 @@ __all__ = [
|
|
|
46
47
|
"prepare_diversity_data",
|
|
47
48
|
"prepare_drift_data",
|
|
48
49
|
"plot_drift_on_axis",
|
|
50
|
+
"CHUNKED_DRIFT_COLUMNS",
|
|
51
|
+
"ChunkedDriftData",
|
|
52
|
+
"prepare_chunked_drift",
|
|
53
|
+
"plot_chunked_drift_on_axis",
|
|
49
54
|
"FEASIBILITY_COLORS",
|
|
50
55
|
"FEASIBILITY_CONFLICT_TEXT",
|
|
51
56
|
"FeasibilityBand",
|
|
@@ -498,6 +503,200 @@ def prepare_drift_data(
|
|
|
498
503
|
return resdf, trndf, tstdf, driftx, True
|
|
499
504
|
|
|
500
505
|
|
|
506
|
+
CHUNKED_DRIFT_COLUMNS = ("index", "value", "lower_threshold", "upper_threshold", "drifted")
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
@dataclass(frozen=True)
|
|
510
|
+
class ChunkedDriftData:
|
|
511
|
+
"""
|
|
512
|
+
The per-chunk results of a chunked drift detector, ready for plotting.
|
|
513
|
+
|
|
514
|
+
Attributes
|
|
515
|
+
----------
|
|
516
|
+
frame : pl.DataFrame
|
|
517
|
+
One row per chunk with the columns ``index``, ``value``, ``lower_threshold``, ``upper_threshold``,
|
|
518
|
+
``drifted`` and ``period``. The rows of the reference chunks come first, numbered from ``-n`` to ``-1``
|
|
519
|
+
so that the analysis chunks keep their own indexes, and have ``period`` set to ``"reference"``. The
|
|
520
|
+
analysis chunks have ``period`` set to ``"analysis"``.
|
|
521
|
+
metric_name : str
|
|
522
|
+
Name of the metric that ``value`` reports
|
|
523
|
+
chunk_label : str
|
|
524
|
+
Label for the axis of chunks, which includes the size of a chunk when every analysis chunk has the same size
|
|
525
|
+
"""
|
|
526
|
+
|
|
527
|
+
frame: pl.DataFrame
|
|
528
|
+
metric_name: str
|
|
529
|
+
chunk_label: str
|
|
530
|
+
|
|
531
|
+
@property
|
|
532
|
+
def reference(self) -> pl.DataFrame:
|
|
533
|
+
"""The rows of the reference chunks, which is empty when the output carries no baseline."""
|
|
534
|
+
return self.frame.filter(pl.col("period") == "reference")
|
|
535
|
+
|
|
536
|
+
@property
|
|
537
|
+
def analysis(self) -> pl.DataFrame:
|
|
538
|
+
"""The rows of the analysis chunks."""
|
|
539
|
+
return self.frame.filter(pl.col("period") == "analysis")
|
|
540
|
+
|
|
541
|
+
|
|
542
|
+
def prepare_chunked_drift(output: PlottableDrift) -> ChunkedDriftData:
|
|
543
|
+
"""
|
|
544
|
+
Read the per-chunk table, the reference chunks and the metric name from a chunked drift output.
|
|
545
|
+
|
|
546
|
+
The reference chunks are the optional ``baseline`` attribute of the output, which holds the metric of each
|
|
547
|
+
reference chunk. An output without it, such as one from an earlier version of DataEval, has no reference rows
|
|
548
|
+
and the plot leaves them out.
|
|
549
|
+
|
|
550
|
+
Parameters
|
|
551
|
+
----------
|
|
552
|
+
output : PlottableDrift
|
|
553
|
+
The drift output object to plot
|
|
554
|
+
|
|
555
|
+
Returns
|
|
556
|
+
-------
|
|
557
|
+
ChunkedDriftData
|
|
558
|
+
The per-chunk results. A threshold that is missing, as for a one-sided detector, is replaced by a bound just
|
|
559
|
+
beyond the data, so that the range of no drift has no limit on that side.
|
|
560
|
+
|
|
561
|
+
Raises
|
|
562
|
+
------
|
|
563
|
+
ValueError
|
|
564
|
+
If the output holds the statistics of a single test and not a per-chunk table, or if the table lacks a column
|
|
565
|
+
that the plot needs
|
|
566
|
+
"""
|
|
567
|
+
details = output.details
|
|
568
|
+
if not isinstance(details, pl.DataFrame):
|
|
569
|
+
raise ValueError("Plotting drift needs chunked results. Call .chunked() on the detector before predict().")
|
|
570
|
+
missing = [name for name in CHUNKED_DRIFT_COLUMNS if name not in details.columns]
|
|
571
|
+
if missing:
|
|
572
|
+
raise ValueError(f"The per-chunk drift results lack the columns {missing}.")
|
|
573
|
+
|
|
574
|
+
analysis = details.select(
|
|
575
|
+
pl.col("index").cast(pl.Int64),
|
|
576
|
+
pl.col("value", "lower_threshold", "upper_threshold").cast(pl.Float64),
|
|
577
|
+
pl.col("drifted").cast(pl.Boolean),
|
|
578
|
+
pl.lit("analysis").alias("period"),
|
|
579
|
+
)
|
|
580
|
+
|
|
581
|
+
# The thresholds come from the reference chunks, so the band spans them too
|
|
582
|
+
baseline = np.asarray(getattr(output, "baseline", None) or (), dtype=np.float64)
|
|
583
|
+
if len(baseline) and len(analysis):
|
|
584
|
+
reference = pl.DataFrame(
|
|
585
|
+
{
|
|
586
|
+
"index": np.arange(-len(baseline), 0, dtype=np.int64),
|
|
587
|
+
"value": baseline,
|
|
588
|
+
"lower_threshold": [analysis["lower_threshold"][0]] * len(baseline),
|
|
589
|
+
"upper_threshold": [analysis["upper_threshold"][0]] * len(baseline),
|
|
590
|
+
"drifted": [False] * len(baseline),
|
|
591
|
+
"period": ["reference"] * len(baseline),
|
|
592
|
+
},
|
|
593
|
+
schema=analysis.schema,
|
|
594
|
+
)
|
|
595
|
+
frame = pl.concat([reference, analysis])
|
|
596
|
+
else:
|
|
597
|
+
frame = analysis
|
|
598
|
+
|
|
599
|
+
if frame["lower_threshold"].null_count() or frame["upper_threshold"].null_count():
|
|
600
|
+
extent = pl.concat([frame["value"], frame["lower_threshold"], frame["upper_threshold"]]).drop_nulls()
|
|
601
|
+
low, high = extent.min(), extent.max()
|
|
602
|
+
pad = 0.1 * (high - low) if low != high else 1.0 # type: ignore[operator]
|
|
603
|
+
frame = frame.with_columns(
|
|
604
|
+
pl.col("lower_threshold").fill_null(low - pad), # type: ignore[operator]
|
|
605
|
+
pl.col("upper_threshold").fill_null(high + pad), # type: ignore[operator]
|
|
606
|
+
)
|
|
607
|
+
|
|
608
|
+
return ChunkedDriftData(frame, output.metric_name, _chunk_label(details))
|
|
609
|
+
|
|
610
|
+
|
|
611
|
+
def _chunk_label(details: pl.DataFrame) -> str:
|
|
612
|
+
"""Label the axis of chunks with the size of a chunk, when every chunk has the same size."""
|
|
613
|
+
if "start_index" in details.columns and "end_index" in details.columns and len(details):
|
|
614
|
+
sizes = (details["end_index"] - details["start_index"] + 1).unique()
|
|
615
|
+
if len(sizes) == 1 and (details["start_index"] >= 0).all():
|
|
616
|
+
return f"Chunk ({sizes[0]} samples each)"
|
|
617
|
+
return "Chunk"
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
def plot_chunked_drift_on_axis(
|
|
621
|
+
ax: Any,
|
|
622
|
+
data: ChunkedDriftData,
|
|
623
|
+
line_color: Any = "tab:blue",
|
|
624
|
+
band_color: Any = "tab:green",
|
|
625
|
+
drift_color: Any = "tab:red",
|
|
626
|
+
reference_color: Any = "tab:gray",
|
|
627
|
+
linewidth: float = 1.5,
|
|
628
|
+
) -> None:
|
|
629
|
+
"""
|
|
630
|
+
Plot the metric of each chunk against the threshold range of the detector on a matplotlib axis.
|
|
631
|
+
|
|
632
|
+
This is a shared helper for the matplotlib and seaborn backends. The metric is a line with a marker per chunk,
|
|
633
|
+
the band is the range between the lower and upper thresholds in which a chunk counts as no drift, and a diamond
|
|
634
|
+
marks each chunk that the detector flagged. When the output carries the metric of the reference chunks, they are
|
|
635
|
+
drawn in a second color to the left of the analysis chunks, with a dotted line between them.
|
|
636
|
+
|
|
637
|
+
Parameters
|
|
638
|
+
----------
|
|
639
|
+
ax : matplotlib.axes.Axes
|
|
640
|
+
Axis to plot on
|
|
641
|
+
data : ChunkedDriftData
|
|
642
|
+
Per-chunk results from :func:`prepare_chunked_drift`
|
|
643
|
+
line_color : Any, default "tab:blue"
|
|
644
|
+
Color of the metric line of the analysis chunks
|
|
645
|
+
band_color : Any, default "tab:green"
|
|
646
|
+
Color of the band between the thresholds
|
|
647
|
+
drift_color : Any, default "tab:red"
|
|
648
|
+
Color of the markers on the chunks with drift
|
|
649
|
+
reference_color : Any, default "tab:gray"
|
|
650
|
+
Color of the metric line of the reference chunks and of the line that separates them
|
|
651
|
+
linewidth : float, default 1.5
|
|
652
|
+
Width of the metric lines
|
|
653
|
+
"""
|
|
654
|
+
from matplotlib.ticker import MaxNLocator
|
|
655
|
+
|
|
656
|
+
frame, reference, analysis = data.frame, data.reference, data.analysis
|
|
657
|
+
ax.fill_between(
|
|
658
|
+
frame["index"].to_numpy(),
|
|
659
|
+
frame["lower_threshold"].to_numpy(),
|
|
660
|
+
frame["upper_threshold"].to_numpy(),
|
|
661
|
+
step="mid",
|
|
662
|
+
color=band_color,
|
|
663
|
+
alpha=0.15,
|
|
664
|
+
label="No-drift range",
|
|
665
|
+
)
|
|
666
|
+
if len(reference):
|
|
667
|
+
ax.plot(
|
|
668
|
+
reference["index"].to_numpy(),
|
|
669
|
+
reference["value"].to_numpy(),
|
|
670
|
+
marker="o",
|
|
671
|
+
color=reference_color,
|
|
672
|
+
linewidth=linewidth,
|
|
673
|
+
label="Reference chunks",
|
|
674
|
+
)
|
|
675
|
+
ax.axvline(-0.5, color=reference_color, linestyle=":", linewidth=1)
|
|
676
|
+
ax.plot(
|
|
677
|
+
analysis["index"].to_numpy(),
|
|
678
|
+
analysis["value"].to_numpy(),
|
|
679
|
+
marker="o",
|
|
680
|
+
color=line_color,
|
|
681
|
+
linewidth=linewidth,
|
|
682
|
+
label="Analysis chunks",
|
|
683
|
+
)
|
|
684
|
+
drifted = analysis.filter(pl.col("drifted"))
|
|
685
|
+
if len(drifted):
|
|
686
|
+
ax.scatter(
|
|
687
|
+
drifted["index"].to_numpy(),
|
|
688
|
+
drifted["value"].to_numpy(),
|
|
689
|
+
marker="D",
|
|
690
|
+
color=drift_color,
|
|
691
|
+
zorder=3,
|
|
692
|
+
label="Drift",
|
|
693
|
+
)
|
|
694
|
+
ax.xaxis.set_major_locator(MaxNLocator(integer=True))
|
|
695
|
+
ax.set_xlabel(data.chunk_label)
|
|
696
|
+
ax.set_ylabel(data.metric_name)
|
|
697
|
+
ax.legend(loc="best")
|
|
698
|
+
|
|
699
|
+
|
|
501
700
|
def plot_drift_on_axis(
|
|
502
701
|
ax: Any,
|
|
503
702
|
resdf: Any,
|
|
@@ -12,6 +12,7 @@ The protocol hierarchy:
|
|
|
12
12
|
- PlottableSufficiency
|
|
13
13
|
- PlottableStats
|
|
14
14
|
- PlottableDriftMVDC
|
|
15
|
+
- PlottableDrift
|
|
15
16
|
- PlottableFeasibility
|
|
16
17
|
"""
|
|
17
18
|
|
|
@@ -122,13 +123,15 @@ class Plottable(Protocol):
|
|
|
122
123
|
"""
|
|
123
124
|
|
|
124
125
|
@property
|
|
125
|
-
def plot_type(
|
|
126
|
+
def plot_type(
|
|
127
|
+
self,
|
|
128
|
+
) -> Literal["balance", "diversity", "sufficiency", "stats", "drift_mvdc", "drift", "feasibility"]:
|
|
126
129
|
"""Return the plot type identifier for routing to appropriate plot function.
|
|
127
130
|
|
|
128
131
|
Returns
|
|
129
132
|
-------
|
|
130
133
|
str
|
|
131
|
-
One of: 'balance', 'diversity', 'sufficiency', 'stats', 'drift_mvdc', 'feasibility'
|
|
134
|
+
One of: 'balance', 'diversity', 'sufficiency', 'stats', 'drift_mvdc', 'drift', 'feasibility'
|
|
132
135
|
"""
|
|
133
136
|
...
|
|
134
137
|
|
|
@@ -289,6 +292,38 @@ class PlottableDriftMVDC(Plottable, Protocol):
|
|
|
289
292
|
def plot_type(self) -> Literal["drift_mvdc"]: ...
|
|
290
293
|
|
|
291
294
|
|
|
295
|
+
@runtime_checkable
|
|
296
|
+
class PlottableDrift(Plottable, Protocol):
|
|
297
|
+
"""Protocol for chunked drift detector outputs.
|
|
298
|
+
|
|
299
|
+
Required attributes:
|
|
300
|
+
- details: Per-chunk results as a polars DataFrame
|
|
301
|
+
- metric_name: Name of the drift metric that each chunk reports
|
|
302
|
+
- plot_type -> 'drift'
|
|
303
|
+
"""
|
|
304
|
+
|
|
305
|
+
@property
|
|
306
|
+
def details(self) -> Any:
|
|
307
|
+
"""Per-chunk drift results.
|
|
308
|
+
|
|
309
|
+
Returns
|
|
310
|
+
-------
|
|
311
|
+
pl.DataFrame
|
|
312
|
+
One row per chunk with the columns ``index``, ``value``, ``lower_threshold``, ``upper_threshold`` and
|
|
313
|
+
``drifted``. A detector that was not chunked holds a mapping of statistics instead, which cannot be
|
|
314
|
+
plotted.
|
|
315
|
+
"""
|
|
316
|
+
...
|
|
317
|
+
|
|
318
|
+
@property
|
|
319
|
+
def metric_name(self) -> str:
|
|
320
|
+
"""Name of the metric that ``value`` reports, such as ``auroc`` or ``mmd2``."""
|
|
321
|
+
...
|
|
322
|
+
|
|
323
|
+
@property
|
|
324
|
+
def plot_type(self) -> Literal["drift"]: ...
|
|
325
|
+
|
|
326
|
+
|
|
292
327
|
class FeasibilityEndpoint(Protocol):
|
|
293
328
|
"""
|
|
294
329
|
One bound on one target, as DataEval's ``Endpoint`` records it; only the fields the plot reads.
|
|
@@ -355,5 +390,6 @@ PlottableType = (
|
|
|
355
390
|
| PlottableSufficiency
|
|
356
391
|
| PlottableStats
|
|
357
392
|
| PlottableDriftMVDC
|
|
393
|
+
| PlottableDrift
|
|
358
394
|
| PlottableFeasibility
|
|
359
395
|
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{dataeval_plots-0.0.15 → dataeval_plots-0.0.16}/src/dataeval_plots/backends/_feasibility_layout.py
RENAMED
|
File without changes
|
|
File without changes
|