dataeval-plots 0.0.9__tar.gz → 0.0.11__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dataeval_plots-0.0.9 → dataeval_plots-0.0.11}/PKG-INFO +86 -1
- {dataeval_plots-0.0.9 → dataeval_plots-0.0.11}/README.md +85 -0
- {dataeval_plots-0.0.9 → dataeval_plots-0.0.11}/src/dataeval_plots/__init__.py +29 -3
- {dataeval_plots-0.0.9 → dataeval_plots-0.0.11}/src/dataeval_plots/_version.py +2 -2
- {dataeval_plots-0.0.9 → dataeval_plots-0.0.11}/src/dataeval_plots/backends/_altair.py +253 -9
- {dataeval_plots-0.0.9 → dataeval_plots-0.0.11}/src/dataeval_plots/backends/_base.py +36 -11
- dataeval_plots-0.0.11/src/dataeval_plots/backends/_feasibility_layout.py +137 -0
- {dataeval_plots-0.0.9 → dataeval_plots-0.0.11}/src/dataeval_plots/backends/_matplotlib.py +183 -0
- {dataeval_plots-0.0.9 → dataeval_plots-0.0.11}/src/dataeval_plots/backends/_plotly.py +259 -9
- {dataeval_plots-0.0.9 → dataeval_plots-0.0.11}/src/dataeval_plots/backends/_seaborn.py +32 -0
- {dataeval_plots-0.0.9 → dataeval_plots-0.0.11}/src/dataeval_plots/backends/_shared.py +364 -1
- {dataeval_plots-0.0.9 → dataeval_plots-0.0.11}/src/dataeval_plots/protocols.py +44 -3
- {dataeval_plots-0.0.9 → dataeval_plots-0.0.11}/.gitignore +0 -0
- {dataeval_plots-0.0.9 → dataeval_plots-0.0.11}/pyproject.toml +0 -0
- {dataeval_plots-0.0.9 → dataeval_plots-0.0.11}/src/dataeval_plots/_registry.py +0 -0
- {dataeval_plots-0.0.9 → dataeval_plots-0.0.11}/src/dataeval_plots/backends/__init__.py +0 -0
- {dataeval_plots-0.0.9 → dataeval_plots-0.0.11}/src/dataeval_plots/py.typed +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: dataeval-plots
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.11
|
|
4
4
|
Summary: DataEval companion package for plotting utilities
|
|
5
5
|
Project-URL: Homepage, https://dataeval.ai/
|
|
6
6
|
Project-URL: Repository, https://github.com/aria-ml/dataeval/
|
|
@@ -175,9 +175,94 @@ This enables:
|
|
|
175
175
|
| `SufficiencyOutput` | "sufficiency" | Learning curves with extrapolation | [dataeval/_workflows.py](../dataeval/src/dataeval/outputs/_workflows.py) |
|
|
176
176
|
| `BaseStatsOutput` | "stats" | Statistical histograms and distributions | [dataeval/_stats.py](../dataeval/src/dataeval/outputs/_stats.py) |
|
|
177
177
|
| `DriftMVDCOutput` | "drift_mvdc" | Drift detection plots (MVDC analysis) | [dataeval/_drift.py](../dataeval/src/dataeval/outputs/_drift.py) |
|
|
178
|
+
| `FeasibilityOutput` | "feasibility" | Which accuracy requirements the evidence demonstrates or rules out | [dataeval/_feasibility.py](../dataeval/src/dataeval/performance/_feasibility.py) |
|
|
178
179
|
|
|
179
180
|
Each output type implements the `Plottable` protocol and can be plotted using any registered backend.
|
|
180
181
|
|
|
182
|
+
## Reading the Feasibility Plot
|
|
183
|
+
|
|
184
|
+
`plot()` draws the result of DataEval's experimental `Feasibility().evaluate(...)`. It answers one question per
|
|
185
|
+
quantity you asked about: for each accuracy or AP requirement, does the evidence demonstrate it, rule it out, or leave
|
|
186
|
+
it unknown?
|
|
187
|
+
|
|
188
|
+
```python
|
|
189
|
+
from dataeval.performance import Feasibility
|
|
190
|
+
from dataeval_plots import plot
|
|
191
|
+
|
|
192
|
+
result = Feasibility().evaluate(dataset, target={"accuracy": 0.95}, embeddings=embeddings)
|
|
193
|
+
|
|
194
|
+
fig = plot(result) # one band per quantity
|
|
195
|
+
fig = plot(result, plot_evidence=True) # one row per evidence source, in place of the bands
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
### What the plot shows
|
|
199
|
+
|
|
200
|
+
Each quantity gets its own band, with a "Required ..." axis (for example "Required accuracy" or "Required AP") and a
|
|
201
|
+
subtitle that says which models the band judges:
|
|
202
|
+
|
|
203
|
+
| Subtitle | Judged models |
|
|
204
|
+
|----------|---------------|
|
|
205
|
+
| Models may use the whole image | Models that may use the whole input |
|
|
206
|
+
| Models may use only the embeddings | Models restricted to the embeddings |
|
|
207
|
+
| Models give one answer per group | Models that give one answer per group |
|
|
208
|
+
|
|
209
|
+
The axis is split into three zones. Each zone classifies the requirements that fall inside it:
|
|
210
|
+
|
|
211
|
+
- **Demonstrated** runs from the axis minimum up to the lower confidence bound of the best tested model. The band
|
|
212
|
+
reads "Requirements through *x*% are demonstrated".
|
|
213
|
+
- **Unknown** lies between the other two zones. The evidence neither demonstrates nor rules out these requirements.
|
|
214
|
+
- **Ruled out** runs from the upper confidence bound of the lowest applicable ceiling to the top of the axis. It holds
|
|
215
|
+
only under the recorded assumptions, and the zone reads "ruled out under the recorded assumptions" wherever the
|
|
216
|
+
label fits.
|
|
217
|
+
|
|
218
|
+
Each of your requirements is marked on the axis, and the marker states the determination in words:
|
|
219
|
+
|
|
220
|
+
| Marker | Meaning |
|
|
221
|
+
|--------|---------|
|
|
222
|
+
| "95% requirement: demonstrated ✓" | The requirement is at or below the lower confidence bound |
|
|
223
|
+
| "not enough evidence ?" | The requirement falls in the unknown zone, and the evidence cannot decide it |
|
|
224
|
+
| "ruled out under the recorded assumptions ✗" | The requirement is above the upper confidence bound of a ceiling |
|
|
225
|
+
| "not ruled out ?" | The requirement also falls in the unknown zone: it is not above any ceiling's upper confidence bound, and no tested model reaches it |
|
|
226
|
+
|
|
227
|
+
The caption under the band reports the observed score next to the lower confidence bound it supports, for example
|
|
228
|
+
"Your model scored 91.0% on the 2,000 evaluation images. Allowing for sampling uncertainty gives a lower
|
|
229
|
+
confidence bound of 88.5%." The sentence about the observed score appears only when an observed score exists. The
|
|
230
|
+
caption then states that the bounds concern new images drawn from the same population, and that confidence holds
|
|
231
|
+
jointly across the whole evaluation (for example "Confidence is 95% jointly across this evaluation").
|
|
232
|
+
|
|
233
|
+
Numbers read as DataEval prints them. Bounds are given to 0.1%, a lower bound rounded down and an upper bound rounded
|
|
234
|
+
up, so rounding never overstates what the evidence shows. Requirements and confidence keep their own decimals, such as
|
|
235
|
+
"95.5% requirement" or "Confidence is 97.5%".
|
|
236
|
+
|
|
237
|
+
When a ceiling bounds the band, the caption adds "Under the recorded assumptions, requirements above *x*% are ruled out
|
|
238
|
+
at the stated confidence." A group-based ceiling carries the extra note that its validity argument awaits lead-scientist
|
|
239
|
+
sign-off.
|
|
240
|
+
|
|
241
|
+
A ceiling that applies only to narrower models than the band judges never draws a zone. It appears as a note instead,
|
|
242
|
+
for example that the group-based ceiling does not limit models that distinguish individual images.
|
|
243
|
+
|
|
244
|
+
### Other outcomes
|
|
245
|
+
|
|
246
|
+
- **Ruled out.** A requirement above the lowest applicable ceiling's upper confidence bound is marked "ruled out
|
|
247
|
+
under the recorded assumptions ✗". The ruled-out zone covers everything above that bound, and the caption reads
|
|
248
|
+
"Under the recorded assumptions, requirements above *x*% are ruled out at the stated confidence."
|
|
249
|
+
- **Evidence conflict.** When bounds from different sources contradict each other, the band draws no zones. A dashed
|
|
250
|
+
warning outline reads "Evidence conflicts: check assumptions", both clashing bounds are marked, and the caption
|
|
251
|
+
says that the bounds disagree and to check the assumptions, including whether evaluation images entered training.
|
|
252
|
+
Even when the assumptions hold, the probability that any bound fails is at most the stated alpha (for example 5% at
|
|
253
|
+
95% confidence).
|
|
254
|
+
- **Bounds at 0% or 100%.** A lower confidence bound of 0% demonstrates nothing and an upper confidence bound of 100%
|
|
255
|
+
rules nothing out, so neither draws a zone.
|
|
256
|
+
|
|
257
|
+
### Showing the evidence
|
|
258
|
+
|
|
259
|
+
`plot_evidence=True` replaces each band with one row per evidence source. Each row shows the observed score as a dot
|
|
260
|
+
and the confidence bound as a tick, with a legend line explaining both: the dot is the score observed on the
|
|
261
|
+
evaluation images, and the tick is the lower (or upper) confidence bound for new images from the same population.
|
|
262
|
+
Evidence that does not apply to the judged models is drawn faint and labeled with the models it applies to. A source
|
|
263
|
+
that produced no bound for the quantity is listed faint as "no bound available", and when no source has a bound the
|
|
264
|
+
view says "No bound is available yet." instead of showing empty axes.
|
|
265
|
+
|
|
181
266
|
## Extending the Package
|
|
182
267
|
|
|
183
268
|
### Creating Custom Outputs
|
|
@@ -129,9 +129,94 @@ This enables:
|
|
|
129
129
|
| `SufficiencyOutput` | "sufficiency" | Learning curves with extrapolation | [dataeval/_workflows.py](../dataeval/src/dataeval/outputs/_workflows.py) |
|
|
130
130
|
| `BaseStatsOutput` | "stats" | Statistical histograms and distributions | [dataeval/_stats.py](../dataeval/src/dataeval/outputs/_stats.py) |
|
|
131
131
|
| `DriftMVDCOutput` | "drift_mvdc" | Drift detection plots (MVDC analysis) | [dataeval/_drift.py](../dataeval/src/dataeval/outputs/_drift.py) |
|
|
132
|
+
| `FeasibilityOutput` | "feasibility" | Which accuracy requirements the evidence demonstrates or rules out | [dataeval/_feasibility.py](../dataeval/src/dataeval/performance/_feasibility.py) |
|
|
132
133
|
|
|
133
134
|
Each output type implements the `Plottable` protocol and can be plotted using any registered backend.
|
|
134
135
|
|
|
136
|
+
## Reading the Feasibility Plot
|
|
137
|
+
|
|
138
|
+
`plot()` draws the result of DataEval's experimental `Feasibility().evaluate(...)`. It answers one question per
|
|
139
|
+
quantity you asked about: for each accuracy or AP requirement, does the evidence demonstrate it, rule it out, or leave
|
|
140
|
+
it unknown?
|
|
141
|
+
|
|
142
|
+
```python
|
|
143
|
+
from dataeval.performance import Feasibility
|
|
144
|
+
from dataeval_plots import plot
|
|
145
|
+
|
|
146
|
+
result = Feasibility().evaluate(dataset, target={"accuracy": 0.95}, embeddings=embeddings)
|
|
147
|
+
|
|
148
|
+
fig = plot(result) # one band per quantity
|
|
149
|
+
fig = plot(result, plot_evidence=True) # one row per evidence source, in place of the bands
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
### What the plot shows
|
|
153
|
+
|
|
154
|
+
Each quantity gets its own band, with a "Required ..." axis (for example "Required accuracy" or "Required AP") and a
|
|
155
|
+
subtitle that says which models the band judges:
|
|
156
|
+
|
|
157
|
+
| Subtitle | Judged models |
|
|
158
|
+
|----------|---------------|
|
|
159
|
+
| Models may use the whole image | Models that may use the whole input |
|
|
160
|
+
| Models may use only the embeddings | Models restricted to the embeddings |
|
|
161
|
+
| Models give one answer per group | Models that give one answer per group |
|
|
162
|
+
|
|
163
|
+
The axis is split into three zones. Each zone classifies the requirements that fall inside it:
|
|
164
|
+
|
|
165
|
+
- **Demonstrated** runs from the axis minimum up to the lower confidence bound of the best tested model. The band
|
|
166
|
+
reads "Requirements through *x*% are demonstrated".
|
|
167
|
+
- **Unknown** lies between the other two zones. The evidence neither demonstrates nor rules out these requirements.
|
|
168
|
+
- **Ruled out** runs from the upper confidence bound of the lowest applicable ceiling to the top of the axis. It holds
|
|
169
|
+
only under the recorded assumptions, and the zone reads "ruled out under the recorded assumptions" wherever the
|
|
170
|
+
label fits.
|
|
171
|
+
|
|
172
|
+
Each of your requirements is marked on the axis, and the marker states the determination in words:
|
|
173
|
+
|
|
174
|
+
| Marker | Meaning |
|
|
175
|
+
|--------|---------|
|
|
176
|
+
| "95% requirement: demonstrated ✓" | The requirement is at or below the lower confidence bound |
|
|
177
|
+
| "not enough evidence ?" | The requirement falls in the unknown zone, and the evidence cannot decide it |
|
|
178
|
+
| "ruled out under the recorded assumptions ✗" | The requirement is above the upper confidence bound of a ceiling |
|
|
179
|
+
| "not ruled out ?" | The requirement also falls in the unknown zone: it is not above any ceiling's upper confidence bound, and no tested model reaches it |
|
|
180
|
+
|
|
181
|
+
The caption under the band reports the observed score next to the lower confidence bound it supports, for example
|
|
182
|
+
"Your model scored 91.0% on the 2,000 evaluation images. Allowing for sampling uncertainty gives a lower
|
|
183
|
+
confidence bound of 88.5%." The sentence about the observed score appears only when an observed score exists. The
|
|
184
|
+
caption then states that the bounds concern new images drawn from the same population, and that confidence holds
|
|
185
|
+
jointly across the whole evaluation (for example "Confidence is 95% jointly across this evaluation").
|
|
186
|
+
|
|
187
|
+
Numbers read as DataEval prints them. Bounds are given to 0.1%, a lower bound rounded down and an upper bound rounded
|
|
188
|
+
up, so rounding never overstates what the evidence shows. Requirements and confidence keep their own decimals, such as
|
|
189
|
+
"95.5% requirement" or "Confidence is 97.5%".
|
|
190
|
+
|
|
191
|
+
When a ceiling bounds the band, the caption adds "Under the recorded assumptions, requirements above *x*% are ruled out
|
|
192
|
+
at the stated confidence." A group-based ceiling carries the extra note that its validity argument awaits lead-scientist
|
|
193
|
+
sign-off.
|
|
194
|
+
|
|
195
|
+
A ceiling that applies only to narrower models than the band judges never draws a zone. It appears as a note instead,
|
|
196
|
+
for example that the group-based ceiling does not limit models that distinguish individual images.
|
|
197
|
+
|
|
198
|
+
### Other outcomes
|
|
199
|
+
|
|
200
|
+
- **Ruled out.** A requirement above the lowest applicable ceiling's upper confidence bound is marked "ruled out
|
|
201
|
+
under the recorded assumptions ✗". The ruled-out zone covers everything above that bound, and the caption reads
|
|
202
|
+
"Under the recorded assumptions, requirements above *x*% are ruled out at the stated confidence."
|
|
203
|
+
- **Evidence conflict.** When bounds from different sources contradict each other, the band draws no zones. A dashed
|
|
204
|
+
warning outline reads "Evidence conflicts: check assumptions", both clashing bounds are marked, and the caption
|
|
205
|
+
says that the bounds disagree and to check the assumptions, including whether evaluation images entered training.
|
|
206
|
+
Even when the assumptions hold, the probability that any bound fails is at most the stated alpha (for example 5% at
|
|
207
|
+
95% confidence).
|
|
208
|
+
- **Bounds at 0% or 100%.** A lower confidence bound of 0% demonstrates nothing and an upper confidence bound of 100%
|
|
209
|
+
rules nothing out, so neither draws a zone.
|
|
210
|
+
|
|
211
|
+
### Showing the evidence
|
|
212
|
+
|
|
213
|
+
`plot_evidence=True` replaces each band with one row per evidence source. Each row shows the observed score as a dot
|
|
214
|
+
and the confidence bound as a tick, with a legend line explaining both: the dot is the score observed on the
|
|
215
|
+
evaluation images, and the tick is the lower (or upper) confidence bound for new images from the same population.
|
|
216
|
+
Evidence that does not apply to the judged models is drawn faint and labeled with the models it applies to. A source
|
|
217
|
+
that produced no bound for the quantity is listed faint as "no bound available", and when no source has a bound the
|
|
218
|
+
view says "No bound is available yet." instead of showing empty axes.
|
|
219
|
+
|
|
135
220
|
## Extending the Package
|
|
136
221
|
|
|
137
222
|
### Creating Custom Outputs
|
|
@@ -20,6 +20,7 @@ from dataeval_plots.protocols import (
|
|
|
20
20
|
PlottableBalance,
|
|
21
21
|
PlottableDiversity,
|
|
22
22
|
PlottableDriftMVDC,
|
|
23
|
+
PlottableFeasibility,
|
|
23
24
|
PlottableStats,
|
|
24
25
|
PlottableSufficiency,
|
|
25
26
|
PlottableType,
|
|
@@ -97,6 +98,31 @@ def plot(
|
|
|
97
98
|
) -> Any: ...
|
|
98
99
|
|
|
99
100
|
|
|
101
|
+
@overload
|
|
102
|
+
def plot(
|
|
103
|
+
output: PlottableFeasibility,
|
|
104
|
+
/,
|
|
105
|
+
figsize: tuple[float, float] | None = None,
|
|
106
|
+
backend: str | None = None,
|
|
107
|
+
*,
|
|
108
|
+
plot_evidence: bool = False,
|
|
109
|
+
) -> Any:
|
|
110
|
+
"""
|
|
111
|
+
Plot a feasibility result as one band per quantity.
|
|
112
|
+
|
|
113
|
+
Each band classifies accuracy requirements as demonstrated (up to the best tested model's lower confidence
|
|
114
|
+
bound), unknown, or ruled out (above the lowest applicable ceiling's upper confidence bound, under the recorded
|
|
115
|
+
assumptions). Confidence holds jointly across the whole evaluation.
|
|
116
|
+
|
|
117
|
+
Parameters
|
|
118
|
+
----------
|
|
119
|
+
plot_evidence : bool, default False
|
|
120
|
+
Replace each band with one row per evidence source, showing the observed score as a dot and the confidence
|
|
121
|
+
bound as a tick.
|
|
122
|
+
"""
|
|
123
|
+
...
|
|
124
|
+
|
|
125
|
+
|
|
100
126
|
@overload
|
|
101
127
|
def plot(
|
|
102
128
|
output: Dataset,
|
|
@@ -104,10 +130,10 @@ def plot(
|
|
|
104
130
|
figsize: tuple[float, float] | None = None,
|
|
105
131
|
backend: str | None = None,
|
|
106
132
|
*,
|
|
107
|
-
indices: Sequence[int],
|
|
133
|
+
indices: Sequence[int] | None = None,
|
|
108
134
|
images_per_row: int = 3,
|
|
109
135
|
show_labels: bool = False,
|
|
110
|
-
show_metadata: bool = False,
|
|
136
|
+
show_metadata: bool | Sequence[str] = False,
|
|
111
137
|
additional_metadata: Sequence[dict[str, Any]] | None = None,
|
|
112
138
|
) -> Any: ...
|
|
113
139
|
|
|
@@ -138,7 +164,7 @@ def plot(
|
|
|
138
164
|
Plotting backend ('matplotlib', 'seaborn', 'plotly', 'altair').
|
|
139
165
|
If None, uses default backend.
|
|
140
166
|
**kwargs
|
|
141
|
-
Backend-specific plotting parameters
|
|
167
|
+
Backend-specific plotting parameters, such as ``plot_evidence`` for feasibility outputs
|
|
142
168
|
|
|
143
169
|
Returns
|
|
144
170
|
-------
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '0.0.
|
|
22
|
-
__version_tuple__ = version_tuple = (0, 0,
|
|
21
|
+
__version__ = version = '0.0.11'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 0, 11)
|
|
23
23
|
|
|
24
24
|
__commit_id__ = commit_id = None
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
import textwrap
|
|
5
6
|
from collections.abc import Iterable, Mapping, Sequence
|
|
6
7
|
from typing import Any, Literal
|
|
7
8
|
|
|
@@ -9,12 +10,17 @@ import numpy as np
|
|
|
9
10
|
import polars as pl
|
|
10
11
|
from numpy.typing import NDArray
|
|
11
12
|
|
|
13
|
+
from dataeval_plots.backends import _feasibility_layout as layout
|
|
12
14
|
from dataeval_plots.backends._base import BasePlottingBackend
|
|
13
15
|
from dataeval_plots.backends._shared import (
|
|
14
16
|
CHANNELWISE_METRICS,
|
|
17
|
+
FEASIBILITY_COLORS,
|
|
18
|
+
FEASIBILITY_CONFLICT_TEXT,
|
|
19
|
+
FeasibilityBand,
|
|
15
20
|
calculate_projection,
|
|
16
21
|
draw_bounding_boxes,
|
|
17
22
|
extract_boxes_and_labels,
|
|
23
|
+
feasibility_zones,
|
|
18
24
|
format_label_from_target,
|
|
19
25
|
image_to_base64_png,
|
|
20
26
|
image_to_hwc,
|
|
@@ -25,6 +31,7 @@ from dataeval_plots.backends._shared import (
|
|
|
25
31
|
prepare_balance_data,
|
|
26
32
|
prepare_diversity_data,
|
|
27
33
|
prepare_drift_data,
|
|
34
|
+
prepare_feasibility_data,
|
|
28
35
|
project_steps,
|
|
29
36
|
validate_class_names,
|
|
30
37
|
)
|
|
@@ -33,10 +40,149 @@ from dataeval_plots.protocols import (
|
|
|
33
40
|
PlottableBalance,
|
|
34
41
|
PlottableDiversity,
|
|
35
42
|
PlottableDriftMVDC,
|
|
43
|
+
PlottableFeasibility,
|
|
36
44
|
PlottableStats,
|
|
37
45
|
PlottableSufficiency,
|
|
38
46
|
)
|
|
39
47
|
|
|
48
|
+
# Feasibility: one layer per mark, positioned in the band's data units (y in inches) like the matplotlib backend.
|
|
49
|
+
# DejaVu Sans, as matplotlib, where installed: the layout is measured in it and it has every glyph used.
|
|
50
|
+
_FONT = "DejaVu Sans, sans-serif"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _mark(layers: list[Any], axes: tuple[Any, Any], mark: str, row: dict[str, Any], **props: Any) -> None:
|
|
54
|
+
"""Draw ``mark`` once at ``row``'s x and y on the band's ``axes``, with its x2, y2 and text if given."""
|
|
55
|
+
import altair as alt
|
|
56
|
+
|
|
57
|
+
x, y = axes
|
|
58
|
+
channels = {k: "text:N" if k == "text" else k for k in ("x2", "y2", "text") if k in row}
|
|
59
|
+
layers.append(getattr(alt.Chart(alt.Data(values=[row])), f"mark_{mark}")(**props).encode(x=x, y=y, **channels))
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _text(
|
|
63
|
+
layers: list[Any],
|
|
64
|
+
axes: tuple[Any, Any],
|
|
65
|
+
x: float,
|
|
66
|
+
y: float,
|
|
67
|
+
text: str,
|
|
68
|
+
size: float = layout.LABEL_PT,
|
|
69
|
+
backing: float | None = None,
|
|
70
|
+
**props: Any,
|
|
71
|
+
) -> None:
|
|
72
|
+
"""Text on the band's axes, centered unless ``align`` or ``baseline`` say otherwise; given the x units per inch
|
|
73
|
+
as ``backing``, a white box behind it keeps it legible over lines."""
|
|
74
|
+
lines, line_h = text.count("\n") + 1, layout.px(size) * 1.25
|
|
75
|
+
align, baseline = props.setdefault("align", "center"), props.setdefault("baseline", "middle")
|
|
76
|
+
if backing is not None:
|
|
77
|
+
bold = 1.1 if props.get("fontWeight") == "bold" else 1.0 # measured at regular weight
|
|
78
|
+
w, h = layout.text_width(text, size) * bold + 0.04, lines * line_h / layout.PX
|
|
79
|
+
x0 = x - {"left": 0.02, "center": w / 2, "right": w - 0.02}[align] * backing
|
|
80
|
+
y0 = y - {"top": h, "middle": h / 2, "bottom": 0}[baseline]
|
|
81
|
+
_mark(layers, axes, "rect", {"x": x0, "x2": x0 + w * backing, "y": y0, "y2": y0 + h}, color="white")
|
|
82
|
+
# Vega puts the first line, not the block, at a middle baseline
|
|
83
|
+
dy = -(lines - 1) * line_h / 2 if baseline == "middle" else 0
|
|
84
|
+
props = {"color": FEASIBILITY_COLORS["ink"], "fontSize": layout.px(size), "lineHeight": line_h, "dy": dy, **props}
|
|
85
|
+
_mark(layers, axes, "text", {"x": x, "y": y, "text": text}, lineBreak="\n", font=_FONT, **props)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _side(x: float, text: str, prefer_left: bool, x_range: tuple[float, float], ax_w: float) -> tuple[float, str]:
|
|
89
|
+
"""Where a label beside its mark starts and how it aligns: the preferred side, unless that leaves the axes."""
|
|
90
|
+
pad = layout.PAD * (x_range[1] - x_range[0]) / ax_w
|
|
91
|
+
return (x - pad, "right") if layout.goes_left(x, text, prefer_left, x_range, ax_w) else (x + pad, "left")
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _altair_requirements(
|
|
95
|
+
layers: list[Any],
|
|
96
|
+
axes: tuple[Any, Any],
|
|
97
|
+
placed: list[layout.Placed],
|
|
98
|
+
per_in: float,
|
|
99
|
+
y0: float,
|
|
100
|
+
y_label: float,
|
|
101
|
+
) -> None:
|
|
102
|
+
"""Dashed requirement lines from ``y0`` up to their labels' level, labels on top."""
|
|
103
|
+
ink = FEASIBILITY_COLORS["ink"]
|
|
104
|
+
for x, text, ha, level in placed:
|
|
105
|
+
y = y_label + level * layout.LEVEL
|
|
106
|
+
_mark(
|
|
107
|
+
layers, axes, "rule", {"x": x, "y": y0, "y2": y}, color=ink, strokeWidth=layout.px(1.2), strokeDash=[4, 3]
|
|
108
|
+
)
|
|
109
|
+
_mark(
|
|
110
|
+
layers, axes, "point", {"x": x, "y": y}, shape="triangle-down", filled=True, size=60, color=ink, opacity=1
|
|
111
|
+
)
|
|
112
|
+
dx = layout.PAD * per_in if ha == "right" else -layout.PAD * per_in
|
|
113
|
+
_text(layers, axes, x + dx, y + 0.06, text, align=ha, baseline="bottom")
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _altair_band(
|
|
117
|
+
layers: list[Any],
|
|
118
|
+
axes: tuple[Any, Any],
|
|
119
|
+
band: FeasibilityBand,
|
|
120
|
+
ax_w: float,
|
|
121
|
+
placed: list[layout.Placed],
|
|
122
|
+
) -> None:
|
|
123
|
+
"""Zones, boundaries and requirement markers of one band."""
|
|
124
|
+
colors = FEASIBILITY_COLORS
|
|
125
|
+
lo, hi = band.x_range
|
|
126
|
+
per_in = (hi - lo) / ax_w
|
|
127
|
+
y0, y1 = layout.BOUNDARY, layout.BOUNDARY + layout.BAR
|
|
128
|
+
for kind, start, end, label in feasibility_zones(band):
|
|
129
|
+
zone = {"x": start, "x2": end, "y": y0, "y2": y1}
|
|
130
|
+
_mark(layers, axes, "rect", zone, color=colors[f"{kind}_fill"], stroke="white", strokeWidth=layout.px(1.5))
|
|
131
|
+
fit = layout.zone_label(band, start, end, label, per_in)
|
|
132
|
+
if fit:
|
|
133
|
+
_text(layers, axes, fit[0], y0 + layout.BAR / 2, fit[1])
|
|
134
|
+
if band.conflict:
|
|
135
|
+
dashed = {"stroke": colors["warning"], "strokeWidth": layout.px(2), "strokeDash": [6, 4]}
|
|
136
|
+
_mark(layers, axes, "rect", {"x": lo, "x2": hi, "y": y0, "y2": y1}, fillOpacity=0, **dashed)
|
|
137
|
+
for x, text, kind, left in band.boundary_labels:
|
|
138
|
+
color = colors["demonstrated" if kind == "lower" else "ruled_out"]
|
|
139
|
+
_mark(layers, axes, "rule", {"x": x, "y": y0 - 0.06, "y2": y1 + 0.06}, color=color, strokeWidth=layout.px(2.5))
|
|
140
|
+
at, align = _side(x, text, left, band.x_range, ax_w)
|
|
141
|
+
_text(layers, axes, at, y0 - 0.1, text, align=align, baseline="top")
|
|
142
|
+
_altair_requirements(layers, axes, placed, per_in, y0, y1 + 0.2)
|
|
143
|
+
if band.conflict: # over the requirement and boundary lines
|
|
144
|
+
center = ((lo + hi) / 2, y0 + layout.BAR / 2)
|
|
145
|
+
_text(layers, axes, *center, FEASIBILITY_CONFLICT_TEXT, 11, per_in, fontWeight="bold")
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _altair_evidence(
|
|
149
|
+
layers: list[Any],
|
|
150
|
+
axes: tuple[Any, Any],
|
|
151
|
+
band: FeasibilityBand,
|
|
152
|
+
x_range: tuple[float, float],
|
|
153
|
+
ax_w: float,
|
|
154
|
+
placed: list[layout.Placed],
|
|
155
|
+
) -> None:
|
|
156
|
+
"""One row per source: a tick at its bound, a dot at its observed score, both labeled."""
|
|
157
|
+
colors, muted = FEASIBILITY_COLORS, FEASIBILITY_COLORS["ink_muted"]
|
|
158
|
+
per_in = (x_range[1] - x_range[0]) / ax_w
|
|
159
|
+
rows_top, ys = layout.evidence_rows(band)
|
|
160
|
+
_altair_requirements(layers, axes, placed, per_in, 0, rows_top + 0.12)
|
|
161
|
+
at = x_range[0] + layout.PAD * per_in # text in place of marks
|
|
162
|
+
if band.evidence_note:
|
|
163
|
+
_text(
|
|
164
|
+
layers, axes, at, rows_top - layout.ROW / 2, band.evidence_note, backing=per_in, align="left", color=muted
|
|
165
|
+
)
|
|
166
|
+
for y, row in zip(ys, band.evidence):
|
|
167
|
+
# Bound labeled below, observed above, so the two never collide; a white backing keeps them legible over
|
|
168
|
+
# the requirement lines
|
|
169
|
+
ink = {"color": muted if row.faint else colors["ink"], "backing": per_in}
|
|
170
|
+
source = textwrap.fill(row.label, layout.SOURCE_WRAP)
|
|
171
|
+
_text(layers, axes, x_range[0] - 0.1 * per_in, y, source, align="right", color=ink["color"])
|
|
172
|
+
if row.bound is None:
|
|
173
|
+
_text(layers, axes, at, y, row.bound_label, backing=per_in, align="left", color=muted)
|
|
174
|
+
continue
|
|
175
|
+
color = colors["demonstrated" if row.bound_type == "lower" else "ruled_out"]
|
|
176
|
+
mark = {"color": color, "opacity": 0.35 if row.faint else 1.0}
|
|
177
|
+
_mark(layers, axes, "rule", {"x": row.bound, "y": y - 0.12, "y2": y + 0.12}, strokeWidth=layout.px(2.5), **mark)
|
|
178
|
+
side, align = _side(row.bound, row.bound_label, row.bound_left, x_range, ax_w)
|
|
179
|
+
_text(layers, axes, side, y - 0.14, row.bound_label, align=align, baseline="top", **ink)
|
|
180
|
+
if row.observed is not None and row.observed_label is not None:
|
|
181
|
+
_mark(layers, axes, "rule", {"x": row.bound, "x2": row.observed, "y": y}, strokeWidth=layout.px(3), **mark)
|
|
182
|
+
_mark(layers, axes, "circle", {"x": row.observed, "y": y}, size=100, **mark)
|
|
183
|
+
side, align = _side(row.observed, row.observed_label, not row.bound_left, x_range, ax_w)
|
|
184
|
+
_text(layers, axes, side, y + 0.29, row.observed_label, align=align, baseline="top", **ink)
|
|
185
|
+
|
|
40
186
|
|
|
41
187
|
class AltairBackend(BasePlottingBackend):
|
|
42
188
|
"""Altair implementation of plotting backend."""
|
|
@@ -676,11 +822,11 @@ class AltairBackend(BasePlottingBackend):
|
|
|
676
822
|
def _plot_image_grid(
|
|
677
823
|
self,
|
|
678
824
|
dataset: Dataset,
|
|
679
|
-
indices: Sequence[int],
|
|
825
|
+
indices: Sequence[int] | None = None,
|
|
680
826
|
images_per_row: int = 3,
|
|
681
827
|
figsize: tuple[float, float] | None = None,
|
|
682
828
|
show_labels: bool = False,
|
|
683
|
-
show_metadata: bool = False,
|
|
829
|
+
show_metadata: bool | Sequence[str] = False,
|
|
684
830
|
additional_metadata: Sequence[dict[str, Any]] | None = None,
|
|
685
831
|
) -> Any:
|
|
686
832
|
"""
|
|
@@ -690,16 +836,17 @@ class AltairBackend(BasePlottingBackend):
|
|
|
690
836
|
----------
|
|
691
837
|
dataset : Dataset
|
|
692
838
|
MAITE-compatible dataset containing images
|
|
693
|
-
indices : Sequence[int]
|
|
694
|
-
Indices of images to plot from the dataset
|
|
839
|
+
indices : Sequence[int] or None, default None
|
|
840
|
+
Indices of images to plot from the dataset. None plots every item.
|
|
695
841
|
images_per_row : int, default 3
|
|
696
842
|
Number of images to display per row
|
|
697
843
|
figsize : tuple[float, float] or None, default None
|
|
698
844
|
Figure size in inches (width, height)
|
|
699
845
|
show_labels : bool, default False
|
|
700
846
|
Whether to display labels extracted from targets
|
|
701
|
-
show_metadata : bool, default False
|
|
702
|
-
Whether to display metadata from the dataset items
|
|
847
|
+
show_metadata : bool or Sequence[str], default False
|
|
848
|
+
Whether to display metadata from the dataset items. A sequence of keys
|
|
849
|
+
displays only those, in the order given.
|
|
703
850
|
additional_metadata : Sequence[dict[str, Any]] or None, default None
|
|
704
851
|
Additional metadata to display for each image (must match length of indices)
|
|
705
852
|
|
|
@@ -716,6 +863,8 @@ class AltairBackend(BasePlottingBackend):
|
|
|
716
863
|
import altair as alt
|
|
717
864
|
import pandas as pd
|
|
718
865
|
|
|
866
|
+
indices = range(len(dataset)) if indices is None else indices
|
|
867
|
+
|
|
719
868
|
# Validate additional_metadata length
|
|
720
869
|
if additional_metadata is not None and len(additional_metadata) != len(indices):
|
|
721
870
|
raise ValueError(
|
|
@@ -790,9 +939,9 @@ class AltairBackend(BasePlottingBackend):
|
|
|
790
939
|
title_parts.append(label_str)
|
|
791
940
|
|
|
792
941
|
if show_metadata and metadata:
|
|
793
|
-
#
|
|
794
|
-
|
|
795
|
-
title_parts.extend(
|
|
942
|
+
# Every key, or just the named ones, as key: value pairs
|
|
943
|
+
keys = metadata if isinstance(show_metadata, bool) else [k for k in show_metadata if k in metadata]
|
|
944
|
+
title_parts.extend(f"{k}: {metadata[k]}" for k in keys)
|
|
796
945
|
|
|
797
946
|
# Create title or use empty string
|
|
798
947
|
title = " | ".join(title_parts) if title_parts else ""
|
|
@@ -1014,3 +1163,98 @@ class AltairBackend(BasePlottingBackend):
|
|
|
1014
1163
|
grid = grid.properties(title=title)
|
|
1015
1164
|
|
|
1016
1165
|
return grid
|
|
1166
|
+
|
|
1167
|
+
def _plot_feasibility(
|
|
1168
|
+
self,
|
|
1169
|
+
output: PlottableFeasibility,
|
|
1170
|
+
figsize: tuple[float, float] | None = None,
|
|
1171
|
+
plot_evidence: bool = False,
|
|
1172
|
+
) -> Any: # alt.VConcatChart
|
|
1173
|
+
"""
|
|
1174
|
+
Plot each quantity's requirements as a band of demonstrated, unknown and ruled-out zones.
|
|
1175
|
+
|
|
1176
|
+
Parameters
|
|
1177
|
+
----------
|
|
1178
|
+
output : PlottableFeasibility
|
|
1179
|
+
The feasibility output object to plot
|
|
1180
|
+
figsize : tuple[float, float] or None, default None
|
|
1181
|
+
Figure size in inches (width, height), drawn at 100 pixels per inch. If None, 9 inches wide and as tall
|
|
1182
|
+
as the bands need. An explicit height stretches or squeezes the band views to meet it; captions keep
|
|
1183
|
+
their height.
|
|
1184
|
+
plot_evidence : bool, default False
|
|
1185
|
+
Whether to plot each source's observed score and confidence bound instead of the bands
|
|
1186
|
+
|
|
1187
|
+
Returns
|
|
1188
|
+
-------
|
|
1189
|
+
alt.VConcatChart
|
|
1190
|
+
One band chart and its caption per quantity, stacked
|
|
1191
|
+
"""
|
|
1192
|
+
import altair as alt
|
|
1193
|
+
|
|
1194
|
+
ink, muted = FEASIBILITY_COLORS["ink"], FEASIBILITY_COLORS["ink_muted"]
|
|
1195
|
+
width = figsize[0] if figsize else 9.0
|
|
1196
|
+
gutter, ax_w, height, blocks = layout.blocks(prepare_feasibility_data(output), plot_evidence, width)
|
|
1197
|
+
# Only the band views stretch, so the rest keeps its height and the total meets the requested one
|
|
1198
|
+
views = sum(block[3] for block in blocks)
|
|
1199
|
+
stretch = max(figsize[1] - (height - views), 0.1) / views if figsize else 1.0
|
|
1200
|
+
view = alt.ViewBackground(stroke=None) # per chart, so the result stays composable
|
|
1201
|
+
|
|
1202
|
+
charts = []
|
|
1203
|
+
for band, x_range, xlabel, ax_h, placed, texts in blocks:
|
|
1204
|
+
axis = alt.Axis(
|
|
1205
|
+
format=".0%",
|
|
1206
|
+
tickCount=max(2, round(ax_w / 1.3)), # about matplotlib's tick density
|
|
1207
|
+
tickMinStep=0.01,
|
|
1208
|
+
title=xlabel,
|
|
1209
|
+
titleFont=_FONT,
|
|
1210
|
+
titleFontSize=layout.px(11),
|
|
1211
|
+
titleFontWeight="bold",
|
|
1212
|
+
titleColor=ink,
|
|
1213
|
+
labelFont=_FONT,
|
|
1214
|
+
labelFontSize=layout.px(layout.LABEL_PT),
|
|
1215
|
+
labelColor=muted,
|
|
1216
|
+
tickColor=muted,
|
|
1217
|
+
domainColor=muted,
|
|
1218
|
+
grid=plot_evidence,
|
|
1219
|
+
gridColor="#e4e3df",
|
|
1220
|
+
)
|
|
1221
|
+
axes = (
|
|
1222
|
+
alt.X("x:Q", scale=alt.Scale(domain=list(x_range), nice=False, zero=False), axis=axis),
|
|
1223
|
+
alt.Y("y:Q", scale=alt.Scale(domain=[0, ax_h], nice=False, zero=False), axis=None),
|
|
1224
|
+
)
|
|
1225
|
+
layers: list[Any] = []
|
|
1226
|
+
if plot_evidence:
|
|
1227
|
+
_altair_evidence(layers, axes, band, x_range, ax_w, placed)
|
|
1228
|
+
else:
|
|
1229
|
+
_altair_band(layers, axes, band, ax_w, placed)
|
|
1230
|
+
title = alt.TitleParams(
|
|
1231
|
+
band.subtitle,
|
|
1232
|
+
anchor="start",
|
|
1233
|
+
frame="group",
|
|
1234
|
+
font=_FONT,
|
|
1235
|
+
fontSize=layout.px(10),
|
|
1236
|
+
fontWeight="normal",
|
|
1237
|
+
color=muted,
|
|
1238
|
+
)
|
|
1239
|
+
size = {"width": ax_w * layout.PX, "height": ax_h * layout.PX * stretch}
|
|
1240
|
+
charts.append(alt.layer(*layers).properties(title=title, view=view, **size))
|
|
1241
|
+
|
|
1242
|
+
# Below the axes, from the figure margin: the evidence legend, the caption paragraph, then the notes
|
|
1243
|
+
lines, top = [], 0.0
|
|
1244
|
+
for text, points, color in texts:
|
|
1245
|
+
style = {
|
|
1246
|
+
"font": _FONT,
|
|
1247
|
+
"fontSize": layout.px(points),
|
|
1248
|
+
"color": color,
|
|
1249
|
+
"lineHeight": layout.LINE * layout.PX,
|
|
1250
|
+
}
|
|
1251
|
+
x, y = alt.value(-(gutter - layout.MARGIN) * layout.PX), alt.value(top * layout.PX)
|
|
1252
|
+
lines.append(
|
|
1253
|
+
alt.Chart(alt.Data(values=[{"text": text}]))
|
|
1254
|
+
.mark_text(align="left", baseline="top", lineBreak="\n", **style)
|
|
1255
|
+
.encode(x=x, y=y, text="text:N")
|
|
1256
|
+
)
|
|
1257
|
+
top += layout.text_height(text)
|
|
1258
|
+
charts.append(alt.layer(*lines).properties(width=ax_w * layout.PX, height=top * layout.PX, view=view))
|
|
1259
|
+
|
|
1260
|
+
return alt.vconcat(*charts, spacing=10)
|