dataeval-plots 0.0.13__tar.gz → 0.0.15__tar.gz

{dataeval_plots-0.0.13 → dataeval_plots-0.0.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: dataeval-plots
-Version: 0.0.13
+Version: 0.0.15
 Summary: DataEval companion package for plotting utilities
 Project-URL: Homepage, https://dataeval.ai/
 Project-URL: Repository, https://github.com/aria-ml/dataeval/
@@ -208,60 +208,70 @@ subtitle that says which models the band judges:
 
 The axis is split into three zones. Each zone classifies the requirements that fall inside it:
 
-- **Demonstrated** runs from the axis minimum up to the lower confidence bound of the best tested model. The band
-  reads "Requirements through *x*% are demonstrated".
+- **Demonstrated** runs from the axis minimum up to the lower confidence bound of the best tested model, which the
+  label under the band names, for example "95.7% lower confidence bound". The zone reads "demonstrated".
 - **Unknown** lies between the other two zones. The evidence neither demonstrates nor rules out these requirements.
-- **Ruled out** runs from the upper confidence bound of the lowest applicable ceiling to the top of the axis. It holds
-  only under the recorded assumptions, and the zone reads "ruled out under the recorded assumptions" wherever the
-  label fits.
+- **Ruled out** runs from the upper confidence bound of the lowest applicable ceiling to the top of the axis. The zone
+  reads "ruled out\*" wherever the label fits.
+
+The `*` marks a footnote: "ruled out" holds only under the recorded assumptions. Wherever "ruled out\*" appears, the
+caption under the same band ends with that footnote, "\* Ruled out under the recorded assumptions, at the stated
+confidence."
 
 Each of your requirements is marked on the axis, and the marker states the determination in words:
 
 | Marker | Meaning |
 |--------|---------|
-| "95% requirement: demonstrated ✓" | The requirement is at or below the lower confidence bound |
-| "not enough evidence ?" | The requirement falls in the unknown zone, and the evidence cannot decide it |
-| "ruled out under the recorded assumptions ✗" | The requirement is above the upper confidence bound of a ceiling |
-| "not ruled out ?" | The requirement also falls in the unknown zone: it is not above any ceiling's upper confidence bound, and no tested model reaches it |
+| "95%: demonstrated ✓" | The requirement is at or below the lower confidence bound |
+| "95%: insufficient evidence ?" | The requirement falls in the unknown zone, and the evidence cannot decide it |
+| "95%: ruled out\* ✗" | The requirement is above the upper confidence bound of a ceiling, under the recorded assumptions (see the footnote) |
+| "95%: not ruled out ?" | The requirement also falls in the unknown zone: it is not above any ceiling's upper confidence bound, and no tested model reaches it |
 
 The caption under the band reports the observed score next to the lower confidence bound it supports, for example
-"Your model scored 91.0% on the 2,000 evaluation images. Allowing for sampling uncertainty gives a lower
-confidence bound of 88.5%." The sentence about the observed score appears only when an observed score exists. The
-caption then states that the bounds concern new images drawn from the same population, and that confidence holds
-jointly across the whole evaluation (for example "Confidence is 95% jointly across this evaluation").
+"Your model scored 91.0% on 2,000 evaluation images; given sampling uncertainty, its lower confidence bound is
+88.5%." The sentence about the observed score appears only when an observed score exists. The caption then states
+that the bounds apply to new images from the same population, and that confidence holds jointly across the whole
+evaluation (for example "Bounds apply to new images from the same population, at 95% joint confidence.").
 
 Numbers read as DataEval prints them. Bounds are given to 0.1%, a lower bound rounded down and an upper bound rounded
 up, so rounding never overstates what the evidence shows. Requirements and confidence keep their own decimals, such as
-"95.5% requirement" or "Confidence is 97.5%".
+"95.5%" or "at 97.5% joint confidence".
+
+When a ceiling bounds the band, the caption ends with the footnote "\* Ruled out under the recorded assumptions, at
+the stated confidence."
 
-When a ceiling bounds the band, the caption adds "Under the recorded assumptions, requirements above *x*% are ruled out
-at the stated confidence." A group-based ceiling carries the extra note that its validity argument awaits lead-scientist
-sign-off.
+A recall requirement is titled "Required recall for Tank (at most 5% false positives)". Only a witness whose own upper
+confidence bound on false positives is within that limit counts; the caption adds its bound after the observed
+sentence, for example "Its upper confidence bound on false positives is 4.1% (observed 2.6% of 651 other images)." In
+the evidence view, a witness over the limit keeps its recall bound and observed score but is drawn faint, its label
+saying why it is not counted, for example "66.0% lower bound · not counted: false positives ≤8.3% (limit 5%)".
 
 A ceiling that applies only to narrower models than the band judges never draws a zone. It appears as a note instead,
-for example that the group-based ceiling does not limit models that distinguish individual images.
+for example "Group-based ceiling (72.1%) applies only to models that give one answer per group."
 
 ### Other outcomes
 
-- **Ruled out.** A requirement above the lowest applicable ceiling's upper confidence bound is marked "ruled out
-  under the recorded assumptions ✗". The ruled-out zone covers everything above that bound, and the caption reads
-  "Under the recorded assumptions, requirements above *x*% are ruled out at the stated confidence."
+- **Ruled out.** A requirement above the lowest applicable ceiling's upper confidence bound is marked "ruled out\*
+  ✗". The ruled-out zone covers everything above that bound, which the label under the band names, for example
+  "93.0% upper confidence bound". The caption ends with the footnote "\* Ruled out under the recorded assumptions, at
+  the stated confidence."
 - **Evidence conflict.** When bounds from different sources contradict each other, the band draws no zones. A dashed
   warning outline reads "Evidence conflicts: check assumptions", both clashing bounds are marked, and the caption
-  says that the bounds disagree and to check the assumptions, including whether evaluation images entered training.
-  Even when the assumptions hold, the probability that any bound fails is at most the stated alpha (for example 5% at
-  95% confidence).
+  reads "Bounds disagree: check the assumptions, such as evaluation images entering training. Even when they hold,
+  the chance that any bound fails is at most 5%." The 5% is the stated alpha, at 95% confidence.
 - **Bounds at 0% or 100%.** A lower confidence bound of 0% demonstrates nothing and an upper confidence bound of 100%
   rules nothing out, so neither draws a zone.
 
 ### Showing the evidence
 
 `plot_evidence=True` replaces each band with one row per evidence source. Each row shows the observed score as a dot
-and the confidence bound as a tick, with a legend line explaining both: the dot is the score observed on the
-evaluation images, and the tick is the lower (or upper) confidence bound for new images from the same population.
-Evidence that does not apply to the judged models is drawn faint and labeled with the models it applies to. A source
-that produced no bound for the quantity is listed faint as "no bound available", and when no source has a bound the
-view says "No bound is available yet." instead of showing empty axes.
+and the confidence bound as a tick, with the legend "● observed · ┃ lower confidence bound" (and "· ┃ upper confidence
+bound" when a ceiling has one). The caption says what both mean: "Observed on the evaluation images; bounds apply to
+new images from the same population, at 95% joint confidence." In a conflict, the conflict caption follows it.
+Evidence that does not apply to the judged models is drawn faint, and its label names the models it applies to, for
+example "Group-based ceiling (one answer per group)", so this view has no narrower-scope notes. A source that produced
+no bound for the quantity is listed faint as "no bound available", and when no source has a bound the view says "No
+bound is available yet." instead of showing empty axes.
 
 ## Extending the Package
 

{dataeval_plots-0.0.13 → dataeval_plots-0.0.15}/README.md

@@ -162,60 +162,70 @@ subtitle that says which models the band judges:
 
 The axis is split into three zones. Each zone classifies the requirements that fall inside it:
 
-- **Demonstrated** runs from the axis minimum up to the lower confidence bound of the best tested model. The band
-  reads "Requirements through *x*% are demonstrated".
+- **Demonstrated** runs from the axis minimum up to the lower confidence bound of the best tested model, which the
+  label under the band names, for example "95.7% lower confidence bound". The zone reads "demonstrated".
 - **Unknown** lies between the other two zones. The evidence neither demonstrates nor rules out these requirements.
-- **Ruled out** runs from the upper confidence bound of the lowest applicable ceiling to the top of the axis. It holds
-  only under the recorded assumptions, and the zone reads "ruled out under the recorded assumptions" wherever the
-  label fits.
+- **Ruled out** runs from the upper confidence bound of the lowest applicable ceiling to the top of the axis. The zone
+  reads "ruled out\*" wherever the label fits.
+
+The `*` marks a footnote: "ruled out" holds only under the recorded assumptions. Wherever "ruled out\*" appears, the
+caption under the same band ends with that footnote, "\* Ruled out under the recorded assumptions, at the stated
+confidence."
 
 Each of your requirements is marked on the axis, and the marker states the determination in words:
 
 | Marker | Meaning |
 |--------|---------|
-| "95% requirement: demonstrated ✓" | The requirement is at or below the lower confidence bound |
-| "not enough evidence ?" | The requirement falls in the unknown zone, and the evidence cannot decide it |
-| "ruled out under the recorded assumptions ✗" | The requirement is above the upper confidence bound of a ceiling |
-| "not ruled out ?" | The requirement also falls in the unknown zone: it is not above any ceiling's upper confidence bound, and no tested model reaches it |
+| "95%: demonstrated ✓" | The requirement is at or below the lower confidence bound |
+| "95%: insufficient evidence ?" | The requirement falls in the unknown zone, and the evidence cannot decide it |
+| "95%: ruled out\* ✗" | The requirement is above the upper confidence bound of a ceiling, under the recorded assumptions (see the footnote) |
+| "95%: not ruled out ?" | The requirement also falls in the unknown zone: it is not above any ceiling's upper confidence bound, and no tested model reaches it |
 
 The caption under the band reports the observed score next to the lower confidence bound it supports, for example
-"Your model scored 91.0% on the 2,000 evaluation images. Allowing for sampling uncertainty gives a lower
-confidence bound of 88.5%." The sentence about the observed score appears only when an observed score exists. The
-caption then states that the bounds concern new images drawn from the same population, and that confidence holds
-jointly across the whole evaluation (for example "Confidence is 95% jointly across this evaluation").
+"Your model scored 91.0% on 2,000 evaluation images; given sampling uncertainty, its lower confidence bound is
+88.5%." The sentence about the observed score appears only when an observed score exists. The caption then states
+that the bounds apply to new images from the same population, and that confidence holds jointly across the whole
+evaluation (for example "Bounds apply to new images from the same population, at 95% joint confidence.").
 
 Numbers read as DataEval prints them. Bounds are given to 0.1%, a lower bound rounded down and an upper bound rounded
 up, so rounding never overstates what the evidence shows. Requirements and confidence keep their own decimals, such as
-"95.5% requirement" or "Confidence is 97.5%".
+"95.5%" or "at 97.5% joint confidence".
+
+When a ceiling bounds the band, the caption ends with the footnote "\* Ruled out under the recorded assumptions, at
+the stated confidence."
 
-When a ceiling bounds the band, the caption adds "Under the recorded assumptions, requirements above *x*% are ruled out
-at the stated confidence." A group-based ceiling carries the extra note that its validity argument awaits lead-scientist
-sign-off.
+A recall requirement is titled "Required recall for Tank (at most 5% false positives)". Only a witness whose own upper
+confidence bound on false positives is within that limit counts; the caption adds its bound after the observed
+sentence, for example "Its upper confidence bound on false positives is 4.1% (observed 2.6% of 651 other images)." In
+the evidence view, a witness over the limit keeps its recall bound and observed score but is drawn faint, its label
+saying why it is not counted, for example "66.0% lower bound · not counted: false positives ≤8.3% (limit 5%)".
 
 A ceiling that applies only to narrower models than the band judges never draws a zone. It appears as a note instead,
-for example that the group-based ceiling does not limit models that distinguish individual images.
+for example "Group-based ceiling (72.1%) applies only to models that give one answer per group."
 
 ### Other outcomes
 
-- **Ruled out.** A requirement above the lowest applicable ceiling's upper confidence bound is marked "ruled out
-  under the recorded assumptions ✗". The ruled-out zone covers everything above that bound, and the caption reads
-  "Under the recorded assumptions, requirements above *x*% are ruled out at the stated confidence."
+- **Ruled out.** A requirement above the lowest applicable ceiling's upper confidence bound is marked "ruled out\*
+  ✗". The ruled-out zone covers everything above that bound, which the label under the band names, for example
+  "93.0% upper confidence bound". The caption ends with the footnote "\* Ruled out under the recorded assumptions, at
+  the stated confidence."
 - **Evidence conflict.** When bounds from different sources contradict each other, the band draws no zones. A dashed
   warning outline reads "Evidence conflicts: check assumptions", both clashing bounds are marked, and the caption
-  says that the bounds disagree and to check the assumptions, including whether evaluation images entered training.
-  Even when the assumptions hold, the probability that any bound fails is at most the stated alpha (for example 5% at
-  95% confidence).
+  reads "Bounds disagree: check the assumptions, such as evaluation images entering training. Even when they hold,
+  the chance that any bound fails is at most 5%." The 5% is the stated alpha, at 95% confidence.
 - **Bounds at 0% or 100%.** A lower confidence bound of 0% demonstrates nothing and an upper confidence bound of 100%
   rules nothing out, so neither draws a zone.
 
 ### Showing the evidence
 
 `plot_evidence=True` replaces each band with one row per evidence source. Each row shows the observed score as a dot
-and the confidence bound as a tick, with a legend line explaining both: the dot is the score observed on the
-evaluation images, and the tick is the lower (or upper) confidence bound for new images from the same population.
-Evidence that does not apply to the judged models is drawn faint and labeled with the models it applies to. A source
-that produced no bound for the quantity is listed faint as "no bound available", and when no source has a bound the
-view says "No bound is available yet." instead of showing empty axes.
+and the confidence bound as a tick, with the legend "● observed · ┃ lower confidence bound" (and "· ┃ upper confidence
+bound" when a ceiling has one). The caption says what both mean: "Observed on the evaluation images; bounds apply to
+new images from the same population, at 95% joint confidence." In a conflict, the conflict caption follows it.
+Evidence that does not apply to the judged models is drawn faint, and its label names the models it applies to, for
+example "Group-based ceiling (one answer per group)", so this view has no narrower-scope notes. A source that produced
+no bound for the quantity is listed faint as "no bound available", and when no source has a bound the view says "No
+bound is available yet." instead of showing empty axes.
 
 ## Extending the Package
 

{dataeval_plots-0.0.13 → dataeval_plots-0.0.15}/src/dataeval_plots/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.0.13'
-__version_tuple__ = version_tuple = (0, 0, 13)
+__version__ = version = '0.0.15'
+__version_tuple__ = version_tuple = (0, 0, 15)
 
 __commit_id__ = commit_id = None

{dataeval_plots-0.0.13 → dataeval_plots-0.0.15}/src/dataeval_plots/backends/_feasibility_layout.py

@@ -48,7 +48,8 @@ def text_width(text: str, size: float = LABEL_PT) -> float:
 
 def wrap(text: str, size: float, width: float) -> str:
     """Wrap to the figure's text width, never leaving a legend glyph at a line end."""
-    glued = text.replace("● ", "●\xa0").replace("┃ ", "┃\xa0")  # textwrap breaks only at ASCII whitespace
+    # textwrap breaks only at ASCII whitespace; a footnote's "*" stays with its first word
+    glued = text.replace("● ", "●\xa0").replace("┃ ", "┃\xa0").replace("* ", "*\xa0")
     return textwrap.fill(glued, int((width - 2 * MARGIN) * 72 / (size * 0.55))).replace("\xa0", " ")
 
 
@@ -114,7 +115,8 @@ def blocks(bands: list[FeasibilityBand], plot_evidence: bool, width: float) -> t
         ax_h = body + (max((p[3] for p in placed), default=-1) + 1) * LEVEL + 0.1
         # Below the axes: the evidence legend, the caption paragraph, then the notes, each wrapped
         texts = [(band.evidence_legend, 10.0, muted)] if plot_evidence else []
-        texts += [(" ".join(band.caption), CAPTION_PT, ink), *((note, 10.0, muted) for note in band.notes)]
+        caption, notes = (band.evidence_caption, []) if plot_evidence else (band.caption, band.notes)
+        texts += [(" ".join(caption), CAPTION_PT, ink), *((note, 10.0, muted) for note in notes)]
         wrapped = [(wrap(text, size, width), size, color) for text, size, color in texts]
         out.append((band, x_range, xlabel, ax_h, placed, wrapped))
         height += TITLE + ax_h + XAXIS + sum(text_height(text) for text, _, _ in wrapped) + GAP

{dataeval_plots-0.0.13 → dataeval_plots-0.0.15}/src/dataeval_plots/backends/_shared.py

@@ -684,16 +684,20 @@ _SCOPE_MODELS = {
     "fixed_representation": "models restricted to the embeddings",
     "partition_restricted": "models that give one answer per group",
 }
+_SCOPE_ROW = {"fixed_representation": "embeddings only", "partition_restricted": "one answer per group"}
+# "ruled out*" never stands alone: its "*" footnote in the same band's caption says it holds only under the recorded
+# assumptions, and an infeasible row always has a ceiling, which writes that footnote
 _MARKER_WORDS = {
     "demonstrated_feasible": "demonstrated ✓",
-    "unresolved": "not enough evidence ?",
-    "infeasible_under_recorded_assumptions": "ruled out under the recorded assumptions ✗",
+    "unresolved": "insufficient evidence ?",
+    "infeasible_under_recorded_assumptions": "ruled out* ✗",
     "not_ruled_out": "not ruled out ?",
 }
-_RULED_OUT = "ruled out under the recorded assumptions"  # never "ruled out" alone: it holds only under them
+_RULED_OUT = "ruled out*"
 _AGGREGATION_WORDS = {None: "", "sequence_mean": "per-sequence mean ", "frame_pooled": "frame-pooled "}
 _GROUP_CEILING = "group-based ceiling"
 _NO_BOUND = "no bound available"
+_FALSE_POSITIVE = "false_positive_rate"
 
 
 def _pct(value: float, direction: str | None = None) -> str:
@@ -709,6 +713,11 @@ def _pct(value: float, direction: str | None = None) -> str:
     return f"{tenths / 10:.1f}%"
 
 
+def _confidence(value: float, direction: str) -> str:
+    """Round a confidence down or an error share up to 0.1%, dropping a trailing ".0", as the printout does."""
+    return _pct(value, direction).replace(".0%", "%")
+
+
 def _percent(value: float) -> str:
     """A threshold, confidence or alpha as DataEval prints it: no rounding to whole percents, no trailing zeros."""
     return f"{value * 100:g}%"
@@ -750,6 +759,7 @@ class FeasibilityBand:
     evidence_markers: list[tuple[float, str]]  # requirement labels in the evidence view
     caption: list[str]
     notes: list[str]
+    evidence_caption: list[str]  # the evidence view's caption, in place of the caption and notes
     evidence: list[FeasibilityEvidenceRow]
     evidence_legend: str
     evidence_note: str | None  # drawn in the evidence view when no source has a bound for the band
@@ -765,6 +775,16 @@ def _feasibility_range(values: Iterable[float]) -> tuple[float, float]:
     return round(max(low, 0.0), 2), 1.0
 
 
+def _not_counted(e: dict[str, Any]) -> str:
+    """The label suffix of a recall witness that does not count: its false-positive bound is over the limit or none."""
+    if not e["uncertified"]:
+        return ""
+    fp = e["false_positive_bound"]
+    seen = "no false-positive bound" if fp is None else f"false positives ≤{_pct(fp, 'upper')}"
+    limit = "" if fp is None else f" (limit {_percent(e['false_positive_limit'])})"
+    return f" · not counted: {seen}{limit}"
+
+
 def _feasibility_evidence(
     evidence: list[dict[str, Any]], scope: str, unit: str
 ) -> tuple[list[FeasibilityEvidenceRow], list[str]]:
@@ -788,7 +808,7 @@ def _feasibility_evidence(
         rank, judged = _SCOPE_RANK[info], _SCOPE_RANK[scope]
         restricted = rank < judged if kind == "upper" else rank > judged
         models = _SCOPE_MODELS[info].format(unit=unit)
-        label = _sentence_case(e["source"]) + (f" ({models})" if restricted else "")
+        label = _sentence_case(e["source"]) + (f" ({_SCOPE_ROW.get(info, models)})" if restricted else "")
         observed = None if bound is None else e["observed"]  # an observed score alone bounds nothing
         rows.append(
             FeasibilityEvidenceRow(
@@ -798,19 +818,18 @@ def _feasibility_evidence(
                 observed=observed,
                 restricted=restricted,
                 label=label,
-                bound_label=_NO_BOUND if bound is None else f"{_pct(bound, kind)} {kind} bound",
+                bound_label=_NO_BOUND if bound is None else f"{_pct(bound, kind)} {kind} bound" + _not_counted(e),
                 observed_label=None if observed is None else f"{_pct(observed)} observed",
                 bound_left=kind == "lower",
-                faint=restricted or bound is None,
+                faint=restricted or bound is None or e["uncertified"],
             )
         )
         if restricted and kind == "upper" and bound is not None:
             ceiling = _pct(bound, "upper")
             notes.append(
-                f"The group-based ceiling ({ceiling}) does not limit models that distinguish individual {unit}s."
+                f"Group-based ceiling ({ceiling}) applies only to models that give one answer per group."
                 if info == "partition_restricted"
-                else f"The {e['source']} ({ceiling}) applies only to {models}; "
-                "it does not limit the models judged here."
+                else f"{_sentence_case(e['source'])} ({ceiling}) applies only to {models}."
             )
     return rows, list(dict.fromkeys(notes))
 
@@ -846,18 +865,29 @@ def _unit(rows: list[dict[str, Any]], evidence: list[dict[str, Any]]) -> str | N
     return None
 
 
+def _other(fp: dict[str, Any]) -> str:
+    """The audit negatives a false-positive bound was measured on, such as ``651 other images``."""
+    return ("" if fp["n"] is None else f"{fp['n']:,} ") + f"other {fp['unit']}s"
+
+
 def _feasibility_title(spec: dict[str, Any]) -> str:
     """The axis title, naming the AP class (or macro) and any aggregation so per-class bands differ."""
     metric = spec["metric"]
     if metric == "ap":
         metric = "macro AP" if spec.get("class_name") is None else f"AP for class {spec['class_name']}"
+    elif metric == "recall":
+        f0 = _percent(spec["max_false_positive_rate"])
+        metric = f"recall for {spec['class_name']} (at most {f0} false positives)"
     return "Required " + _AGGREGATION_WORDS[spec.get("aggregation")] + metric
 
 
 def _feasibility_band(
-    rows: list[dict[str, Any]], evidence: list[dict[str, Any]], alpha: float, noun: str
+    rows: list[dict[str, Any]], all_evidence: list[dict[str, Any]], alpha: float, noun: str
 ) -> FeasibilityBand:
     """One band; ``noun`` is the unit the printout's footer names for the whole call."""
+    # Bands and evidence rows draw the metric; a recall witness's false-positive bound only joins the caption
+    evidence = [e for e in all_evidence if e["quantity"] == "metric"]
+    false_positives = [e for e in all_evidence if e["quantity"] == _FALSE_POSITIVE]
     scope = rows[0]["judged_scope"]
     unit = _unit(rows, evidence) or "image"
     # A lower bound of 0% demonstrates nothing and an upper bound of 100% rules nothing out
@@ -880,37 +910,56 @@ def _feasibility_band(
     markers = []
     for r in sorted(rows, key=lambda r: r["threshold"]):
         words = _MARKER_WORDS.get(r["determination"])
-        markers.append((r["threshold"], f"{_percent(r['threshold'])} requirement" + (f": {words}" if words else "")))
+        markers.append((r["threshold"], _percent(r["threshold"]) + (f": {words}" if words else "")))
 
+    conflict_caption = (
+        f"Bounds disagree: check the assumptions, such as evaluation {unit}s entering training. "
+        f"Even when they hold, the chance that any bound fails is at most {_confidence(alpha, 'upper')}."
+    )
     if conflict:
-        caption = [
-            f"The bounds disagree. Check the assumptions, including whether evaluation {unit}s entered training. "
-            f"Even when assumptions hold, the probability that any bound fails is at most {_percent(alpha)}."
-        ]
+        caption = [conflict_caption]
     else:
         caption = []
         witness = None if best is None else _deciding(best, evidence)[0]
         if witness and witness["observed"] is not None and w is not None:
-            on = "" if witness["n"] is None else f"{witness['n']:,} "
+            on = "the " if witness["n"] is None else f"{witness['n']:,} "
+            # A recall witness's count is the class's positives, which the printout names by class
+            spec = json.loads(rows[0]["target_spec"])
+            scored = f"{_pct(witness['observed'])} on {on}evaluation {unit}s"
+            if spec["metric"] == "recall":
+                scored = f"{_pct(witness['observed'])} recall on {on}{spec['class_name']} {unit}s"
             caption.append(
-                f"{_sentence_case(witness['source'])} scored {_pct(witness['observed'])} on the {on}evaluation "
-                f"{unit}s. Allowing for sampling uncertainty gives a lower confidence bound of {_pct(w, 'lower')}."
+                f"{_sentence_case(witness['source'])} scored {scored}; given sampling uncertainty, its lower "
+                f"confidence bound is {_pct(w, 'lower')}."
             )
-        if deciding is not None and c is not None:
-            ceiling = _deciding(deciding, evidence)[1]
-            sign_off = ceiling is not None and ceiling["source"] == _GROUP_CEILING
-            caption.append(
-                f"Under the recorded assumptions, requirements above {_pct(c, 'upper')} are ruled out at the stated "
-                "confidence"
-                + (" (group-based ceiling; its validity argument awaits lead-scientist sign-off)" if sign_off else "")
-                + "."
+        fp = None
+        if witness and w is not None:
+            fp = next(
+                (
+                    e
+                    for e in false_positives
+                    if e["evidence_id"] == witness["evidence_id"]
+                    and e["target_spec"] == witness["target_spec"]
+                    and e["bound"] is not None
+                ),
+                None,
             )
-        caption.append(
-            f"Bounds concern new {noun}s drawn from the same population. "
-            f"Confidence is {_percent(1 - alpha)} jointly across this evaluation."
-        )
+        if fp is not None:
+            seen = "" if fp["observed"] is None else f" (observed {_pct(fp['observed'])} of {_other(fp)})"
+            caption.append(f"Its upper confidence bound on false positives is {_pct(fp['bound'], 'upper')}{seen}.")
+        joint = _confidence(1 - alpha, "lower")
+        caption.append(f"Bounds apply to new {noun}s from the same population, at {joint} joint confidence.")
+        # The "*" footnote to every "ruled out*" goes last, as a footnote does
+        if deciding is not None and c is not None:
+            caption.append("* Ruled out under the recorded assumptions, at the stated confidence.")
+    evidence_caption = [
+        f"Observed on the evaluation {unit}s; bounds apply to new {unit}s from the same population, "
+        f"at {_confidence(1 - alpha, 'lower')} joint confidence."
+    ]
+    if conflict:
+        evidence_caption.append(conflict_caption)
 
-    legend = f"● observed on the evaluation {unit}s · ┃ lower confidence bound for new {unit}s from the same population"
+    legend = "● observed · ┃ lower confidence bound"
     if any(r.bound_type == "upper" and r.bound is not None for r in ev_rows):
         legend += " · ┃ upper confidence bound"
 
@@ -926,6 +975,7 @@ def _feasibility_band(
         evidence_markers=[(t, f"{_percent(t)} required") for t, _ in markers],
         caption=caption,
         notes=notes,
+        evidence_caption=evidence_caption,
         evidence=ev_rows,
         evidence_legend=legend,
         evidence_note=None if any(r.bound is not None for r in ev_rows) else "No bound is available yet.",
@@ -952,6 +1002,10 @@ _EVIDENCE_SCHEMA = {
     "bound": pl.Float64,
     "n": pl.Int64,
     "state": pl.String,
+    "quantity": pl.String,
+    "uncertified": pl.Boolean,
+    "false_positive_bound": pl.Float64,
+    "false_positive_limit": pl.Float64,
 }
 
 
@@ -961,10 +1015,14 @@ def _evidence_rows(output: PlottableFeasibility) -> pl.DataFrame:
     rows = []
     for spec, target in targets.items():
         for eid, ev in output.evidence.items():
-            for e in ev.endpoints:
-                if e.target_spec != spec:
-                    continue
-                stats = ev.details["endpoints"][spec] or {}
+            # An older DataEval has no quantity; its endpoints all bound the metric
+            mine = [(e, getattr(e, "quantity", "metric")) for e in ev.endpoints if e.target_spec == spec]
+            fp = next((e.value for e, q in mine if q == _FALSE_POSITIVE), None)
+            f0 = json.loads(spec).get("max_false_positive_rate")
+            for e, q in mine:
+                stats = ev.details["endpoints"].get(spec if q == "metric" else f"{spec}#{q}") or {}
+                # A recall witness counts only when its own false-positive upper bound is within the limit
+                uncertified = q == "metric" and e.direction == "lower" and f0 is not None and (fp is None or fp > f0)
                 rows.append(
                     {
                         "target_spec": spec,
@@ -978,10 +1036,14 @@ def _evidence_rows(output: PlottableFeasibility) -> pl.DataFrame:
                         "unit": "sequence" if ev.unit == "group" else "image",
                         "bound_type": e.direction,
                         # A ceiling's mean is panel agreement, not an observed score
-                        "observed": stats.get("mean") if e.direction == "lower" else None,
+                        "observed": stats.get("mean") if e.direction == "lower" or q == _FALSE_POSITIVE else None,
                         "bound": e.value,
                         "n": stats.get("n"),
                         "state": e.state,
+                        "quantity": q,
+                        "uncertified": uncertified,
+                        "false_positive_bound": fp,
+                        "false_positive_limit": f0,
                     }
                 )
     return pl.DataFrame(rows, schema=_EVIDENCE_SCHEMA)
@@ -1012,7 +1074,7 @@ def prepare_feasibility_data(output: PlottableFeasibility) -> list[FeasibilityBa
         groups.setdefault(json.dumps(spec, sort_keys=True), []).append(row)
     evidence = _evidence_rows(output).to_dicts()
     alpha = output.meta().state["alpha"]
-    noun = _unit(rows, evidence) or "image"  # the printout's footer noun, for the whole call
+    noun = _unit(rows, [e for e in evidence if e["quantity"] == "metric"]) or "image"  # the printout's footer noun
     bands = []
     for band in groups.values():
         specs = {r["target_spec"] for r in band}
@@ -1042,7 +1104,7 @@ def feasibility_zones(band: FeasibilityBand) -> list[tuple[str, float, float, st
     end = hi if band.ruled_out is None else band.ruled_out
     zones = [("unknown", start, end, "unknown")]
     if band.demonstrated is not None:
-        zones.insert(0, ("demonstrated", lo, start, f"Requirements through {_pct(start, 'lower')} are demonstrated"))
+        zones.insert(0, ("demonstrated", lo, start, "demonstrated"))
     if band.ruled_out is not None:
         zones.append(("ruled_out", end, hi, _RULED_OUT))
     return [zone for zone in zones if zone[2] > zone[1]]  # a zero-width zone has nothing to show

{dataeval_plots-0.0.13 → dataeval_plots-0.0.15}/src/dataeval_plots/protocols.py

@@ -290,7 +290,12 @@ class PlottableDriftMVDC(Plottable, Protocol):
 
 
 class FeasibilityEndpoint(Protocol):
-    """One bound on one target, as DataEval's ``Endpoint`` records it; only the fields the plot reads."""
+    """
+    One bound on one target, as DataEval's ``Endpoint`` records it; only the fields the plot reads.
+
+    Newer outputs also give ``quantity``, ``"metric"`` or ``"false_positive_rate"``; the plot reads it with a default
+    of ``"metric"`` so older outputs still plot.
+    """
 
     @property
     def target_spec(self) -> str: ...