rolloutdiff 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. rolloutdiff/__init__.py +130 -0
  2. rolloutdiff/_aria_replay.py +346 -0
  3. rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
  4. rolloutdiff/_dependence.py +195 -0
  5. rolloutdiff/_feature_values.py +79 -0
  6. rolloutdiff/_replay_container.py +568 -0
  7. rolloutdiff/_report_validation.py +758 -0
  8. rolloutdiff/_retrospective_policy.py +33 -0
  9. rolloutdiff/_sampling.py +127 -0
  10. rolloutdiff/_series_validation.py +67 -0
  11. rolloutdiff/_watch_report.py +436 -0
  12. rolloutdiff/_watch_storage.py +800 -0
  13. rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
  14. rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
  15. rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
  16. rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
  17. rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
  18. rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
  19. rolloutdiff/_web/assets/third-party-notices.txt +112 -0
  20. rolloutdiff/_web/index.html +14 -0
  21. rolloutdiff/adapters.py +264 -0
  22. rolloutdiff/app.py +956 -0
  23. rolloutdiff/app_evidence.py +678 -0
  24. rolloutdiff/benchmark_reference.py +458 -0
  25. rolloutdiff/cli.py +1292 -0
  26. rolloutdiff/comparison.py +572 -0
  27. rolloutdiff/comparison_report.py +982 -0
  28. rolloutdiff/corroboration.py +287 -0
  29. rolloutdiff/detect.py +307 -0
  30. rolloutdiff/events.py +173 -0
  31. rolloutdiff/evidence.py +285 -0
  32. rolloutdiff/exemplars.py +240 -0
  33. rolloutdiff/features.py +823 -0
  34. rolloutdiff/html_report.py +726 -0
  35. rolloutdiff/inspect_logs.py +383 -0
  36. rolloutdiff/inspection.py +207 -0
  37. rolloutdiff/integrations.py +552 -0
  38. rolloutdiff/investigation.py +1146 -0
  39. rolloutdiff/notifications.py +406 -0
  40. rolloutdiff/pipeline.py +809 -0
  41. rolloutdiff/producer.py +663 -0
  42. rolloutdiff/prompt_mix.py +258 -0
  43. rolloutdiff/ranking.py +388 -0
  44. rolloutdiff/replay.py +589 -0
  45. rolloutdiff/replication.py +572 -0
  46. rolloutdiff/report.py +838 -0
  47. rolloutdiff/retrospective.py +386 -0
  48. rolloutdiff/schema.py +109 -0
  49. rolloutdiff/sensitivity.py +202 -0
  50. rolloutdiff/series.py +430 -0
  51. rolloutdiff/tool_features.py +422 -0
  52. rolloutdiff/trajectories.py +236 -0
  53. rolloutdiff/watch-compatibility.json +13 -0
  54. rolloutdiff/watch.py +742 -0
  55. rolloutdiff-0.1.0.dist-info/METADATA +158 -0
  56. rolloutdiff-0.1.0.dist-info/RECORD +62 -0
  57. rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
  58. rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
  59. rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
  60. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
  61. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
  62. rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,258 @@
1
+ """Prompt-mix evidence for caller-selected windows of one run."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ from collections.abc import Mapping, Sequence
7
+ from dataclasses import dataclass
8
+
9
+ import numpy as np
10
+ import pandas as pd
11
+
12
+ from ._feature_values import feature_values
13
+ from ._sampling import (
14
+ cluster_standard_error,
15
+ resolve_sampling_units,
16
+ subset_sampling_units,
17
+ )
18
+ from .features import FeatureTables
19
+
20
+
21
+ @dataclass(frozen=True)
22
+ class PromptMixEvidence:
23
+ run: str
24
+ feature: str
25
+ kind: str
26
+ before: tuple[int, int]
27
+ after: tuple[int, int]
28
+ positive_prompts: int | None
29
+ total_prompts: int
30
+ shared_prompts: int
31
+ shared_sampling_units: int
32
+ shared_change: float | None
33
+ shared_se: float | None
34
+ per_prompt: pd.DataFrame
35
+ prompt_shift: pd.DataFrame
36
+
37
+
38
+ def _window(value: tuple[int, int]) -> tuple[int, int]:
39
+ if (
40
+ not isinstance(value, tuple)
41
+ or len(value) != 2
42
+ or any(type(bound) is not int or bound < 0 for bound in value)
43
+ or value[0] > value[1]
44
+ ):
45
+ raise ValueError(
46
+ "window must be an inclusive (start, end) pair of nonnegative integers"
47
+ )
48
+ return value
49
+
50
+
51
+ def _keywords(values: Sequence[str]) -> list[str]:
52
+ if isinstance(values, str) or not isinstance(values, Sequence):
53
+ raise ValueError("keywords must be a sequence of literal strings")
54
+ result = []
55
+ for value in values:
56
+ if not isinstance(value, str) or not value.strip():
57
+ raise ValueError("keyword must be a nonempty string")
58
+ word = value.strip().casefold()
59
+ if word in result:
60
+ raise ValueError("duplicate keyword after case normalization")
61
+ result.append(word)
62
+ return result
63
+
64
+
65
+ def _ks(before: np.ndarray, after: np.ndarray) -> float:
66
+ points = np.union1d(before, after)
67
+ a = np.searchsorted(np.sort(before), points, side="right") / len(before)
68
+ b = np.searchsorted(np.sort(after), points, side="right") / len(after)
69
+ return float(np.max(np.abs(a - b)))
70
+
71
+
72
+ def _prompt_shift(
73
+ detail: pd.DataFrame, keywords: list[str], has_difficulty: bool
74
+ ) -> pd.DataFrame:
75
+ before = detail.loc[detail.before_n.gt(0)]
76
+ after = detail.loc[detail.after_n.gt(0)]
77
+ rows = []
78
+ for descriptor in ["prompt_chars", *(f"keyword:{word}" for word in keywords)]:
79
+ a, b = (
80
+ before[descriptor].to_numpy(dtype=float),
81
+ after[descriptor].to_numpy(dtype=float),
82
+ )
83
+ delta = float(b.mean() - a.mean())
84
+ rows.append(
85
+ {
86
+ "descriptor": descriptor,
87
+ "statistic": "ks"
88
+ if descriptor == "prompt_chars"
89
+ else "absolute_prevalence",
90
+ "before": float(a.mean()),
91
+ "after": float(b.mean()),
92
+ "delta": delta,
93
+ "distance": _ks(a, b) if descriptor == "prompt_chars" else abs(delta),
94
+ "n_before": len(a),
95
+ "n_after": len(b),
96
+ "missing_before": 0,
97
+ "missing_after": 0,
98
+ }
99
+ )
100
+ if has_difficulty:
101
+ a, b = before.difficulty.dropna(), after.difficulty.dropna()
102
+ distance = float("nan")
103
+ if len(a) and len(b):
104
+ pa, pb = a.value_counts(normalize=True), b.value_counts(normalize=True)
105
+ distance = float(pa.sub(pb, fill_value=0).abs().sum() / 2)
106
+ rows.append(
107
+ {
108
+ "descriptor": "difficulty",
109
+ "statistic": "total_variation",
110
+ "before": float("nan"),
111
+ "after": float("nan"),
112
+ "delta": float("nan"),
113
+ "distance": distance,
114
+ "n_before": len(a),
115
+ "n_after": len(b),
116
+ "missing_before": len(before) - len(a),
117
+ "missing_after": len(after) - len(b),
118
+ }
119
+ )
120
+ return pd.DataFrame(rows)
121
+
122
+
123
+ def prompt_mix(
124
+ rollouts: pd.DataFrame,
125
+ features: FeatureTables,
126
+ *,
127
+ run: str,
128
+ feature: str,
129
+ before: tuple[int, int],
130
+ after: tuple[int, int],
131
+ keywords: Sequence[str] = (),
132
+ difficulty: Mapping[str, str] | None = None,
133
+ sampling_units: pd.DataFrame | None = None,
134
+ ) -> PromptMixEvidence:
135
+ """Measure breadth, paired prompt change, and prompt descriptor shift.
136
+
137
+ Windows are inclusive, ordered, disjoint, and nonempty. Pool completions per
138
+ prompt/window, then weight shared prompts equally. Default SE is paired-change
139
+ SD(ddof=1) / sqrt(shared prompts); explicit units cluster those residuals.
140
+ Fewer than two shared units makes SE unavailable; no shared prompts also
141
+ makes change unavailable.
142
+
143
+ Binary breadth counts after-window prompts with any positive value; continuous
144
+ breadth is unavailable. Descriptors count distinct prompts once per window:
145
+ length uses empirical KS distance. Literal, case-insensitive keywords bounded
146
+ by non-word characters use absolute prevalence change. Difficulty total variation uses
147
+ known tags only and is unavailable if either window has none.
148
+ """
149
+ before, after = _window(before), _window(after)
150
+ if before[1] >= after[0]:
151
+ raise ValueError("windows must not overlap and must be in before/after order")
152
+ words = _keywords(keywords)
153
+ units = resolve_sampling_units(rollouts, sampling_units)
154
+ frame, kind = feature_values(rollouts, features, run=run, feature=feature)
155
+ return _prompt_mix_prepared(
156
+ frame,
157
+ kind,
158
+ run=run,
159
+ feature=feature,
160
+ before=before,
161
+ after=after,
162
+ keywords=words,
163
+ difficulty=difficulty,
164
+ sampling_units=subset_sampling_units(units, frame),
165
+ )
166
+
167
+
168
+ def _prompt_mix_prepared(
169
+ run_rows: pd.DataFrame,
170
+ kind: str,
171
+ *,
172
+ run: str,
173
+ feature: str,
174
+ before: tuple[int, int],
175
+ after: tuple[int, int],
176
+ keywords: Sequence[str] = (),
177
+ difficulty: Mapping[str, str] | None = None,
178
+ sampling_units: pd.DataFrame | None = None,
179
+ ) -> PromptMixEvidence:
180
+ """Compute prompt-mix evidence from identity-aligned, validated values."""
181
+ before, after = _window(before), _window(after)
182
+ if before[1] >= after[0]:
183
+ raise ValueError("windows must not overlap and must be in before/after order")
184
+ words = _keywords(keywords)
185
+ if difficulty is not None and (
186
+ not isinstance(difficulty, Mapping)
187
+ or any(
188
+ not isinstance(k, str) or not isinstance(v, str) or not v.strip()
189
+ for k, v in difficulty.items()
190
+ )
191
+ or set(difficulty) - set(run_rows.prompt_id)
192
+ ):
193
+ raise ValueError(
194
+ "difficulty must map prompt IDs in this run to nonempty string tags"
195
+ )
196
+ masks = [run_rows.step.between(*window) for window in (before, after)]
197
+ if any(not mask.any() for mask in masks):
198
+ raise ValueError("comparison window is empty")
199
+ selected = run_rows.loc[masks[0] | masks[1]]
200
+ frame = selected[["run", "sample_id", "step", "prompt_id", "_value"]].rename(
201
+ columns={"_value": "value"}
202
+ )
203
+
204
+ grouped = []
205
+ for side, window in [("before", before), ("after", after)]:
206
+ group = (
207
+ frame.loc[frame.step.between(*window)]
208
+ .groupby("prompt_id", sort=True)
209
+ .value.agg(["mean", "size"])
210
+ )
211
+ grouped.append(
212
+ group.rename(columns={"mean": f"{side}_mean", "size": f"{side}_n"})
213
+ )
214
+ detail = grouped[0].join(grouped[1], how="outer").sort_index()
215
+ for side in ("before", "after"):
216
+ detail[f"{side}_n"] = detail[f"{side}_n"].fillna(0).astype("int64")
217
+ detail["change"] = detail.after_mean - detail.before_mean
218
+ if sampling_units is None:
219
+ detail["sampling_unit_id"] = detail.index
220
+ else:
221
+ detail["sampling_unit_id"] = sampling_units.set_index(
222
+ "prompt_id"
223
+ ).sampling_unit_id.reindex(detail.index)
224
+ prompts = selected.drop_duplicates("prompt_id").set_index("prompt_id").prompt
225
+ detail["prompt_chars"] = prompts.str.len()
226
+ for word in words:
227
+ pattern = re.compile(r"(?<!\w)" + re.escape(word) + r"(?!\w)")
228
+ detail[f"keyword:{word}"] = prompts.str.casefold().str.contains(pattern)
229
+ if difficulty is not None:
230
+ # Ignore mapping defaults: only explicitly supplied tags are evidence.
231
+ detail["difficulty"] = detail.index.to_series().map(dict(difficulty))
232
+
233
+ changes = detail.change.dropna()
234
+ shared_count = len(changes)
235
+ shared_change = float(changes.mean()) if shared_count else None
236
+ shared_units = detail.loc[changes.index, "sampling_unit_id"]
237
+ if shared_count > 1 and shared_units.is_unique:
238
+ # Preserve independent-prompt arithmetic when units are unique.
239
+ shared_se = float(changes.std(ddof=1) / np.sqrt(shared_count))
240
+ else:
241
+ shared_se = cluster_standard_error(changes.to_numpy(), shared_units.to_numpy())
242
+ return PromptMixEvidence(
243
+ run=run,
244
+ feature=feature,
245
+ kind=kind,
246
+ before=before,
247
+ after=after,
248
+ positive_prompts=int(detail.after_mean.gt(0).sum())
249
+ if kind == "binary"
250
+ else None,
251
+ total_prompts=int(detail.after_n.gt(0).sum()),
252
+ shared_prompts=shared_count,
253
+ shared_sampling_units=int(shared_units.nunique()),
254
+ shared_change=shared_change,
255
+ shared_se=shared_se,
256
+ per_prompt=detail.reset_index(),
257
+ prompt_shift=_prompt_shift(detail, words, difficulty is not None),
258
+ )
rolloutdiff/ranking.py ADDED
@@ -0,0 +1,388 @@
1
+ """Within-run inspection order from existing feature evidence."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+ from fractions import Fraction
7
+ from numbers import Integral, Real
8
+
9
+ import numpy as np
10
+ import pandas as pd
11
+
12
+ from .events import _EVENT_COLUMNS, _MEMBER_COLUMNS
13
+ from .series import _require_columns
14
+
15
+ _EVIDENCE_FIELDS = [
16
+ "tier",
17
+ "kind",
18
+ "sequential_status",
19
+ "onset_step",
20
+ "alert_step",
21
+ "scale_source",
22
+ "numerical_floor_dominant",
23
+ "replication_status",
24
+ "replicates",
25
+ "replication_coverage_policy",
26
+ "replication_horizon_complete",
27
+ "persistence_status",
28
+ "persistence_fraction",
29
+ "persistence_coverage",
30
+ "persistence_n_observations",
31
+ "persistence_n_evaluable",
32
+ "persistence_n_retained",
33
+ "persistence_n_unavailable",
34
+ "reward_status",
35
+ "reward_mean_difference",
36
+ "reward_n_groups",
37
+ "reward_n_mixed_groups",
38
+ "reward_n_rollouts",
39
+ "reward_n_included_rollouts",
40
+ "promptmix_status",
41
+ "prompt_positive_prompts",
42
+ "prompt_total_prompts",
43
+ ]
44
+ _COLUMNS = [
45
+ *_EVENT_COLUMNS,
46
+ "rank",
47
+ "representative_feature",
48
+ "direction",
49
+ "member_status",
50
+ "ranking_status",
51
+ "n_prioritizable_members",
52
+ *_EVIDENCE_FIELDS,
53
+ "replication_availability",
54
+ "persistence_availability",
55
+ "reward_availability",
56
+ "breadth_availability",
57
+ "prompt_positive_fraction",
58
+ "missing_evidence_members",
59
+ "numerical_floor_members",
60
+ "numerical_floor_unavailable_members",
61
+ ]
62
+
63
+
64
+ def _ranking_table(rows=()) -> pd.DataFrame:
65
+ result = pd.DataFrame(rows, columns=_COLUMNS, dtype=object)
66
+ for name in ["rank", "n_members", "n_prioritizable_members"]:
67
+ result[name] = result[name].astype("int64")
68
+ for name in [
69
+ "replicates",
70
+ "replication_horizon_complete",
71
+ "numerical_floor_dominant",
72
+ ]:
73
+ result[name] = pd.array(result[name], dtype="boolean")
74
+ for name in [
75
+ "persistence_fraction",
76
+ "persistence_coverage",
77
+ "reward_mean_difference",
78
+ "prompt_positive_fraction",
79
+ ]:
80
+ result[name] = pd.to_numeric(result[name]).astype(float)
81
+ return result
82
+
83
+
84
+ def _validate_inputs(events, members, evidence) -> dict:
85
+ for frame, columns, keys, name in [
86
+ (events, _EVENT_COLUMNS, ["event_id"], "events"),
87
+ (members, _MEMBER_COLUMNS, ["run", "feature"], "event members"),
88
+ (
89
+ evidence,
90
+ ["run", "feature", *_EVIDENCE_FIELDS],
91
+ ["run", "feature"],
92
+ "evidence",
93
+ ),
94
+ ]:
95
+ _require_columns(frame, columns, name)
96
+ if frame.duplicated(keys).any():
97
+ raise ValueError(f"duplicate {name} keys")
98
+ for column in set(keys) | ({"run", "event_id"} & set(columns)):
99
+ if (
100
+ not frame[column]
101
+ .map(lambda v: isinstance(v, str) and bool(v.strip()))
102
+ .all()
103
+ ):
104
+ raise ValueError(f"{name} {column} must be a nonempty string")
105
+ if (
106
+ not members.kind.isin(["binary", "continuous"]).all()
107
+ or not members.direction.isin(["up", "down"]).all()
108
+ ):
109
+ raise ValueError("invalid member kind or direction")
110
+ for name in ["onset_step", "alert_step"]:
111
+ if (
112
+ not members[name]
113
+ .map(
114
+ lambda v: isinstance(v, Integral) and not isinstance(v, bool) and v >= 0
115
+ )
116
+ .all()
117
+ ):
118
+ raise ValueError(f"member {name} must be a nonnegative integer")
119
+ if (members.onset_step > members.alert_step).any():
120
+ raise ValueError("member onset exceeds alert step")
121
+ groups = {
122
+ key: frame.sort_values(["onset_step", "alert_step", "feature"]).to_dict(
123
+ "records"
124
+ )
125
+ for key, frame in members.groupby("event_id", sort=False)
126
+ }
127
+ if set(groups) != set(events.event_id):
128
+ raise ValueError("events and memberships must cover the same event IDs")
129
+ for event in events.to_dict("records"):
130
+ group = groups[event["event_id"]]
131
+ refs = tuple((row["run"], row["feature"]) for row in group)
132
+ if (
133
+ event["n_members"] != len(group)
134
+ or any(row["run"] != event["run"] for row in group)
135
+ or event["onset_start"] != min(row["onset_step"] for row in group)
136
+ or event["onset_end"] != max(row["onset_step"] for row in group)
137
+ or not isinstance(event["members"], tuple)
138
+ or set(event["members"]) != set(refs)
139
+ or len(event["members"]) != len(refs)
140
+ ):
141
+ raise ValueError("event metadata conflicts with its membership")
142
+ return groups
143
+
144
+
145
+ def _finite(value) -> bool:
146
+ return (
147
+ isinstance(value, Real)
148
+ and not isinstance(value, (bool, np.bool_))
149
+ and math.isfinite(value)
150
+ )
151
+
152
+
153
+ def _count(value) -> bool:
154
+ return _finite(value) and value >= 0 and int(value) == value
155
+
156
+
157
+ def _validate_persistence(row: dict) -> None:
158
+ status = row["persistence_status"]
159
+ if status not in ("available", "partial"):
160
+ return
161
+ counts = {}
162
+ for name in ["retained", "evaluable", "unavailable", "observations"]:
163
+ value = row[f"persistence_n_{name}"]
164
+ if pd.isna(value):
165
+ counts[name] = None
166
+ elif not _count(value):
167
+ raise ValueError(f"persistence_n_{name} must be a nonnegative integer")
168
+ else:
169
+ counts[name] = int(value)
170
+ retained, evaluable, unavailable, observed = (
171
+ counts[name]
172
+ for name in ["retained", "evaluable", "unavailable", "observations"]
173
+ )
174
+ if evaluable == 0 or observed == 0:
175
+ raise ValueError(
176
+ "observed persistence requires positive evaluable and total counts"
177
+ )
178
+ for left, right in [
179
+ (retained, evaluable),
180
+ (evaluable, observed),
181
+ (retained, observed),
182
+ ]:
183
+ if left is not None and right is not None and left > right:
184
+ raise ValueError("inconsistent persistence observation counts")
185
+ if (
186
+ unavailable is not None
187
+ and observed is not None
188
+ and (
189
+ unavailable >= observed
190
+ or (retained is not None and retained + unavailable > observed)
191
+ )
192
+ ):
193
+ raise ValueError("inconsistent persistence unavailable observation count")
194
+ expected_unavailable = unavailable
195
+ if evaluable is not None and observed is not None:
196
+ expected_unavailable = observed - evaluable
197
+ if unavailable is not None and unavailable != expected_unavailable:
198
+ raise ValueError(
199
+ "persistence evaluable and unavailable counts must sum to observations"
200
+ )
201
+ if expected_unavailable is not None and (status == "available") != (
202
+ expected_unavailable == 0
203
+ ):
204
+ raise ValueError("persistence status conflicts with observation counts")
205
+ for name, numerator, denominator in [
206
+ ("fraction", retained, evaluable),
207
+ ("coverage", evaluable, observed),
208
+ ]:
209
+ value = row[f"persistence_{name}"]
210
+ if pd.isna(value):
211
+ continue
212
+ if not _finite(value):
213
+ raise ValueError(f"persistence_{name} must be finite")
214
+ # Relative roundoff tolerance must not equate tiny positive ratios with zero.
215
+ if (
216
+ numerator is not None
217
+ and denominator is not None
218
+ and not math.isclose(
219
+ value,
220
+ numerator / denominator,
221
+ rel_tol=8 * np.finfo(float).eps,
222
+ abs_tol=0.0,
223
+ )
224
+ ):
225
+ raise ValueError(f"persistence_{name} conflicts with observation counts")
226
+
227
+
228
+ def _descending(available: bool, *values) -> tuple:
229
+ return (0, *(-value for value in values)) if available else (1,)
230
+
231
+
232
+ def _candidate(member: dict, source: dict | None) -> tuple[tuple, dict]:
233
+ row = {
234
+ name: source[name] if source is not None else None for name in _EVIDENCE_FIELDS
235
+ }
236
+ if source is not None:
237
+ for name in ["kind", "onset_step", "alert_step"]:
238
+ if pd.isna(source[name]) or source[name] != member[name]:
239
+ raise ValueError("member metadata conflicts with feature evidence")
240
+ if source["sequential_status"] != "alerted":
241
+ raise ValueError("event member evidence must describe a sequential alert")
242
+ if source["tier"] not in ("structural", "prose", None) and not pd.isna(
243
+ source["tier"]
244
+ ):
245
+ raise ValueError("invalid feature tier")
246
+ row.update(
247
+ representative_feature=member["feature"],
248
+ direction=member["direction"],
249
+ member_status=member["status"],
250
+ kind=member["kind"],
251
+ onset_step=member["onset_step"],
252
+ alert_step=member["alert_step"],
253
+ )
254
+
255
+ status, replicated = row["replication_status"], row["replicates"]
256
+ known = isinstance(replicated, (bool, np.bool_))
257
+ if known and (
258
+ (status == "replicated" and not replicated)
259
+ or (status == "not_replicated" and replicated)
260
+ ):
261
+ raise ValueError("replication status conflicts with replicates")
262
+ replication = known and status in ("replicated", "not_replicated")
263
+ _validate_persistence(row)
264
+ fraction, coverage = row["persistence_fraction"], row["persistence_coverage"]
265
+ evaluable, observed = (
266
+ row["persistence_n_evaluable"],
267
+ row["persistence_n_observations"],
268
+ )
269
+ persistence = (
270
+ row["persistence_status"] in ("available", "partial")
271
+ and _finite(fraction)
272
+ and 0 <= fraction <= 1
273
+ and _finite(coverage)
274
+ and 0 < coverage <= 1
275
+ and _count(evaluable)
276
+ and _count(observed)
277
+ and 0 < evaluable <= observed
278
+ )
279
+ difference, mixed = row["reward_mean_difference"], row["reward_n_mixed_groups"]
280
+ included, total = row["reward_n_included_rollouts"], row["reward_n_rollouts"]
281
+ groups = row["reward_n_groups"]
282
+ reward = (
283
+ row["reward_status"] == "available"
284
+ and _finite(difference)
285
+ and _count(mixed)
286
+ and _count(groups)
287
+ and 0 < mixed <= groups
288
+ and _count(included)
289
+ and _count(total)
290
+ and 0 < included <= total
291
+ )
292
+ positive, prompts = row["prompt_positive_prompts"], row["prompt_total_prompts"]
293
+ # Shared-prompt comparison status does not describe after-window breadth.
294
+ breadth = (
295
+ member["kind"] == "binary"
296
+ and _count(positive)
297
+ and _count(prompts)
298
+ and 0 <= positive <= prompts
299
+ and prompts > 0
300
+ )
301
+ for name, available in [
302
+ ("replication", replication),
303
+ ("persistence", persistence),
304
+ ("reward", reward),
305
+ ("breadth", breadth),
306
+ ]:
307
+ row[f"{name}_availability"] = "available" if available else "unavailable"
308
+ breadth_fraction = Fraction(int(positive), int(prompts)) if breadth else None
309
+ row["prompt_positive_fraction"] = float(breadth_fraction) if breadth else None
310
+ if member["kind"] != "binary":
311
+ ranking_status = "contextual"
312
+ elif member["direction"] != "up":
313
+ ranking_status = "unsupported_direction"
314
+ elif member["status"] not in ("grouped", "singleton_no_match"):
315
+ ranking_status = "unavailable_member"
316
+ elif row["tier"] not in ("structural", "prose") or not any(
317
+ [replication, persistence, reward, breadth]
318
+ ):
319
+ ranking_status = "unavailable_evidence"
320
+ else:
321
+ ranking_status = "behavioral"
322
+ row["ranking_status"] = ranking_status
323
+ tie = (member["onset_step"], member["alert_step"], member["feature"])
324
+ if ranking_status != "behavioral":
325
+ return (1, *tie), row
326
+ return (
327
+ 0,
328
+ 0 if row["tier"] == "structural" else 1,
329
+ 0 if replication and replicated else 1 if replication else 2,
330
+ _descending(persistence, fraction, coverage, evaluable),
331
+ _descending(reward, difference, mixed, included),
332
+ _descending(breadth, breadth_fraction, positive),
333
+ *tie,
334
+ ), row
335
+
336
+
337
+ def rank_events(
338
+ events: pd.DataFrame, event_members: pd.DataFrame, evidence: pd.DataFrame
339
+ ) -> pd.DataFrame:
340
+ """Order each run using the best single rising-binary member's evidence.
341
+
342
+ Lexicographic priority: structural before prose; replicated, not replicated,
343
+ then unavailable; persistence fraction, coverage, evaluable count; signed
344
+ reward difference, mixed groups, included rollouts; positive-prompt fraction,
345
+ then positive count. Numeric components descend, with observed values before
346
+ unavailable values. Ties use onset, alert, feature, then event ID.
347
+
348
+ Contextual/unsupported events and members lacking all support follow in
349
+ onset/alert/feature/ID order. Ranking is a heuristic inspection order.
350
+ Floor warnings remain visible.
351
+ """
352
+ groups = _validate_inputs(events, event_members, evidence)
353
+ sources = {(row["run"], row["feature"]): row for row in evidence.to_dict("records")}
354
+ rows = []
355
+ for event in events.to_dict("records"):
356
+ members = groups[event["event_id"]]
357
+ candidates, missing, floor, unknown_floor = [], [], [], []
358
+ for member in members:
359
+ reference = (member["run"], member["feature"])
360
+ source = sources.get(reference)
361
+ candidates.append(_candidate(member, source))
362
+ if source is None:
363
+ missing.append(reference)
364
+ warning = source["numerical_floor_dominant"] if source is not None else None
365
+ if not isinstance(warning, (bool, np.bool_)):
366
+ unknown_floor.append(reference)
367
+ elif warning:
368
+ floor.append(reference)
369
+ key, representative = min(candidates, key=lambda item: item[0])
370
+ rows.append(
371
+ {
372
+ **event,
373
+ **representative,
374
+ "_key": key,
375
+ "n_prioritizable_members": sum(
376
+ row["ranking_status"] == "behavioral" for _, row in candidates
377
+ ),
378
+ "missing_evidence_members": tuple(missing),
379
+ "numerical_floor_members": tuple(floor),
380
+ "numerical_floor_unavailable_members": tuple(unknown_floor),
381
+ }
382
+ )
383
+ rows.sort(key=lambda row: (row["run"], row["_key"], row["event_id"]))
384
+ previous, rank = None, 0
385
+ for row in rows:
386
+ rank = rank + 1 if row["run"] == previous else 1
387
+ previous, row["rank"] = row["run"], rank
388
+ return _ranking_table(rows)