rolloutdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rolloutdiff/__init__.py +130 -0
- rolloutdiff/_aria_replay.py +346 -0
- rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
- rolloutdiff/_dependence.py +195 -0
- rolloutdiff/_feature_values.py +79 -0
- rolloutdiff/_replay_container.py +568 -0
- rolloutdiff/_report_validation.py +758 -0
- rolloutdiff/_retrospective_policy.py +33 -0
- rolloutdiff/_sampling.py +127 -0
- rolloutdiff/_series_validation.py +67 -0
- rolloutdiff/_watch_report.py +436 -0
- rolloutdiff/_watch_storage.py +800 -0
- rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
- rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
- rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
- rolloutdiff/_web/assets/third-party-notices.txt +112 -0
- rolloutdiff/_web/index.html +14 -0
- rolloutdiff/adapters.py +264 -0
- rolloutdiff/app.py +956 -0
- rolloutdiff/app_evidence.py +678 -0
- rolloutdiff/benchmark_reference.py +458 -0
- rolloutdiff/cli.py +1292 -0
- rolloutdiff/comparison.py +572 -0
- rolloutdiff/comparison_report.py +982 -0
- rolloutdiff/corroboration.py +287 -0
- rolloutdiff/detect.py +307 -0
- rolloutdiff/events.py +173 -0
- rolloutdiff/evidence.py +285 -0
- rolloutdiff/exemplars.py +240 -0
- rolloutdiff/features.py +823 -0
- rolloutdiff/html_report.py +726 -0
- rolloutdiff/inspect_logs.py +383 -0
- rolloutdiff/inspection.py +207 -0
- rolloutdiff/integrations.py +552 -0
- rolloutdiff/investigation.py +1146 -0
- rolloutdiff/notifications.py +406 -0
- rolloutdiff/pipeline.py +809 -0
- rolloutdiff/producer.py +663 -0
- rolloutdiff/prompt_mix.py +258 -0
- rolloutdiff/ranking.py +388 -0
- rolloutdiff/replay.py +589 -0
- rolloutdiff/replication.py +572 -0
- rolloutdiff/report.py +838 -0
- rolloutdiff/retrospective.py +386 -0
- rolloutdiff/schema.py +109 -0
- rolloutdiff/sensitivity.py +202 -0
- rolloutdiff/series.py +430 -0
- rolloutdiff/tool_features.py +422 -0
- rolloutdiff/trajectories.py +236 -0
- rolloutdiff/watch-compatibility.json +13 -0
- rolloutdiff/watch.py +742 -0
- rolloutdiff-0.1.0.dist-info/METADATA +158 -0
- rolloutdiff-0.1.0.dist-info/RECORD +62 -0
- rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
- rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
- rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
- rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
"""Prompt-mix evidence for caller-selected windows of one run."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from collections.abc import Mapping, Sequence
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
|
|
9
|
+
import numpy as np
|
|
10
|
+
import pandas as pd
|
|
11
|
+
|
|
12
|
+
from ._feature_values import feature_values
|
|
13
|
+
from ._sampling import (
|
|
14
|
+
cluster_standard_error,
|
|
15
|
+
resolve_sampling_units,
|
|
16
|
+
subset_sampling_units,
|
|
17
|
+
)
|
|
18
|
+
from .features import FeatureTables
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class PromptMixEvidence:
|
|
23
|
+
run: str
|
|
24
|
+
feature: str
|
|
25
|
+
kind: str
|
|
26
|
+
before: tuple[int, int]
|
|
27
|
+
after: tuple[int, int]
|
|
28
|
+
positive_prompts: int | None
|
|
29
|
+
total_prompts: int
|
|
30
|
+
shared_prompts: int
|
|
31
|
+
shared_sampling_units: int
|
|
32
|
+
shared_change: float | None
|
|
33
|
+
shared_se: float | None
|
|
34
|
+
per_prompt: pd.DataFrame
|
|
35
|
+
prompt_shift: pd.DataFrame
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _window(value: tuple[int, int]) -> tuple[int, int]:
|
|
39
|
+
if (
|
|
40
|
+
not isinstance(value, tuple)
|
|
41
|
+
or len(value) != 2
|
|
42
|
+
or any(type(bound) is not int or bound < 0 for bound in value)
|
|
43
|
+
or value[0] > value[1]
|
|
44
|
+
):
|
|
45
|
+
raise ValueError(
|
|
46
|
+
"window must be an inclusive (start, end) pair of nonnegative integers"
|
|
47
|
+
)
|
|
48
|
+
return value
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _keywords(values: Sequence[str]) -> list[str]:
|
|
52
|
+
if isinstance(values, str) or not isinstance(values, Sequence):
|
|
53
|
+
raise ValueError("keywords must be a sequence of literal strings")
|
|
54
|
+
result = []
|
|
55
|
+
for value in values:
|
|
56
|
+
if not isinstance(value, str) or not value.strip():
|
|
57
|
+
raise ValueError("keyword must be a nonempty string")
|
|
58
|
+
word = value.strip().casefold()
|
|
59
|
+
if word in result:
|
|
60
|
+
raise ValueError("duplicate keyword after case normalization")
|
|
61
|
+
result.append(word)
|
|
62
|
+
return result
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _ks(before: np.ndarray, after: np.ndarray) -> float:
|
|
66
|
+
points = np.union1d(before, after)
|
|
67
|
+
a = np.searchsorted(np.sort(before), points, side="right") / len(before)
|
|
68
|
+
b = np.searchsorted(np.sort(after), points, side="right") / len(after)
|
|
69
|
+
return float(np.max(np.abs(a - b)))
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _prompt_shift(
|
|
73
|
+
detail: pd.DataFrame, keywords: list[str], has_difficulty: bool
|
|
74
|
+
) -> pd.DataFrame:
|
|
75
|
+
before = detail.loc[detail.before_n.gt(0)]
|
|
76
|
+
after = detail.loc[detail.after_n.gt(0)]
|
|
77
|
+
rows = []
|
|
78
|
+
for descriptor in ["prompt_chars", *(f"keyword:{word}" for word in keywords)]:
|
|
79
|
+
a, b = (
|
|
80
|
+
before[descriptor].to_numpy(dtype=float),
|
|
81
|
+
after[descriptor].to_numpy(dtype=float),
|
|
82
|
+
)
|
|
83
|
+
delta = float(b.mean() - a.mean())
|
|
84
|
+
rows.append(
|
|
85
|
+
{
|
|
86
|
+
"descriptor": descriptor,
|
|
87
|
+
"statistic": "ks"
|
|
88
|
+
if descriptor == "prompt_chars"
|
|
89
|
+
else "absolute_prevalence",
|
|
90
|
+
"before": float(a.mean()),
|
|
91
|
+
"after": float(b.mean()),
|
|
92
|
+
"delta": delta,
|
|
93
|
+
"distance": _ks(a, b) if descriptor == "prompt_chars" else abs(delta),
|
|
94
|
+
"n_before": len(a),
|
|
95
|
+
"n_after": len(b),
|
|
96
|
+
"missing_before": 0,
|
|
97
|
+
"missing_after": 0,
|
|
98
|
+
}
|
|
99
|
+
)
|
|
100
|
+
if has_difficulty:
|
|
101
|
+
a, b = before.difficulty.dropna(), after.difficulty.dropna()
|
|
102
|
+
distance = float("nan")
|
|
103
|
+
if len(a) and len(b):
|
|
104
|
+
pa, pb = a.value_counts(normalize=True), b.value_counts(normalize=True)
|
|
105
|
+
distance = float(pa.sub(pb, fill_value=0).abs().sum() / 2)
|
|
106
|
+
rows.append(
|
|
107
|
+
{
|
|
108
|
+
"descriptor": "difficulty",
|
|
109
|
+
"statistic": "total_variation",
|
|
110
|
+
"before": float("nan"),
|
|
111
|
+
"after": float("nan"),
|
|
112
|
+
"delta": float("nan"),
|
|
113
|
+
"distance": distance,
|
|
114
|
+
"n_before": len(a),
|
|
115
|
+
"n_after": len(b),
|
|
116
|
+
"missing_before": len(before) - len(a),
|
|
117
|
+
"missing_after": len(after) - len(b),
|
|
118
|
+
}
|
|
119
|
+
)
|
|
120
|
+
return pd.DataFrame(rows)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def prompt_mix(
|
|
124
|
+
rollouts: pd.DataFrame,
|
|
125
|
+
features: FeatureTables,
|
|
126
|
+
*,
|
|
127
|
+
run: str,
|
|
128
|
+
feature: str,
|
|
129
|
+
before: tuple[int, int],
|
|
130
|
+
after: tuple[int, int],
|
|
131
|
+
keywords: Sequence[str] = (),
|
|
132
|
+
difficulty: Mapping[str, str] | None = None,
|
|
133
|
+
sampling_units: pd.DataFrame | None = None,
|
|
134
|
+
) -> PromptMixEvidence:
|
|
135
|
+
"""Measure breadth, paired prompt change, and prompt descriptor shift.
|
|
136
|
+
|
|
137
|
+
Windows are inclusive, ordered, disjoint, and nonempty. Pool completions per
|
|
138
|
+
prompt/window, then weight shared prompts equally. Default SE is paired-change
|
|
139
|
+
SD(ddof=1) / sqrt(shared prompts); explicit units cluster those residuals.
|
|
140
|
+
Fewer than two shared units makes SE unavailable; no shared prompts also
|
|
141
|
+
makes change unavailable.
|
|
142
|
+
|
|
143
|
+
Binary breadth counts after-window prompts with any positive value; continuous
|
|
144
|
+
breadth is unavailable. Descriptors count distinct prompts once per window:
|
|
145
|
+
length uses empirical KS distance. Literal, case-insensitive keywords bounded
|
|
146
|
+
by non-word characters use absolute prevalence change. Difficulty total variation uses
|
|
147
|
+
known tags only and is unavailable if either window has none.
|
|
148
|
+
"""
|
|
149
|
+
before, after = _window(before), _window(after)
|
|
150
|
+
if before[1] >= after[0]:
|
|
151
|
+
raise ValueError("windows must not overlap and must be in before/after order")
|
|
152
|
+
words = _keywords(keywords)
|
|
153
|
+
units = resolve_sampling_units(rollouts, sampling_units)
|
|
154
|
+
frame, kind = feature_values(rollouts, features, run=run, feature=feature)
|
|
155
|
+
return _prompt_mix_prepared(
|
|
156
|
+
frame,
|
|
157
|
+
kind,
|
|
158
|
+
run=run,
|
|
159
|
+
feature=feature,
|
|
160
|
+
before=before,
|
|
161
|
+
after=after,
|
|
162
|
+
keywords=words,
|
|
163
|
+
difficulty=difficulty,
|
|
164
|
+
sampling_units=subset_sampling_units(units, frame),
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _prompt_mix_prepared(
|
|
169
|
+
run_rows: pd.DataFrame,
|
|
170
|
+
kind: str,
|
|
171
|
+
*,
|
|
172
|
+
run: str,
|
|
173
|
+
feature: str,
|
|
174
|
+
before: tuple[int, int],
|
|
175
|
+
after: tuple[int, int],
|
|
176
|
+
keywords: Sequence[str] = (),
|
|
177
|
+
difficulty: Mapping[str, str] | None = None,
|
|
178
|
+
sampling_units: pd.DataFrame | None = None,
|
|
179
|
+
) -> PromptMixEvidence:
|
|
180
|
+
"""Compute prompt-mix evidence from identity-aligned, validated values."""
|
|
181
|
+
before, after = _window(before), _window(after)
|
|
182
|
+
if before[1] >= after[0]:
|
|
183
|
+
raise ValueError("windows must not overlap and must be in before/after order")
|
|
184
|
+
words = _keywords(keywords)
|
|
185
|
+
if difficulty is not None and (
|
|
186
|
+
not isinstance(difficulty, Mapping)
|
|
187
|
+
or any(
|
|
188
|
+
not isinstance(k, str) or not isinstance(v, str) or not v.strip()
|
|
189
|
+
for k, v in difficulty.items()
|
|
190
|
+
)
|
|
191
|
+
or set(difficulty) - set(run_rows.prompt_id)
|
|
192
|
+
):
|
|
193
|
+
raise ValueError(
|
|
194
|
+
"difficulty must map prompt IDs in this run to nonempty string tags"
|
|
195
|
+
)
|
|
196
|
+
masks = [run_rows.step.between(*window) for window in (before, after)]
|
|
197
|
+
if any(not mask.any() for mask in masks):
|
|
198
|
+
raise ValueError("comparison window is empty")
|
|
199
|
+
selected = run_rows.loc[masks[0] | masks[1]]
|
|
200
|
+
frame = selected[["run", "sample_id", "step", "prompt_id", "_value"]].rename(
|
|
201
|
+
columns={"_value": "value"}
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
grouped = []
|
|
205
|
+
for side, window in [("before", before), ("after", after)]:
|
|
206
|
+
group = (
|
|
207
|
+
frame.loc[frame.step.between(*window)]
|
|
208
|
+
.groupby("prompt_id", sort=True)
|
|
209
|
+
.value.agg(["mean", "size"])
|
|
210
|
+
)
|
|
211
|
+
grouped.append(
|
|
212
|
+
group.rename(columns={"mean": f"{side}_mean", "size": f"{side}_n"})
|
|
213
|
+
)
|
|
214
|
+
detail = grouped[0].join(grouped[1], how="outer").sort_index()
|
|
215
|
+
for side in ("before", "after"):
|
|
216
|
+
detail[f"{side}_n"] = detail[f"{side}_n"].fillna(0).astype("int64")
|
|
217
|
+
detail["change"] = detail.after_mean - detail.before_mean
|
|
218
|
+
if sampling_units is None:
|
|
219
|
+
detail["sampling_unit_id"] = detail.index
|
|
220
|
+
else:
|
|
221
|
+
detail["sampling_unit_id"] = sampling_units.set_index(
|
|
222
|
+
"prompt_id"
|
|
223
|
+
).sampling_unit_id.reindex(detail.index)
|
|
224
|
+
prompts = selected.drop_duplicates("prompt_id").set_index("prompt_id").prompt
|
|
225
|
+
detail["prompt_chars"] = prompts.str.len()
|
|
226
|
+
for word in words:
|
|
227
|
+
pattern = re.compile(r"(?<!\w)" + re.escape(word) + r"(?!\w)")
|
|
228
|
+
detail[f"keyword:{word}"] = prompts.str.casefold().str.contains(pattern)
|
|
229
|
+
if difficulty is not None:
|
|
230
|
+
# Ignore mapping defaults: only explicitly supplied tags are evidence.
|
|
231
|
+
detail["difficulty"] = detail.index.to_series().map(dict(difficulty))
|
|
232
|
+
|
|
233
|
+
changes = detail.change.dropna()
|
|
234
|
+
shared_count = len(changes)
|
|
235
|
+
shared_change = float(changes.mean()) if shared_count else None
|
|
236
|
+
shared_units = detail.loc[changes.index, "sampling_unit_id"]
|
|
237
|
+
if shared_count > 1 and shared_units.is_unique:
|
|
238
|
+
# Preserve independent-prompt arithmetic when units are unique.
|
|
239
|
+
shared_se = float(changes.std(ddof=1) / np.sqrt(shared_count))
|
|
240
|
+
else:
|
|
241
|
+
shared_se = cluster_standard_error(changes.to_numpy(), shared_units.to_numpy())
|
|
242
|
+
return PromptMixEvidence(
|
|
243
|
+
run=run,
|
|
244
|
+
feature=feature,
|
|
245
|
+
kind=kind,
|
|
246
|
+
before=before,
|
|
247
|
+
after=after,
|
|
248
|
+
positive_prompts=int(detail.after_mean.gt(0).sum())
|
|
249
|
+
if kind == "binary"
|
|
250
|
+
else None,
|
|
251
|
+
total_prompts=int(detail.after_n.gt(0).sum()),
|
|
252
|
+
shared_prompts=shared_count,
|
|
253
|
+
shared_sampling_units=int(shared_units.nunique()),
|
|
254
|
+
shared_change=shared_change,
|
|
255
|
+
shared_se=shared_se,
|
|
256
|
+
per_prompt=detail.reset_index(),
|
|
257
|
+
prompt_shift=_prompt_shift(detail, words, difficulty is not None),
|
|
258
|
+
)
|
rolloutdiff/ranking.py
ADDED
|
@@ -0,0 +1,388 @@
|
|
|
1
|
+
"""Within-run inspection order from existing feature evidence."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
from fractions import Fraction
|
|
7
|
+
from numbers import Integral, Real
|
|
8
|
+
|
|
9
|
+
import numpy as np
|
|
10
|
+
import pandas as pd
|
|
11
|
+
|
|
12
|
+
from .events import _EVENT_COLUMNS, _MEMBER_COLUMNS
|
|
13
|
+
from .series import _require_columns
|
|
14
|
+
|
|
15
|
+
_EVIDENCE_FIELDS = [
|
|
16
|
+
"tier",
|
|
17
|
+
"kind",
|
|
18
|
+
"sequential_status",
|
|
19
|
+
"onset_step",
|
|
20
|
+
"alert_step",
|
|
21
|
+
"scale_source",
|
|
22
|
+
"numerical_floor_dominant",
|
|
23
|
+
"replication_status",
|
|
24
|
+
"replicates",
|
|
25
|
+
"replication_coverage_policy",
|
|
26
|
+
"replication_horizon_complete",
|
|
27
|
+
"persistence_status",
|
|
28
|
+
"persistence_fraction",
|
|
29
|
+
"persistence_coverage",
|
|
30
|
+
"persistence_n_observations",
|
|
31
|
+
"persistence_n_evaluable",
|
|
32
|
+
"persistence_n_retained",
|
|
33
|
+
"persistence_n_unavailable",
|
|
34
|
+
"reward_status",
|
|
35
|
+
"reward_mean_difference",
|
|
36
|
+
"reward_n_groups",
|
|
37
|
+
"reward_n_mixed_groups",
|
|
38
|
+
"reward_n_rollouts",
|
|
39
|
+
"reward_n_included_rollouts",
|
|
40
|
+
"promptmix_status",
|
|
41
|
+
"prompt_positive_prompts",
|
|
42
|
+
"prompt_total_prompts",
|
|
43
|
+
]
|
|
44
|
+
_COLUMNS = [
|
|
45
|
+
*_EVENT_COLUMNS,
|
|
46
|
+
"rank",
|
|
47
|
+
"representative_feature",
|
|
48
|
+
"direction",
|
|
49
|
+
"member_status",
|
|
50
|
+
"ranking_status",
|
|
51
|
+
"n_prioritizable_members",
|
|
52
|
+
*_EVIDENCE_FIELDS,
|
|
53
|
+
"replication_availability",
|
|
54
|
+
"persistence_availability",
|
|
55
|
+
"reward_availability",
|
|
56
|
+
"breadth_availability",
|
|
57
|
+
"prompt_positive_fraction",
|
|
58
|
+
"missing_evidence_members",
|
|
59
|
+
"numerical_floor_members",
|
|
60
|
+
"numerical_floor_unavailable_members",
|
|
61
|
+
]
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _ranking_table(rows=()) -> pd.DataFrame:
|
|
65
|
+
result = pd.DataFrame(rows, columns=_COLUMNS, dtype=object)
|
|
66
|
+
for name in ["rank", "n_members", "n_prioritizable_members"]:
|
|
67
|
+
result[name] = result[name].astype("int64")
|
|
68
|
+
for name in [
|
|
69
|
+
"replicates",
|
|
70
|
+
"replication_horizon_complete",
|
|
71
|
+
"numerical_floor_dominant",
|
|
72
|
+
]:
|
|
73
|
+
result[name] = pd.array(result[name], dtype="boolean")
|
|
74
|
+
for name in [
|
|
75
|
+
"persistence_fraction",
|
|
76
|
+
"persistence_coverage",
|
|
77
|
+
"reward_mean_difference",
|
|
78
|
+
"prompt_positive_fraction",
|
|
79
|
+
]:
|
|
80
|
+
result[name] = pd.to_numeric(result[name]).astype(float)
|
|
81
|
+
return result
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _validate_inputs(events, members, evidence) -> dict:
|
|
85
|
+
for frame, columns, keys, name in [
|
|
86
|
+
(events, _EVENT_COLUMNS, ["event_id"], "events"),
|
|
87
|
+
(members, _MEMBER_COLUMNS, ["run", "feature"], "event members"),
|
|
88
|
+
(
|
|
89
|
+
evidence,
|
|
90
|
+
["run", "feature", *_EVIDENCE_FIELDS],
|
|
91
|
+
["run", "feature"],
|
|
92
|
+
"evidence",
|
|
93
|
+
),
|
|
94
|
+
]:
|
|
95
|
+
_require_columns(frame, columns, name)
|
|
96
|
+
if frame.duplicated(keys).any():
|
|
97
|
+
raise ValueError(f"duplicate {name} keys")
|
|
98
|
+
for column in set(keys) | ({"run", "event_id"} & set(columns)):
|
|
99
|
+
if (
|
|
100
|
+
not frame[column]
|
|
101
|
+
.map(lambda v: isinstance(v, str) and bool(v.strip()))
|
|
102
|
+
.all()
|
|
103
|
+
):
|
|
104
|
+
raise ValueError(f"{name} {column} must be a nonempty string")
|
|
105
|
+
if (
|
|
106
|
+
not members.kind.isin(["binary", "continuous"]).all()
|
|
107
|
+
or not members.direction.isin(["up", "down"]).all()
|
|
108
|
+
):
|
|
109
|
+
raise ValueError("invalid member kind or direction")
|
|
110
|
+
for name in ["onset_step", "alert_step"]:
|
|
111
|
+
if (
|
|
112
|
+
not members[name]
|
|
113
|
+
.map(
|
|
114
|
+
lambda v: isinstance(v, Integral) and not isinstance(v, bool) and v >= 0
|
|
115
|
+
)
|
|
116
|
+
.all()
|
|
117
|
+
):
|
|
118
|
+
raise ValueError(f"member {name} must be a nonnegative integer")
|
|
119
|
+
if (members.onset_step > members.alert_step).any():
|
|
120
|
+
raise ValueError("member onset exceeds alert step")
|
|
121
|
+
groups = {
|
|
122
|
+
key: frame.sort_values(["onset_step", "alert_step", "feature"]).to_dict(
|
|
123
|
+
"records"
|
|
124
|
+
)
|
|
125
|
+
for key, frame in members.groupby("event_id", sort=False)
|
|
126
|
+
}
|
|
127
|
+
if set(groups) != set(events.event_id):
|
|
128
|
+
raise ValueError("events and memberships must cover the same event IDs")
|
|
129
|
+
for event in events.to_dict("records"):
|
|
130
|
+
group = groups[event["event_id"]]
|
|
131
|
+
refs = tuple((row["run"], row["feature"]) for row in group)
|
|
132
|
+
if (
|
|
133
|
+
event["n_members"] != len(group)
|
|
134
|
+
or any(row["run"] != event["run"] for row in group)
|
|
135
|
+
or event["onset_start"] != min(row["onset_step"] for row in group)
|
|
136
|
+
or event["onset_end"] != max(row["onset_step"] for row in group)
|
|
137
|
+
or not isinstance(event["members"], tuple)
|
|
138
|
+
or set(event["members"]) != set(refs)
|
|
139
|
+
or len(event["members"]) != len(refs)
|
|
140
|
+
):
|
|
141
|
+
raise ValueError("event metadata conflicts with its membership")
|
|
142
|
+
return groups
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _finite(value) -> bool:
|
|
146
|
+
return (
|
|
147
|
+
isinstance(value, Real)
|
|
148
|
+
and not isinstance(value, (bool, np.bool_))
|
|
149
|
+
and math.isfinite(value)
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _count(value) -> bool:
|
|
154
|
+
return _finite(value) and value >= 0 and int(value) == value
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _validate_persistence(row: dict) -> None:
|
|
158
|
+
status = row["persistence_status"]
|
|
159
|
+
if status not in ("available", "partial"):
|
|
160
|
+
return
|
|
161
|
+
counts = {}
|
|
162
|
+
for name in ["retained", "evaluable", "unavailable", "observations"]:
|
|
163
|
+
value = row[f"persistence_n_{name}"]
|
|
164
|
+
if pd.isna(value):
|
|
165
|
+
counts[name] = None
|
|
166
|
+
elif not _count(value):
|
|
167
|
+
raise ValueError(f"persistence_n_{name} must be a nonnegative integer")
|
|
168
|
+
else:
|
|
169
|
+
counts[name] = int(value)
|
|
170
|
+
retained, evaluable, unavailable, observed = (
|
|
171
|
+
counts[name]
|
|
172
|
+
for name in ["retained", "evaluable", "unavailable", "observations"]
|
|
173
|
+
)
|
|
174
|
+
if evaluable == 0 or observed == 0:
|
|
175
|
+
raise ValueError(
|
|
176
|
+
"observed persistence requires positive evaluable and total counts"
|
|
177
|
+
)
|
|
178
|
+
for left, right in [
|
|
179
|
+
(retained, evaluable),
|
|
180
|
+
(evaluable, observed),
|
|
181
|
+
(retained, observed),
|
|
182
|
+
]:
|
|
183
|
+
if left is not None and right is not None and left > right:
|
|
184
|
+
raise ValueError("inconsistent persistence observation counts")
|
|
185
|
+
if (
|
|
186
|
+
unavailable is not None
|
|
187
|
+
and observed is not None
|
|
188
|
+
and (
|
|
189
|
+
unavailable >= observed
|
|
190
|
+
or (retained is not None and retained + unavailable > observed)
|
|
191
|
+
)
|
|
192
|
+
):
|
|
193
|
+
raise ValueError("inconsistent persistence unavailable observation count")
|
|
194
|
+
expected_unavailable = unavailable
|
|
195
|
+
if evaluable is not None and observed is not None:
|
|
196
|
+
expected_unavailable = observed - evaluable
|
|
197
|
+
if unavailable is not None and unavailable != expected_unavailable:
|
|
198
|
+
raise ValueError(
|
|
199
|
+
"persistence evaluable and unavailable counts must sum to observations"
|
|
200
|
+
)
|
|
201
|
+
if expected_unavailable is not None and (status == "available") != (
|
|
202
|
+
expected_unavailable == 0
|
|
203
|
+
):
|
|
204
|
+
raise ValueError("persistence status conflicts with observation counts")
|
|
205
|
+
for name, numerator, denominator in [
|
|
206
|
+
("fraction", retained, evaluable),
|
|
207
|
+
("coverage", evaluable, observed),
|
|
208
|
+
]:
|
|
209
|
+
value = row[f"persistence_{name}"]
|
|
210
|
+
if pd.isna(value):
|
|
211
|
+
continue
|
|
212
|
+
if not _finite(value):
|
|
213
|
+
raise ValueError(f"persistence_{name} must be finite")
|
|
214
|
+
# Relative roundoff tolerance must not equate tiny positive ratios with zero.
|
|
215
|
+
if (
|
|
216
|
+
numerator is not None
|
|
217
|
+
and denominator is not None
|
|
218
|
+
and not math.isclose(
|
|
219
|
+
value,
|
|
220
|
+
numerator / denominator,
|
|
221
|
+
rel_tol=8 * np.finfo(float).eps,
|
|
222
|
+
abs_tol=0.0,
|
|
223
|
+
)
|
|
224
|
+
):
|
|
225
|
+
raise ValueError(f"persistence_{name} conflicts with observation counts")
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _descending(available: bool, *values) -> tuple:
|
|
229
|
+
return (0, *(-value for value in values)) if available else (1,)
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def _candidate(member: dict, source: dict | None) -> tuple[tuple, dict]:
|
|
233
|
+
row = {
|
|
234
|
+
name: source[name] if source is not None else None for name in _EVIDENCE_FIELDS
|
|
235
|
+
}
|
|
236
|
+
if source is not None:
|
|
237
|
+
for name in ["kind", "onset_step", "alert_step"]:
|
|
238
|
+
if pd.isna(source[name]) or source[name] != member[name]:
|
|
239
|
+
raise ValueError("member metadata conflicts with feature evidence")
|
|
240
|
+
if source["sequential_status"] != "alerted":
|
|
241
|
+
raise ValueError("event member evidence must describe a sequential alert")
|
|
242
|
+
if source["tier"] not in ("structural", "prose", None) and not pd.isna(
|
|
243
|
+
source["tier"]
|
|
244
|
+
):
|
|
245
|
+
raise ValueError("invalid feature tier")
|
|
246
|
+
row.update(
|
|
247
|
+
representative_feature=member["feature"],
|
|
248
|
+
direction=member["direction"],
|
|
249
|
+
member_status=member["status"],
|
|
250
|
+
kind=member["kind"],
|
|
251
|
+
onset_step=member["onset_step"],
|
|
252
|
+
alert_step=member["alert_step"],
|
|
253
|
+
)
|
|
254
|
+
|
|
255
|
+
status, replicated = row["replication_status"], row["replicates"]
|
|
256
|
+
known = isinstance(replicated, (bool, np.bool_))
|
|
257
|
+
if known and (
|
|
258
|
+
(status == "replicated" and not replicated)
|
|
259
|
+
or (status == "not_replicated" and replicated)
|
|
260
|
+
):
|
|
261
|
+
raise ValueError("replication status conflicts with replicates")
|
|
262
|
+
replication = known and status in ("replicated", "not_replicated")
|
|
263
|
+
_validate_persistence(row)
|
|
264
|
+
fraction, coverage = row["persistence_fraction"], row["persistence_coverage"]
|
|
265
|
+
evaluable, observed = (
|
|
266
|
+
row["persistence_n_evaluable"],
|
|
267
|
+
row["persistence_n_observations"],
|
|
268
|
+
)
|
|
269
|
+
persistence = (
|
|
270
|
+
row["persistence_status"] in ("available", "partial")
|
|
271
|
+
and _finite(fraction)
|
|
272
|
+
and 0 <= fraction <= 1
|
|
273
|
+
and _finite(coverage)
|
|
274
|
+
and 0 < coverage <= 1
|
|
275
|
+
and _count(evaluable)
|
|
276
|
+
and _count(observed)
|
|
277
|
+
and 0 < evaluable <= observed
|
|
278
|
+
)
|
|
279
|
+
difference, mixed = row["reward_mean_difference"], row["reward_n_mixed_groups"]
|
|
280
|
+
included, total = row["reward_n_included_rollouts"], row["reward_n_rollouts"]
|
|
281
|
+
groups = row["reward_n_groups"]
|
|
282
|
+
reward = (
|
|
283
|
+
row["reward_status"] == "available"
|
|
284
|
+
and _finite(difference)
|
|
285
|
+
and _count(mixed)
|
|
286
|
+
and _count(groups)
|
|
287
|
+
and 0 < mixed <= groups
|
|
288
|
+
and _count(included)
|
|
289
|
+
and _count(total)
|
|
290
|
+
and 0 < included <= total
|
|
291
|
+
)
|
|
292
|
+
positive, prompts = row["prompt_positive_prompts"], row["prompt_total_prompts"]
|
|
293
|
+
# Shared-prompt comparison status does not describe after-window breadth.
|
|
294
|
+
breadth = (
|
|
295
|
+
member["kind"] == "binary"
|
|
296
|
+
and _count(positive)
|
|
297
|
+
and _count(prompts)
|
|
298
|
+
and 0 <= positive <= prompts
|
|
299
|
+
and prompts > 0
|
|
300
|
+
)
|
|
301
|
+
for name, available in [
|
|
302
|
+
("replication", replication),
|
|
303
|
+
("persistence", persistence),
|
|
304
|
+
("reward", reward),
|
|
305
|
+
("breadth", breadth),
|
|
306
|
+
]:
|
|
307
|
+
row[f"{name}_availability"] = "available" if available else "unavailable"
|
|
308
|
+
breadth_fraction = Fraction(int(positive), int(prompts)) if breadth else None
|
|
309
|
+
row["prompt_positive_fraction"] = float(breadth_fraction) if breadth else None
|
|
310
|
+
if member["kind"] != "binary":
|
|
311
|
+
ranking_status = "contextual"
|
|
312
|
+
elif member["direction"] != "up":
|
|
313
|
+
ranking_status = "unsupported_direction"
|
|
314
|
+
elif member["status"] not in ("grouped", "singleton_no_match"):
|
|
315
|
+
ranking_status = "unavailable_member"
|
|
316
|
+
elif row["tier"] not in ("structural", "prose") or not any(
|
|
317
|
+
[replication, persistence, reward, breadth]
|
|
318
|
+
):
|
|
319
|
+
ranking_status = "unavailable_evidence"
|
|
320
|
+
else:
|
|
321
|
+
ranking_status = "behavioral"
|
|
322
|
+
row["ranking_status"] = ranking_status
|
|
323
|
+
tie = (member["onset_step"], member["alert_step"], member["feature"])
|
|
324
|
+
if ranking_status != "behavioral":
|
|
325
|
+
return (1, *tie), row
|
|
326
|
+
return (
|
|
327
|
+
0,
|
|
328
|
+
0 if row["tier"] == "structural" else 1,
|
|
329
|
+
0 if replication and replicated else 1 if replication else 2,
|
|
330
|
+
_descending(persistence, fraction, coverage, evaluable),
|
|
331
|
+
_descending(reward, difference, mixed, included),
|
|
332
|
+
_descending(breadth, breadth_fraction, positive),
|
|
333
|
+
*tie,
|
|
334
|
+
), row
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def rank_events(
|
|
338
|
+
events: pd.DataFrame, event_members: pd.DataFrame, evidence: pd.DataFrame
|
|
339
|
+
) -> pd.DataFrame:
|
|
340
|
+
"""Order each run using the best single rising-binary member's evidence.
|
|
341
|
+
|
|
342
|
+
Lexicographic priority: structural before prose; replicated, not replicated,
|
|
343
|
+
then unavailable; persistence fraction, coverage, evaluable count; signed
|
|
344
|
+
reward difference, mixed groups, included rollouts; positive-prompt fraction,
|
|
345
|
+
then positive count. Numeric components descend, with observed values before
|
|
346
|
+
unavailable values. Ties use onset, alert, feature, then event ID.
|
|
347
|
+
|
|
348
|
+
Contextual/unsupported events and members lacking all support follow in
|
|
349
|
+
onset/alert/feature/ID order. Ranking is a heuristic inspection order.
|
|
350
|
+
Floor warnings remain visible.
|
|
351
|
+
"""
|
|
352
|
+
groups = _validate_inputs(events, event_members, evidence)
|
|
353
|
+
sources = {(row["run"], row["feature"]): row for row in evidence.to_dict("records")}
|
|
354
|
+
rows = []
|
|
355
|
+
for event in events.to_dict("records"):
|
|
356
|
+
members = groups[event["event_id"]]
|
|
357
|
+
candidates, missing, floor, unknown_floor = [], [], [], []
|
|
358
|
+
for member in members:
|
|
359
|
+
reference = (member["run"], member["feature"])
|
|
360
|
+
source = sources.get(reference)
|
|
361
|
+
candidates.append(_candidate(member, source))
|
|
362
|
+
if source is None:
|
|
363
|
+
missing.append(reference)
|
|
364
|
+
warning = source["numerical_floor_dominant"] if source is not None else None
|
|
365
|
+
if not isinstance(warning, (bool, np.bool_)):
|
|
366
|
+
unknown_floor.append(reference)
|
|
367
|
+
elif warning:
|
|
368
|
+
floor.append(reference)
|
|
369
|
+
key, representative = min(candidates, key=lambda item: item[0])
|
|
370
|
+
rows.append(
|
|
371
|
+
{
|
|
372
|
+
**event,
|
|
373
|
+
**representative,
|
|
374
|
+
"_key": key,
|
|
375
|
+
"n_prioritizable_members": sum(
|
|
376
|
+
row["ranking_status"] == "behavioral" for _, row in candidates
|
|
377
|
+
),
|
|
378
|
+
"missing_evidence_members": tuple(missing),
|
|
379
|
+
"numerical_floor_members": tuple(floor),
|
|
380
|
+
"numerical_floor_unavailable_members": tuple(unknown_floor),
|
|
381
|
+
}
|
|
382
|
+
)
|
|
383
|
+
rows.sort(key=lambda row: (row["run"], row["_key"], row["event_id"]))
|
|
384
|
+
previous, rank = None, 0
|
|
385
|
+
for row in rows:
|
|
386
|
+
rank = rank + 1 if row["run"] == previous else 1
|
|
387
|
+
previous, row["rank"] = row["run"], rank
|
|
388
|
+
return _ranking_table(rows)
|