rolloutdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rolloutdiff/__init__.py +130 -0
- rolloutdiff/_aria_replay.py +346 -0
- rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
- rolloutdiff/_dependence.py +195 -0
- rolloutdiff/_feature_values.py +79 -0
- rolloutdiff/_replay_container.py +568 -0
- rolloutdiff/_report_validation.py +758 -0
- rolloutdiff/_retrospective_policy.py +33 -0
- rolloutdiff/_sampling.py +127 -0
- rolloutdiff/_series_validation.py +67 -0
- rolloutdiff/_watch_report.py +436 -0
- rolloutdiff/_watch_storage.py +800 -0
- rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
- rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
- rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
- rolloutdiff/_web/assets/third-party-notices.txt +112 -0
- rolloutdiff/_web/index.html +14 -0
- rolloutdiff/adapters.py +264 -0
- rolloutdiff/app.py +956 -0
- rolloutdiff/app_evidence.py +678 -0
- rolloutdiff/benchmark_reference.py +458 -0
- rolloutdiff/cli.py +1292 -0
- rolloutdiff/comparison.py +572 -0
- rolloutdiff/comparison_report.py +982 -0
- rolloutdiff/corroboration.py +287 -0
- rolloutdiff/detect.py +307 -0
- rolloutdiff/events.py +173 -0
- rolloutdiff/evidence.py +285 -0
- rolloutdiff/exemplars.py +240 -0
- rolloutdiff/features.py +823 -0
- rolloutdiff/html_report.py +726 -0
- rolloutdiff/inspect_logs.py +383 -0
- rolloutdiff/inspection.py +207 -0
- rolloutdiff/integrations.py +552 -0
- rolloutdiff/investigation.py +1146 -0
- rolloutdiff/notifications.py +406 -0
- rolloutdiff/pipeline.py +809 -0
- rolloutdiff/producer.py +663 -0
- rolloutdiff/prompt_mix.py +258 -0
- rolloutdiff/ranking.py +388 -0
- rolloutdiff/replay.py +589 -0
- rolloutdiff/replication.py +572 -0
- rolloutdiff/report.py +838 -0
- rolloutdiff/retrospective.py +386 -0
- rolloutdiff/schema.py +109 -0
- rolloutdiff/sensitivity.py +202 -0
- rolloutdiff/series.py +430 -0
- rolloutdiff/tool_features.py +422 -0
- rolloutdiff/trajectories.py +236 -0
- rolloutdiff/watch-compatibility.json +13 -0
- rolloutdiff/watch.py +742 -0
- rolloutdiff-0.1.0.dist-info/METADATA +158 -0
- rolloutdiff-0.1.0.dist-info/RECORD +62 -0
- rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
- rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
- rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
- rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
rolloutdiff/events.py
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
"""Heuristic associations between sequential feature alerts."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from numbers import Integral
|
|
7
|
+
|
|
8
|
+
import numpy as np
|
|
9
|
+
import pandas as pd
|
|
10
|
+
|
|
11
|
+
from ._feature_values import _FeatureValueIndex
|
|
12
|
+
from .features import FeatureTables
|
|
13
|
+
from .schema import validate_rollouts
|
|
14
|
+
from .series import _require_columns, _validate_features
|
|
15
|
+
|
|
16
|
+
_EVENT_COLUMNS = ["event_id", "run", "onset_start", "onset_end", "n_members", "members"]
|
|
17
|
+
_ALERT_COLUMNS = ["run", "feature", "kind", "direction", "onset_step", "alert_step"]
|
|
18
|
+
_MEMBER_COLUMNS = ["event_id", *_ALERT_COLUMNS, "status"]
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _event_table(rows=()) -> pd.DataFrame:
|
|
22
|
+
result = pd.DataFrame(rows, columns=_EVENT_COLUMNS, dtype=object)
|
|
23
|
+
result["n_members"] = result.n_members.astype("int64")
|
|
24
|
+
return result
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _member_table(rows=()) -> pd.DataFrame:
|
|
28
|
+
return pd.DataFrame(rows, columns=_MEMBER_COLUMNS, dtype=object)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass(frozen=True)
|
|
32
|
+
class EventGrouping:
|
|
33
|
+
events: pd.DataFrame
|
|
34
|
+
members: pd.DataFrame
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def group_events(
|
|
38
|
+
rollouts: pd.DataFrame, features: FeatureTables, alerts: pd.DataFrame
|
|
39
|
+
) -> EventGrouping:
|
|
40
|
+
"""Group rising binary alerts by deterministic first fit within each run.
|
|
41
|
+
|
|
42
|
+
Process (onset_step, alert_step, feature) order; join the first created
|
|
43
|
+
event whose every member matches. Pairs require numeric onset difference
|
|
44
|
+
<= 3 and positive-rollout Jaccard >= 0.5 over observed rollouts in the
|
|
45
|
+
inclusive window from the earlier onset to the later alert. Sparse
|
|
46
|
+
absence is zero.
|
|
47
|
+
|
|
48
|
+
Unsupported alerts, missing anchors, or no positives in an alert's own
|
|
49
|
+
onset-to-alert window yield singletons. Event IDs are analysis-local;
|
|
50
|
+
grouping does not establish a shared cause.
|
|
51
|
+
"""
|
|
52
|
+
rollouts = validate_rollouts(rollouts)
|
|
53
|
+
_validate_features(rollouts, features)
|
|
54
|
+
return _group_events_prepared(_FeatureValueIndex(rollouts, features), alerts)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _validate_alerts(values: _FeatureValueIndex, alerts: pd.DataFrame) -> None:
|
|
58
|
+
_require_columns(alerts, _ALERT_COLUMNS, "sequential alerts")
|
|
59
|
+
if alerts.duplicated(["run", "feature"]).any():
|
|
60
|
+
raise ValueError("duplicate sequential alert for run/feature")
|
|
61
|
+
for name in ["run", "feature"]:
|
|
62
|
+
if not alerts[name].map(lambda v: isinstance(v, str) and bool(v.strip())).all():
|
|
63
|
+
raise ValueError(f"alert {name} must be a nonempty string")
|
|
64
|
+
if not alerts.direction.isin(["up", "down"]).all():
|
|
65
|
+
raise ValueError("alert direction must be up or down")
|
|
66
|
+
if not alerts.kind.isin(["binary", "continuous"]).all():
|
|
67
|
+
raise ValueError("alert kind must be binary or continuous")
|
|
68
|
+
for name in ["onset_step", "alert_step"]:
|
|
69
|
+
if (
|
|
70
|
+
not alerts[name]
|
|
71
|
+
.map(
|
|
72
|
+
lambda v: isinstance(v, Integral) and not isinstance(v, bool) and v >= 0
|
|
73
|
+
)
|
|
74
|
+
.all()
|
|
75
|
+
):
|
|
76
|
+
raise ValueError(f"{name} must be a nonnegative integer")
|
|
77
|
+
if (alerts.onset_step > alerts.alert_step).any():
|
|
78
|
+
raise ValueError("onset_step must not exceed alert_step")
|
|
79
|
+
for alert in alerts.itertuples(index=False):
|
|
80
|
+
if alert.run not in values.runs:
|
|
81
|
+
raise ValueError("alert run has no rollouts")
|
|
82
|
+
if alert.feature == "scalar:reward":
|
|
83
|
+
kind = "continuous"
|
|
84
|
+
elif alert.feature in values.metadata.index:
|
|
85
|
+
kind = values.metadata.at[alert.feature, "kind"]
|
|
86
|
+
else:
|
|
87
|
+
raise ValueError("alert feature is not in the retained catalog")
|
|
88
|
+
if alert.kind != kind:
|
|
89
|
+
raise ValueError("alert kind does not match the retained feature")
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _matches(left: dict, right: dict, steps: np.ndarray, masks: dict) -> bool:
|
|
93
|
+
if abs(left["onset_step"] - right["onset_step"]) > 3:
|
|
94
|
+
return False
|
|
95
|
+
start = min(left["onset_step"], right["onset_step"])
|
|
96
|
+
end = max(left["alert_step"], right["alert_step"])
|
|
97
|
+
low, high = steps.searchsorted(start), steps.searchsorted(end, side="right")
|
|
98
|
+
a, b = masks[left["feature"]][low:high], masks[right["feature"]][low:high]
|
|
99
|
+
union = np.count_nonzero(a | b)
|
|
100
|
+
return bool(union and 2 * np.count_nonzero(a & b) >= union)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _group_run(values: _FeatureValueIndex, run: str, alerts: pd.DataFrame) -> list:
|
|
104
|
+
frame = values.runs[run]
|
|
105
|
+
order = frame.sort_values(["step", "sample_id"]).index.to_numpy()
|
|
106
|
+
steps = frame.step.to_numpy()[order]
|
|
107
|
+
observed = set(steps)
|
|
108
|
+
groups, masks = [], {}
|
|
109
|
+
for alert in alerts[_ALERT_COLUMNS].to_dict("records"):
|
|
110
|
+
status = "eligible"
|
|
111
|
+
if alert["kind"] != "binary":
|
|
112
|
+
status = "unsupported_kind"
|
|
113
|
+
elif alert["direction"] != "up":
|
|
114
|
+
status = "unsupported_direction"
|
|
115
|
+
elif not {alert["onset_step"], alert["alert_step"]}.issubset(observed):
|
|
116
|
+
status = "unavailable_window"
|
|
117
|
+
else:
|
|
118
|
+
aligned, _ = values.values(run, alert["feature"])
|
|
119
|
+
# Every mask uses the same validated (run, sample_id) order.
|
|
120
|
+
present = aligned._value.to_numpy()[order] == 1
|
|
121
|
+
low = steps.searchsorted(alert["onset_step"])
|
|
122
|
+
high = steps.searchsorted(alert["alert_step"], side="right")
|
|
123
|
+
if present[low:high].any():
|
|
124
|
+
masks[alert["feature"]] = present
|
|
125
|
+
else:
|
|
126
|
+
status = "no_positive_rollouts"
|
|
127
|
+
alert["status"] = status
|
|
128
|
+
if status == "eligible":
|
|
129
|
+
for members in groups:
|
|
130
|
+
if members[0]["status"] == "eligible" and all(
|
|
131
|
+
_matches(alert, member, steps, masks) for member in members
|
|
132
|
+
):
|
|
133
|
+
members.append(alert)
|
|
134
|
+
break
|
|
135
|
+
else:
|
|
136
|
+
groups.append([alert])
|
|
137
|
+
else:
|
|
138
|
+
groups.append([alert])
|
|
139
|
+
return groups
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _group_events_prepared(
|
|
143
|
+
values: _FeatureValueIndex, alerts: pd.DataFrame
|
|
144
|
+
) -> EventGrouping:
|
|
145
|
+
_validate_alerts(values, alerts)
|
|
146
|
+
ordered = alerts.sort_values(["run", "onset_step", "alert_step", "feature"])
|
|
147
|
+
events, members = [], []
|
|
148
|
+
for run, run_alerts in ordered.groupby("run", sort=True):
|
|
149
|
+
for group in _group_run(values, run, run_alerts):
|
|
150
|
+
event_id = f"event-{len(events) + 1:06d}"
|
|
151
|
+
events.append(
|
|
152
|
+
{
|
|
153
|
+
"event_id": event_id,
|
|
154
|
+
"run": run,
|
|
155
|
+
"onset_start": min(row["onset_step"] for row in group),
|
|
156
|
+
"onset_end": max(row["onset_step"] for row in group),
|
|
157
|
+
"n_members": len(group),
|
|
158
|
+
"members": tuple((run, row["feature"]) for row in group),
|
|
159
|
+
}
|
|
160
|
+
)
|
|
161
|
+
for row in group:
|
|
162
|
+
members.append(
|
|
163
|
+
{
|
|
164
|
+
**row,
|
|
165
|
+
"event_id": event_id,
|
|
166
|
+
"status": (
|
|
167
|
+
"grouped" if len(group) > 1 else "singleton_no_match"
|
|
168
|
+
)
|
|
169
|
+
if row["status"] == "eligible"
|
|
170
|
+
else row["status"],
|
|
171
|
+
}
|
|
172
|
+
)
|
|
173
|
+
return EventGrouping(_event_table(events), _member_table(members))
|
rolloutdiff/evidence.py
ADDED
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
"""Persistence and within-group reward evidence."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from numbers import Real
|
|
7
|
+
|
|
8
|
+
import numpy as np
|
|
9
|
+
import pandas as pd
|
|
10
|
+
|
|
11
|
+
from ._feature_values import feature_values
|
|
12
|
+
from ._series_validation import validate_series
|
|
13
|
+
from .features import FeatureTables
|
|
14
|
+
from .prompt_mix import _window
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass(frozen=True)
|
|
18
|
+
class PersistenceEvidence:
|
|
19
|
+
run: str
|
|
20
|
+
feature: str
|
|
21
|
+
after_step: int
|
|
22
|
+
reference_mean: float
|
|
23
|
+
change: float
|
|
24
|
+
effect_fraction: float
|
|
25
|
+
threshold: float
|
|
26
|
+
direction: str
|
|
27
|
+
status: str
|
|
28
|
+
n_observations: int
|
|
29
|
+
n_evaluable: int
|
|
30
|
+
n_retained: int
|
|
31
|
+
n_unavailable: int
|
|
32
|
+
fraction_retained: float | None
|
|
33
|
+
coverage: float | None
|
|
34
|
+
steps: pd.DataFrame
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass(frozen=True)
|
|
38
|
+
class RewardAssociation:
|
|
39
|
+
run: str
|
|
40
|
+
feature: str
|
|
41
|
+
window: tuple[int, int]
|
|
42
|
+
status: str
|
|
43
|
+
mean_difference: float | None
|
|
44
|
+
n_groups: int
|
|
45
|
+
n_mixed_groups: int
|
|
46
|
+
n_rollouts: int
|
|
47
|
+
n_included_rollouts: int
|
|
48
|
+
groups: pd.DataFrame
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def persistence(
|
|
52
|
+
series: pd.DataFrame,
|
|
53
|
+
*,
|
|
54
|
+
run: str,
|
|
55
|
+
feature: str,
|
|
56
|
+
after_step: int,
|
|
57
|
+
reference_mean: float,
|
|
58
|
+
change: float,
|
|
59
|
+
effect_fraction: float = 0.5,
|
|
60
|
+
) -> PersistenceEvidence:
|
|
61
|
+
"""Measure signed effect retention strictly after an observed anchor.
|
|
62
|
+
|
|
63
|
+
Retained means sign(change)*(mean-reference_mean) >= effect_fraction*abs(change).
|
|
64
|
+
The returned threshold is an absolute mean cutoff. Fractions use evaluable
|
|
65
|
+
logged observations, excluding unknown SE and reporting reduced coverage.
|
|
66
|
+
No tail or no evaluable rows yields no fraction.
|
|
67
|
+
"""
|
|
68
|
+
frame = validate_series(series)
|
|
69
|
+
frame = frame.loc[frame.run.eq(run) & frame.feature.eq(feature)].copy()
|
|
70
|
+
return _persistence_prepared(
|
|
71
|
+
frame,
|
|
72
|
+
run=run,
|
|
73
|
+
feature=feature,
|
|
74
|
+
after_step=after_step,
|
|
75
|
+
reference_mean=reference_mean,
|
|
76
|
+
change=change,
|
|
77
|
+
effect_fraction=effect_fraction,
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _persistence_prepared(
|
|
82
|
+
frame: pd.DataFrame,
|
|
83
|
+
*,
|
|
84
|
+
run: str,
|
|
85
|
+
feature: str,
|
|
86
|
+
after_step: int,
|
|
87
|
+
reference_mean: float,
|
|
88
|
+
change: float,
|
|
89
|
+
effect_fraction: float = 0.5,
|
|
90
|
+
) -> PersistenceEvidence:
|
|
91
|
+
"""Measure persistence in one validated run/feature series."""
|
|
92
|
+
if type(after_step) is not int or after_step < 0:
|
|
93
|
+
raise ValueError("after_step must be a nonnegative integer")
|
|
94
|
+
for name, value in [
|
|
95
|
+
("reference_mean", reference_mean),
|
|
96
|
+
("change", change),
|
|
97
|
+
("effect_fraction", effect_fraction),
|
|
98
|
+
]:
|
|
99
|
+
if (
|
|
100
|
+
isinstance(value, bool)
|
|
101
|
+
or not isinstance(value, Real)
|
|
102
|
+
or not np.isfinite(value)
|
|
103
|
+
):
|
|
104
|
+
raise ValueError(f"{name} must be finite and numeric")
|
|
105
|
+
if change == 0 or not 0 < effect_fraction <= 1:
|
|
106
|
+
raise ValueError("change must be nonzero and effect_fraction in (0, 1]")
|
|
107
|
+
if frame.empty:
|
|
108
|
+
raise ValueError("run/feature has no series")
|
|
109
|
+
if not frame.step.eq(after_step).any():
|
|
110
|
+
raise ValueError("after_step must identify an observed step")
|
|
111
|
+
binary = frame.kind.iloc[0] == "binary"
|
|
112
|
+
with np.errstate(over="raise", invalid="raise"):
|
|
113
|
+
try:
|
|
114
|
+
endpoint = np.float64(reference_mean) + np.float64(change)
|
|
115
|
+
threshold = np.float64(reference_mean) + effect_fraction * np.float64(
|
|
116
|
+
change
|
|
117
|
+
)
|
|
118
|
+
except FloatingPointError:
|
|
119
|
+
raise ValueError(
|
|
120
|
+
"numerical overflow computing persistence threshold"
|
|
121
|
+
) from None
|
|
122
|
+
if binary and (
|
|
123
|
+
not 0 <= reference_mean <= 1
|
|
124
|
+
or not -4 * np.finfo(float).eps <= endpoint <= 1 + 4 * np.finfo(float).eps
|
|
125
|
+
):
|
|
126
|
+
raise ValueError("binary reference and full effect endpoint must be in [0, 1]")
|
|
127
|
+
target = effect_fraction * abs(change)
|
|
128
|
+
if target == 0:
|
|
129
|
+
raise ValueError("retained effect target underflow")
|
|
130
|
+
tail = frame.loc[frame.step.gt(after_step)].reset_index(drop=True)
|
|
131
|
+
valid = tail.se.notna()
|
|
132
|
+
retained = pd.Series(pd.NA, index=tail.index, dtype="boolean")
|
|
133
|
+
sign = 1 if change > 0 else -1
|
|
134
|
+
try:
|
|
135
|
+
with np.errstate(over="raise", invalid="raise"):
|
|
136
|
+
departure = sign * (tail.loc[valid, "mean"].to_numpy() - reference_mean)
|
|
137
|
+
# Relative roundoff only: an absolute epsilon can exceed a tiny
|
|
138
|
+
# positive target and falsely mark an unchanged value as retained.
|
|
139
|
+
tolerance = 8 * np.finfo(float).eps * np.maximum(np.abs(departure), target)
|
|
140
|
+
retained.loc[valid] = departure >= target - tolerance
|
|
141
|
+
except FloatingPointError:
|
|
142
|
+
raise ValueError("numerical overflow computing persistence") from None
|
|
143
|
+
tail["retained"] = retained
|
|
144
|
+
n, evaluable = len(tail), int(valid.sum())
|
|
145
|
+
n_retained = int(retained.fillna(False).sum())
|
|
146
|
+
status = (
|
|
147
|
+
"no_future_observations"
|
|
148
|
+
if not n
|
|
149
|
+
else "unavailable_uncertainty"
|
|
150
|
+
if not evaluable
|
|
151
|
+
else "partial"
|
|
152
|
+
if evaluable < n
|
|
153
|
+
else "available"
|
|
154
|
+
)
|
|
155
|
+
return PersistenceEvidence(
|
|
156
|
+
run,
|
|
157
|
+
feature,
|
|
158
|
+
after_step,
|
|
159
|
+
float(reference_mean),
|
|
160
|
+
float(change),
|
|
161
|
+
float(effect_fraction),
|
|
162
|
+
float(threshold),
|
|
163
|
+
"up" if sign > 0 else "down",
|
|
164
|
+
status,
|
|
165
|
+
n,
|
|
166
|
+
evaluable,
|
|
167
|
+
n_retained,
|
|
168
|
+
n - evaluable,
|
|
169
|
+
n_retained / evaluable if evaluable else None,
|
|
170
|
+
evaluable / n if n else None,
|
|
171
|
+
tail,
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
_GROUP_COLUMNS = [
|
|
176
|
+
"run",
|
|
177
|
+
"step",
|
|
178
|
+
"prompt_id",
|
|
179
|
+
"n_rollouts",
|
|
180
|
+
"n_present",
|
|
181
|
+
"n_absent",
|
|
182
|
+
"reward_present",
|
|
183
|
+
"reward_absent",
|
|
184
|
+
"difference",
|
|
185
|
+
]
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def reward_association(
|
|
189
|
+
rollouts: pd.DataFrame,
|
|
190
|
+
features: FeatureTables,
|
|
191
|
+
*,
|
|
192
|
+
run: str,
|
|
193
|
+
feature: str,
|
|
194
|
+
window: tuple[int, int],
|
|
195
|
+
) -> RewardAssociation:
|
|
196
|
+
"""Average present-minus-absent reward differences over mixed prompt/step groups.
|
|
197
|
+
|
|
198
|
+
Window bounds are inclusive. Mixed groups have equal weight regardless of
|
|
199
|
+
completion count; pure groups remain in the detail table only.
|
|
200
|
+
Continuous features return ``not_binary``.
|
|
201
|
+
"""
|
|
202
|
+
window = _window(window)
|
|
203
|
+
frame, kind = feature_values(rollouts, features, run=run, feature=feature)
|
|
204
|
+
return _reward_association_prepared(
|
|
205
|
+
frame, kind, run=run, feature=feature, window=window
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _reward_association_prepared(
|
|
210
|
+
frame: pd.DataFrame,
|
|
211
|
+
kind: str,
|
|
212
|
+
*,
|
|
213
|
+
run: str,
|
|
214
|
+
feature: str,
|
|
215
|
+
window: tuple[int, int],
|
|
216
|
+
) -> RewardAssociation:
|
|
217
|
+
"""Compute reward association from identity-aligned, validated values."""
|
|
218
|
+
window = _window(window)
|
|
219
|
+
frame = frame.loc[frame.step.between(*window)].copy()
|
|
220
|
+
if frame.empty:
|
|
221
|
+
raise ValueError("reward association window is empty")
|
|
222
|
+
keys = ["run", "step", "prompt_id"]
|
|
223
|
+
n_groups = len(frame[keys].drop_duplicates())
|
|
224
|
+
if kind != "binary":
|
|
225
|
+
return RewardAssociation(
|
|
226
|
+
run,
|
|
227
|
+
feature,
|
|
228
|
+
window,
|
|
229
|
+
"not_binary",
|
|
230
|
+
None,
|
|
231
|
+
n_groups,
|
|
232
|
+
0,
|
|
233
|
+
len(frame),
|
|
234
|
+
0,
|
|
235
|
+
pd.DataFrame(columns=_GROUP_COLUMNS),
|
|
236
|
+
)
|
|
237
|
+
present = frame["_value"].eq(1)
|
|
238
|
+
frame["_present_reward"] = frame.reward.where(present)
|
|
239
|
+
frame["_absent_reward"] = frame.reward.where(~present)
|
|
240
|
+
groups = (
|
|
241
|
+
frame.groupby(keys, sort=True)
|
|
242
|
+
.agg(
|
|
243
|
+
n_rollouts=("sample_id", "size"),
|
|
244
|
+
n_present=("_value", "sum"),
|
|
245
|
+
reward_present=("_present_reward", "mean"),
|
|
246
|
+
reward_absent=("_absent_reward", "mean"),
|
|
247
|
+
)
|
|
248
|
+
.reset_index()
|
|
249
|
+
)
|
|
250
|
+
groups["n_present"] = groups.n_present.astype("int64")
|
|
251
|
+
groups["n_absent"] = groups.n_rollouts - groups.n_present
|
|
252
|
+
for column, count in [
|
|
253
|
+
("reward_present", "n_present"),
|
|
254
|
+
("reward_absent", "n_absent"),
|
|
255
|
+
]:
|
|
256
|
+
if not np.isfinite(groups.loc[groups[count].gt(0), column]).all():
|
|
257
|
+
raise ValueError("numerical overflow computing within-group reward means")
|
|
258
|
+
mixed = groups.n_present.gt(0) & groups.n_absent.gt(0)
|
|
259
|
+
groups["difference"] = np.nan
|
|
260
|
+
mean = None
|
|
261
|
+
try:
|
|
262
|
+
with np.errstate(over="raise", invalid="raise"):
|
|
263
|
+
differences = (
|
|
264
|
+
groups.loc[mixed, "reward_present"].to_numpy()
|
|
265
|
+
- groups.loc[mixed, "reward_absent"].to_numpy()
|
|
266
|
+
)
|
|
267
|
+
if len(differences):
|
|
268
|
+
mean = float(differences.mean())
|
|
269
|
+
if not np.isfinite(mean):
|
|
270
|
+
raise FloatingPointError
|
|
271
|
+
groups.loc[mixed, "difference"] = differences
|
|
272
|
+
except FloatingPointError:
|
|
273
|
+
raise ValueError("numerical overflow computing reward association") from None
|
|
274
|
+
return RewardAssociation(
|
|
275
|
+
run,
|
|
276
|
+
feature,
|
|
277
|
+
window,
|
|
278
|
+
"available" if mixed.any() else "no_mixed_groups",
|
|
279
|
+
mean,
|
|
280
|
+
n_groups,
|
|
281
|
+
int(mixed.sum()),
|
|
282
|
+
len(frame),
|
|
283
|
+
int(groups.loc[mixed, "n_rollouts"].sum()),
|
|
284
|
+
groups[_GROUP_COLUMNS],
|
|
285
|
+
)
|
rolloutdiff/exemplars.py
ADDED
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
"""Rollout references for sequential alerts."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from fractions import Fraction
|
|
6
|
+
from numbers import Integral
|
|
7
|
+
|
|
8
|
+
import pandas as pd
|
|
9
|
+
|
|
10
|
+
from ._feature_values import _FeatureValueIndex
|
|
11
|
+
from .features import FeatureTables
|
|
12
|
+
from .schema import validate_rollouts
|
|
13
|
+
from .series import _require_columns, _validate_features
|
|
14
|
+
|
|
15
|
+
_COLUMNS = [
|
|
16
|
+
"run",
|
|
17
|
+
"feature",
|
|
18
|
+
"kind",
|
|
19
|
+
"direction",
|
|
20
|
+
"onset_step",
|
|
21
|
+
"alert_step",
|
|
22
|
+
"status",
|
|
23
|
+
"n_window_rollouts",
|
|
24
|
+
"n_candidates",
|
|
25
|
+
"reward_selection_status",
|
|
26
|
+
"selection_order",
|
|
27
|
+
"sample_id",
|
|
28
|
+
"step",
|
|
29
|
+
"prompt_id",
|
|
30
|
+
"feature_value",
|
|
31
|
+
"reward",
|
|
32
|
+
"reward_group_size",
|
|
33
|
+
"reward_group_mean",
|
|
34
|
+
"reward_deviation",
|
|
35
|
+
"reward_deviation_status",
|
|
36
|
+
"selection_reason",
|
|
37
|
+
]
|
|
38
|
+
_ALERT_COLUMNS = ["run", "feature", "kind", "direction", "onset_step", "alert_step"]
|
|
39
|
+
_ORDER = ["step", "run", "sample_id"]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _exemplar_table(rows=()) -> pd.DataFrame:
|
|
43
|
+
result = pd.DataFrame(rows, columns=_COLUMNS)
|
|
44
|
+
# Null status rows must not coerce exact integer identities through float64.
|
|
45
|
+
for name in [
|
|
46
|
+
"onset_step",
|
|
47
|
+
"alert_step",
|
|
48
|
+
"step",
|
|
49
|
+
"selection_order",
|
|
50
|
+
"n_window_rollouts",
|
|
51
|
+
"n_candidates",
|
|
52
|
+
"reward_group_size",
|
|
53
|
+
]:
|
|
54
|
+
result[name] = pd.Series([row.get(name) for row in rows], dtype=object)
|
|
55
|
+
for name in ["feature_value", "reward", "reward_group_mean", "reward_deviation"]:
|
|
56
|
+
result[name] = pd.Series([row.get(name) for row in rows], dtype=float)
|
|
57
|
+
return result
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def select_exemplars(
|
|
61
|
+
rollouts: pd.DataFrame,
|
|
62
|
+
features: FeatureTables,
|
|
63
|
+
alerts: pd.DataFrame,
|
|
64
|
+
) -> pd.DataFrame:
|
|
65
|
+
"""Return distinct rollout references for rising binary sequential alerts.
|
|
66
|
+
|
|
67
|
+
From onset through alert, select up to three positives: earliest, largest
|
|
68
|
+
positive within-(run, step, prompt_id) reward deviation among remaining
|
|
69
|
+
candidates, then prompt diversity. Ties use step and canonical identity.
|
|
70
|
+
Response text is excluded. Unsupported/unavailable alerts yield status rows
|
|
71
|
+
without sample_id; no alerts yield an empty table.
|
|
72
|
+
"""
|
|
73
|
+
rollouts = validate_rollouts(rollouts)
|
|
74
|
+
_validate_features(rollouts, features)
|
|
75
|
+
return _select_exemplars_prepared(_FeatureValueIndex(rollouts, features), alerts)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _reward_deviations(frame: pd.DataFrame) -> pd.DataFrame:
|
|
79
|
+
"""Center against all completions of the same prompt at the same step."""
|
|
80
|
+
if frame.empty:
|
|
81
|
+
raise ValueError("No objects to concatenate")
|
|
82
|
+
result = frame.copy()
|
|
83
|
+
sizes, means = [0] * len(frame), [0.0] * len(frame)
|
|
84
|
+
deviations, statuses = [None] * len(frame), [None] * len(frame)
|
|
85
|
+
reward_values = frame.reward.to_numpy()
|
|
86
|
+
groups = frame.groupby(["run", "step", "prompt_id"], sort=True).indices
|
|
87
|
+
for positions in groups.values():
|
|
88
|
+
rewards = [
|
|
89
|
+
Fraction.from_float(float(reward_values[position]))
|
|
90
|
+
for position in positions
|
|
91
|
+
]
|
|
92
|
+
# Center before rounding: large offsets must not erase small deviations.
|
|
93
|
+
exact_mean = sum(rewards, Fraction()) / len(positions)
|
|
94
|
+
group_mean = float(exact_mean)
|
|
95
|
+
for position, reward in zip(positions, rewards, strict=True):
|
|
96
|
+
sizes[position], means[position] = len(positions), group_mean
|
|
97
|
+
try:
|
|
98
|
+
deviation = float(reward - exact_mean)
|
|
99
|
+
except OverflowError:
|
|
100
|
+
deviation = None
|
|
101
|
+
if len(positions) < 2:
|
|
102
|
+
statuses[position] = "insufficient_group_size"
|
|
103
|
+
elif deviation is None or (deviation == 0 and reward != exact_mean):
|
|
104
|
+
statuses[position] = "unavailable_numeric"
|
|
105
|
+
else:
|
|
106
|
+
deviations[position] = deviation
|
|
107
|
+
statuses[position] = "available"
|
|
108
|
+
result["reward_group_size"] = sizes
|
|
109
|
+
result["reward_group_mean"] = means
|
|
110
|
+
result["reward_deviation"] = pd.Series(deviations, dtype=float).to_numpy()
|
|
111
|
+
result["reward_deviation_status"] = statuses
|
|
112
|
+
return result.sort_values(_ORDER).reset_index(drop=True)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _choose(frame: pd.DataFrame, common: dict) -> list[dict]:
|
|
116
|
+
candidates = frame.loc[frame._value.eq(1)].reset_index(drop=True)
|
|
117
|
+
common["n_candidates"] = len(candidates)
|
|
118
|
+
if candidates.empty:
|
|
119
|
+
return [
|
|
120
|
+
{
|
|
121
|
+
**common,
|
|
122
|
+
"status": "no_positive_rollouts",
|
|
123
|
+
"reward_selection_status": "no_candidates",
|
|
124
|
+
}
|
|
125
|
+
]
|
|
126
|
+
|
|
127
|
+
chosen = [(0, "earliest_occurrence")]
|
|
128
|
+
remaining = candidates.iloc[1:]
|
|
129
|
+
comparable = remaining.loc[remaining.reward_deviation_status.eq("available")]
|
|
130
|
+
positive = comparable.loc[comparable.reward_deviation.gt(0)]
|
|
131
|
+
if remaining.empty:
|
|
132
|
+
reward_status = "no_remaining_candidates"
|
|
133
|
+
elif comparable.empty:
|
|
134
|
+
reward_status = "no_comparable_candidates"
|
|
135
|
+
elif positive.empty:
|
|
136
|
+
reward_status = "no_positive_deviation"
|
|
137
|
+
else:
|
|
138
|
+
reward_status = "selected"
|
|
139
|
+
index = positive.sort_values(
|
|
140
|
+
["reward_deviation", *_ORDER], ascending=[False, True, True, True]
|
|
141
|
+
).index[0]
|
|
142
|
+
chosen.append((index, "high_within_prompt_reward_deviation"))
|
|
143
|
+
|
|
144
|
+
while len(chosen) < min(3, len(candidates)):
|
|
145
|
+
indices = [index for index, _ in chosen]
|
|
146
|
+
remaining = candidates.drop(index=indices)
|
|
147
|
+
new_prompts = remaining.loc[
|
|
148
|
+
~remaining.prompt_id.isin(candidates.loc[indices, "prompt_id"])
|
|
149
|
+
]
|
|
150
|
+
if new_prompts.empty:
|
|
151
|
+
chosen.append((remaining.index[0], "earliest_remaining"))
|
|
152
|
+
else:
|
|
153
|
+
chosen.append((new_prompts.index[0], "prompt_diversity"))
|
|
154
|
+
|
|
155
|
+
rows = []
|
|
156
|
+
for order, (index, reason) in enumerate(chosen, start=1):
|
|
157
|
+
candidate = candidates.loc[index]
|
|
158
|
+
rows.append(
|
|
159
|
+
{
|
|
160
|
+
**common,
|
|
161
|
+
"status": "selected",
|
|
162
|
+
"selection_order": order,
|
|
163
|
+
"reward_selection_status": reward_status,
|
|
164
|
+
**{
|
|
165
|
+
name: candidate[name]
|
|
166
|
+
for name in [
|
|
167
|
+
"sample_id",
|
|
168
|
+
"step",
|
|
169
|
+
"prompt_id",
|
|
170
|
+
"reward",
|
|
171
|
+
"reward_group_size",
|
|
172
|
+
"reward_group_mean",
|
|
173
|
+
"reward_deviation",
|
|
174
|
+
"reward_deviation_status",
|
|
175
|
+
]
|
|
176
|
+
},
|
|
177
|
+
"feature_value": candidate["_value"],
|
|
178
|
+
"selection_reason": reason,
|
|
179
|
+
}
|
|
180
|
+
)
|
|
181
|
+
return rows
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _select_exemplars_prepared(
|
|
185
|
+
values: _FeatureValueIndex, alerts: pd.DataFrame
|
|
186
|
+
) -> pd.DataFrame:
|
|
187
|
+
_require_columns(alerts, _ALERT_COLUMNS, "sequential alerts")
|
|
188
|
+
if alerts.duplicated(["run", "feature"]).any():
|
|
189
|
+
raise ValueError("duplicate sequential alert for run/feature")
|
|
190
|
+
for name in ["run", "feature"]:
|
|
191
|
+
if not alerts[name].map(lambda v: isinstance(v, str) and bool(v.strip())).all():
|
|
192
|
+
raise ValueError(f"alert {name} must be a nonempty string")
|
|
193
|
+
if not alerts.direction.isin(["up", "down"]).all():
|
|
194
|
+
raise ValueError("alert direction must be up or down")
|
|
195
|
+
if not alerts.kind.isin(["binary", "continuous"]).all():
|
|
196
|
+
raise ValueError("alert kind must be binary or continuous")
|
|
197
|
+
for name in ["onset_step", "alert_step"]:
|
|
198
|
+
if (
|
|
199
|
+
not alerts[name]
|
|
200
|
+
.map(
|
|
201
|
+
lambda v: isinstance(v, Integral) and not isinstance(v, bool) and v >= 0
|
|
202
|
+
)
|
|
203
|
+
.all()
|
|
204
|
+
):
|
|
205
|
+
raise ValueError(f"{name} must be a nonnegative integer")
|
|
206
|
+
if (alerts.onset_step > alerts.alert_step).any():
|
|
207
|
+
raise ValueError("onset_step must not exceed alert_step")
|
|
208
|
+
|
|
209
|
+
rows = []
|
|
210
|
+
for alert in alerts.sort_values(["run", "feature"]).itertuples(index=False):
|
|
211
|
+
if alert.run not in values.runs:
|
|
212
|
+
raise ValueError("alert run has no rollouts")
|
|
213
|
+
if (
|
|
214
|
+
alert.feature != "scalar:reward"
|
|
215
|
+
and alert.feature not in values.metadata.index
|
|
216
|
+
):
|
|
217
|
+
raise ValueError("alert feature is not in the retained catalog")
|
|
218
|
+
frame, kind = values.values(alert.run, alert.feature)
|
|
219
|
+
if alert.kind != kind:
|
|
220
|
+
raise ValueError("alert kind does not match the retained feature")
|
|
221
|
+
window = frame.loc[frame.step.between(alert.onset_step, alert.alert_step)]
|
|
222
|
+
common = {
|
|
223
|
+
"run": alert.run,
|
|
224
|
+
"feature": alert.feature,
|
|
225
|
+
"kind": kind,
|
|
226
|
+
"direction": alert.direction,
|
|
227
|
+
"onset_step": int(alert.onset_step),
|
|
228
|
+
"alert_step": int(alert.alert_step),
|
|
229
|
+
"n_window_rollouts": len(window),
|
|
230
|
+
}
|
|
231
|
+
if kind != "binary":
|
|
232
|
+
rows.append({**common, "status": "unsupported_kind"})
|
|
233
|
+
elif alert.direction != "up":
|
|
234
|
+
rows.append({**common, "status": "unsupported_direction"})
|
|
235
|
+
elif not {alert.onset_step, alert.alert_step}.issubset(set(window.step)):
|
|
236
|
+
rows.append({**common, "status": "unavailable_window"})
|
|
237
|
+
else:
|
|
238
|
+
ordered = window.sort_values(_ORDER).reset_index(drop=True)
|
|
239
|
+
rows.extend(_choose(_reward_deviations(ordered), common))
|
|
240
|
+
return _exemplar_table(rows)
|