rolloutdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rolloutdiff/__init__.py +130 -0
- rolloutdiff/_aria_replay.py +346 -0
- rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
- rolloutdiff/_dependence.py +195 -0
- rolloutdiff/_feature_values.py +79 -0
- rolloutdiff/_replay_container.py +568 -0
- rolloutdiff/_report_validation.py +758 -0
- rolloutdiff/_retrospective_policy.py +33 -0
- rolloutdiff/_sampling.py +127 -0
- rolloutdiff/_series_validation.py +67 -0
- rolloutdiff/_watch_report.py +436 -0
- rolloutdiff/_watch_storage.py +800 -0
- rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
- rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
- rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
- rolloutdiff/_web/assets/third-party-notices.txt +112 -0
- rolloutdiff/_web/index.html +14 -0
- rolloutdiff/adapters.py +264 -0
- rolloutdiff/app.py +956 -0
- rolloutdiff/app_evidence.py +678 -0
- rolloutdiff/benchmark_reference.py +458 -0
- rolloutdiff/cli.py +1292 -0
- rolloutdiff/comparison.py +572 -0
- rolloutdiff/comparison_report.py +982 -0
- rolloutdiff/corroboration.py +287 -0
- rolloutdiff/detect.py +307 -0
- rolloutdiff/events.py +173 -0
- rolloutdiff/evidence.py +285 -0
- rolloutdiff/exemplars.py +240 -0
- rolloutdiff/features.py +823 -0
- rolloutdiff/html_report.py +726 -0
- rolloutdiff/inspect_logs.py +383 -0
- rolloutdiff/inspection.py +207 -0
- rolloutdiff/integrations.py +552 -0
- rolloutdiff/investigation.py +1146 -0
- rolloutdiff/notifications.py +406 -0
- rolloutdiff/pipeline.py +809 -0
- rolloutdiff/producer.py +663 -0
- rolloutdiff/prompt_mix.py +258 -0
- rolloutdiff/ranking.py +388 -0
- rolloutdiff/replay.py +589 -0
- rolloutdiff/replication.py +572 -0
- rolloutdiff/report.py +838 -0
- rolloutdiff/retrospective.py +386 -0
- rolloutdiff/schema.py +109 -0
- rolloutdiff/sensitivity.py +202 -0
- rolloutdiff/series.py +430 -0
- rolloutdiff/tool_features.py +422 -0
- rolloutdiff/trajectories.py +236 -0
- rolloutdiff/watch-compatibility.json +13 -0
- rolloutdiff/watch.py +742 -0
- rolloutdiff-0.1.0.dist-info/METADATA +158 -0
- rolloutdiff-0.1.0.dist-info/RECORD +62 -0
- rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
- rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
- rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
- rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,287 @@
|
|
|
1
|
+
"""Bounded event-window evidence for a sequential alert in sampling-unit halves."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
import pandas as pd
|
|
9
|
+
|
|
10
|
+
from .detect import DetectionResult
|
|
11
|
+
from .features import FeatureTables
|
|
12
|
+
from .replication import _detector_parameters, _fixed_baseline_scores, _prepare_halves
|
|
13
|
+
|
|
14
|
+
_HALF_COLUMNS = [
|
|
15
|
+
"half",
|
|
16
|
+
"status",
|
|
17
|
+
"reason",
|
|
18
|
+
"n_prompts",
|
|
19
|
+
"n_sampling_units",
|
|
20
|
+
"n_rollouts",
|
|
21
|
+
"baseline_expected",
|
|
22
|
+
"baseline_required",
|
|
23
|
+
"baseline_used",
|
|
24
|
+
"baseline_used_steps",
|
|
25
|
+
"baseline_excluded_steps",
|
|
26
|
+
"baseline_mean",
|
|
27
|
+
"baseline_sd",
|
|
28
|
+
"baseline_se_floor",
|
|
29
|
+
"scale",
|
|
30
|
+
"numerical_floor_dominant",
|
|
31
|
+
"window_expected",
|
|
32
|
+
"window_observed",
|
|
33
|
+
"window_available",
|
|
34
|
+
"n_scored",
|
|
35
|
+
"n_unscorable",
|
|
36
|
+
"future_steps",
|
|
37
|
+
"missing_steps",
|
|
38
|
+
"unavailable_se_steps",
|
|
39
|
+
"match_start_step",
|
|
40
|
+
"match_end_step",
|
|
41
|
+
"opposite_start_step",
|
|
42
|
+
"opposite_end_step",
|
|
43
|
+
"mixed_direction",
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass(frozen=True)
|
|
48
|
+
class EventWindowEvidence:
|
|
49
|
+
run: str
|
|
50
|
+
feature: str
|
|
51
|
+
status: str
|
|
52
|
+
corroborates: bool | None
|
|
53
|
+
reason: str
|
|
54
|
+
full: DetectionResult
|
|
55
|
+
halves: pd.DataFrame
|
|
56
|
+
assignments: pd.DataFrame
|
|
57
|
+
radius: int
|
|
58
|
+
window_start_step: int | None
|
|
59
|
+
window_end_step: int | None
|
|
60
|
+
horizon_complete: bool
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _validate_radius(radius: int) -> None:
|
|
64
|
+
if type(radius) is not int or radius < 0:
|
|
65
|
+
raise ValueError("radius must be a nonnegative integer")
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def event_window_corroboration(
|
|
69
|
+
rollouts: pd.DataFrame,
|
|
70
|
+
features: FeatureTables,
|
|
71
|
+
*,
|
|
72
|
+
run: str,
|
|
73
|
+
feature: str,
|
|
74
|
+
baseline_steps: int = 15,
|
|
75
|
+
z_threshold: float = 4.0,
|
|
76
|
+
sustain: int = 3,
|
|
77
|
+
min_prevalence_change: float = 0.10,
|
|
78
|
+
numerical_floor: float = 1e-9,
|
|
79
|
+
radius: int = 10,
|
|
80
|
+
sampling_units: pd.DataFrame | None = None,
|
|
81
|
+
) -> EventWindowEvidence:
|
|
82
|
+
"""Find same-direction sustained witnesses near the full sequential alert.
|
|
83
|
+
|
|
84
|
+
Halves use fixed-window replication rules within ``radius`` full-grid
|
|
85
|
+
positions of the alert, after the baseline; streaks cannot span missing
|
|
86
|
+
rows or SE. A matching witness in each half corroborates. A half contradicts
|
|
87
|
+
only when its window is complete, fully scorable, and long enough for a
|
|
88
|
+
streak, and one contradicting half decides the result; anything else is
|
|
89
|
+
unavailable.
|
|
90
|
+
"""
|
|
91
|
+
_validate_radius(radius)
|
|
92
|
+
parameters = {
|
|
93
|
+
"baseline_steps": baseline_steps,
|
|
94
|
+
"z_threshold": z_threshold,
|
|
95
|
+
"sustain": sustain,
|
|
96
|
+
"min_prevalence_change": min_prevalence_change,
|
|
97
|
+
"numerical_floor": numerical_floor,
|
|
98
|
+
}
|
|
99
|
+
full, series, schedules, assignments = _prepare_halves(
|
|
100
|
+
rollouts,
|
|
101
|
+
features,
|
|
102
|
+
run=run,
|
|
103
|
+
feature=feature,
|
|
104
|
+
sampling_units=sampling_units,
|
|
105
|
+
parameters=parameters,
|
|
106
|
+
)
|
|
107
|
+
return _corroboration_prepared(
|
|
108
|
+
full,
|
|
109
|
+
series,
|
|
110
|
+
schedules,
|
|
111
|
+
assignments,
|
|
112
|
+
run=run,
|
|
113
|
+
feature=feature,
|
|
114
|
+
radius=radius,
|
|
115
|
+
parameters=parameters,
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _corroboration_prepared(
|
|
120
|
+
full, half_series, half_schedules, assignments, *, run, feature, radius, parameters
|
|
121
|
+
) -> EventWindowEvidence:
|
|
122
|
+
_validate_radius(radius)
|
|
123
|
+
if full.alerts.empty:
|
|
124
|
+
reason = (
|
|
125
|
+
"no_full_alert" if full.diagnostics.n_scored.sum() else "full_unscorable"
|
|
126
|
+
)
|
|
127
|
+
return EventWindowEvidence(
|
|
128
|
+
run,
|
|
129
|
+
feature,
|
|
130
|
+
"unavailable",
|
|
131
|
+
None,
|
|
132
|
+
reason,
|
|
133
|
+
full,
|
|
134
|
+
pd.DataFrame(columns=_HALF_COLUMNS),
|
|
135
|
+
assignments,
|
|
136
|
+
radius,
|
|
137
|
+
None,
|
|
138
|
+
None,
|
|
139
|
+
False,
|
|
140
|
+
)
|
|
141
|
+
resolved = _detector_parameters(parameters)
|
|
142
|
+
steps = full.scores.step.tolist()
|
|
143
|
+
alert = full.alerts.iloc[0]
|
|
144
|
+
position = steps.index(alert.alert_step)
|
|
145
|
+
start = max(resolved["baseline_steps"], position - radius)
|
|
146
|
+
end = position + radius
|
|
147
|
+
window = steps[start : end + 1]
|
|
148
|
+
expected = end - start + 1
|
|
149
|
+
horizon_complete = end < len(steps)
|
|
150
|
+
direction = 1 if alert.direction == "up" else -1
|
|
151
|
+
rows = []
|
|
152
|
+
for half in (0, 1):
|
|
153
|
+
rows.append(
|
|
154
|
+
_corroborate_half(
|
|
155
|
+
half,
|
|
156
|
+
half_series[half],
|
|
157
|
+
half_schedules[half],
|
|
158
|
+
steps,
|
|
159
|
+
window,
|
|
160
|
+
expected,
|
|
161
|
+
resolved,
|
|
162
|
+
direction,
|
|
163
|
+
horizon_complete,
|
|
164
|
+
)
|
|
165
|
+
)
|
|
166
|
+
halves = pd.DataFrame(rows, columns=_HALF_COLUMNS)
|
|
167
|
+
for name in (
|
|
168
|
+
"match_start_step",
|
|
169
|
+
"match_end_step",
|
|
170
|
+
"opposite_start_step",
|
|
171
|
+
"opposite_end_step",
|
|
172
|
+
):
|
|
173
|
+
halves[name] = pd.Series([row[name] for row in rows], dtype=object)
|
|
174
|
+
if halves.status.eq("contradicted").any():
|
|
175
|
+
status, value, reason = (
|
|
176
|
+
"contradicted",
|
|
177
|
+
False,
|
|
178
|
+
"complete_half_without_matching_streak",
|
|
179
|
+
)
|
|
180
|
+
elif halves.status.eq("corroborated").all():
|
|
181
|
+
status, value, reason = (
|
|
182
|
+
"corroborated",
|
|
183
|
+
True,
|
|
184
|
+
"both_halves_have_matching_streaks",
|
|
185
|
+
)
|
|
186
|
+
else:
|
|
187
|
+
status, value, reason = "unavailable", None, "insufficient_half_evidence"
|
|
188
|
+
return EventWindowEvidence(
|
|
189
|
+
run,
|
|
190
|
+
feature,
|
|
191
|
+
status,
|
|
192
|
+
value,
|
|
193
|
+
reason,
|
|
194
|
+
full,
|
|
195
|
+
halves,
|
|
196
|
+
assignments,
|
|
197
|
+
radius,
|
|
198
|
+
int(window[0]),
|
|
199
|
+
int(window[-1]),
|
|
200
|
+
horizon_complete,
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _window_streaks(
|
|
205
|
+
signs: dict, window: list[int], sustain: int, direction: int
|
|
206
|
+
) -> dict:
|
|
207
|
+
witnesses = {
|
|
208
|
+
"match_start_step": None,
|
|
209
|
+
"match_end_step": None,
|
|
210
|
+
"opposite_start_step": None,
|
|
211
|
+
"opposite_end_step": None,
|
|
212
|
+
}
|
|
213
|
+
streak, previous = 0, 0
|
|
214
|
+
for index, step in enumerate(window):
|
|
215
|
+
sign = signs.get(step, 0)
|
|
216
|
+
streak = streak + 1 if sign and sign == previous else int(bool(sign))
|
|
217
|
+
previous = sign
|
|
218
|
+
if streak == sustain:
|
|
219
|
+
prefix = "match" if sign == direction else "opposite"
|
|
220
|
+
if witnesses[prefix + "_end_step"] is None:
|
|
221
|
+
witnesses[prefix + "_start_step"] = int(window[index - streak + 1])
|
|
222
|
+
witnesses[prefix + "_end_step"] = int(step)
|
|
223
|
+
return witnesses
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _corroborate_half(
|
|
227
|
+
half,
|
|
228
|
+
series,
|
|
229
|
+
schedule,
|
|
230
|
+
steps,
|
|
231
|
+
window,
|
|
232
|
+
expected,
|
|
233
|
+
resolved,
|
|
234
|
+
direction,
|
|
235
|
+
horizon_complete,
|
|
236
|
+
):
|
|
237
|
+
coverage, result = _fixed_baseline_scores(
|
|
238
|
+
series, steps[: resolved["baseline_steps"]] + window, resolved
|
|
239
|
+
)
|
|
240
|
+
schedule = schedule.loc[schedule.step.isin(window)]
|
|
241
|
+
observed = set(schedule.step)
|
|
242
|
+
row = {name: coverage[name] for name in _HALF_COLUMNS if name in coverage}
|
|
243
|
+
row.update(
|
|
244
|
+
half=half,
|
|
245
|
+
status="unavailable",
|
|
246
|
+
reason="insufficient_fixed_baseline",
|
|
247
|
+
n_prompts=int(schedule.prompt_id.nunique()),
|
|
248
|
+
n_sampling_units=int(schedule.sampling_unit_id.nunique()),
|
|
249
|
+
n_rollouts=int(schedule.n_rollouts.sum()),
|
|
250
|
+
window_expected=expected,
|
|
251
|
+
window_observed=len(observed),
|
|
252
|
+
window_available=coverage["monitoring_available"],
|
|
253
|
+
n_unscorable=expected - coverage["n_scored"],
|
|
254
|
+
future_steps=expected - len(window),
|
|
255
|
+
missing_steps=[int(step) for step in window if step not in observed],
|
|
256
|
+
unavailable_se_steps=[
|
|
257
|
+
step for step in coverage["unavailable_se_steps"] if step in window
|
|
258
|
+
],
|
|
259
|
+
match_start_step=None,
|
|
260
|
+
match_end_step=None,
|
|
261
|
+
opposite_start_step=None,
|
|
262
|
+
opposite_end_step=None,
|
|
263
|
+
mixed_direction=False,
|
|
264
|
+
)
|
|
265
|
+
if result is not None:
|
|
266
|
+
# Use finite-SE point directions on the full event grid; compressed
|
|
267
|
+
# detector streaks and alerts do not preserve gaps.
|
|
268
|
+
signs = {
|
|
269
|
+
int(point.step): int(point.direction)
|
|
270
|
+
for point in result.scores.loc[result.scores.step.isin(window)].itertuples()
|
|
271
|
+
if np.isfinite(point.se)
|
|
272
|
+
}
|
|
273
|
+
row.update(_window_streaks(signs, window, resolved["sustain"], direction))
|
|
274
|
+
row["mixed_direction"] = (
|
|
275
|
+
row["match_end_step"] is not None and row["opposite_end_step"] is not None
|
|
276
|
+
)
|
|
277
|
+
if row["match_end_step"] is not None:
|
|
278
|
+
row.update(status="corroborated", reason="observed_matching_streak")
|
|
279
|
+
elif expected < resolved["sustain"]:
|
|
280
|
+
row["reason"] = "insufficient_window_observations"
|
|
281
|
+
elif horizon_complete and row["n_scored"] == expected:
|
|
282
|
+
row.update(
|
|
283
|
+
status="contradicted", reason="no_matching_streak_in_complete_window"
|
|
284
|
+
)
|
|
285
|
+
else:
|
|
286
|
+
row["reason"] = "incomplete_window_evidence"
|
|
287
|
+
return row
|
rolloutdiff/detect.py
ADDED
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
"""Sequential sustained-z screening against a frozen baseline scale."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from numbers import Real
|
|
7
|
+
|
|
8
|
+
import numpy as np
|
|
9
|
+
import pandas as pd
|
|
10
|
+
|
|
11
|
+
from ._series_validation import validate_series
|
|
12
|
+
|
|
13
|
+
_ALERT_COLUMNS = [
|
|
14
|
+
"run",
|
|
15
|
+
"feature",
|
|
16
|
+
"tier",
|
|
17
|
+
"kind",
|
|
18
|
+
"onset_step",
|
|
19
|
+
"alert_step",
|
|
20
|
+
"direction",
|
|
21
|
+
"baseline_mean",
|
|
22
|
+
"value_at_alert",
|
|
23
|
+
"z_at_alert",
|
|
24
|
+
"delta_at_alert",
|
|
25
|
+
"scale",
|
|
26
|
+
"se_at_alert",
|
|
27
|
+
"n_prompts_at_alert",
|
|
28
|
+
"n_sampling_units_at_alert",
|
|
29
|
+
"n_rollouts_at_alert",
|
|
30
|
+
"numerical_floor_dominant",
|
|
31
|
+
]
|
|
32
|
+
_DIAGNOSTIC_COLUMNS = [
|
|
33
|
+
"run",
|
|
34
|
+
"feature",
|
|
35
|
+
"tier",
|
|
36
|
+
"kind",
|
|
37
|
+
"status",
|
|
38
|
+
"baseline_n",
|
|
39
|
+
"baseline_start",
|
|
40
|
+
"baseline_end",
|
|
41
|
+
"baseline_mean",
|
|
42
|
+
"baseline_sd",
|
|
43
|
+
"baseline_se_floor",
|
|
44
|
+
"numerical_floor",
|
|
45
|
+
"scale",
|
|
46
|
+
"scale_source",
|
|
47
|
+
"numerical_floor_dominant",
|
|
48
|
+
"n_scored",
|
|
49
|
+
"n_unscorable",
|
|
50
|
+
]
|
|
51
|
+
_SCREENING_COLUMNS = ["run", "n_series", "n_screened", "n_alerted", "n_unscorable"]
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass(frozen=True)
|
|
55
|
+
class DetectionResult:
|
|
56
|
+
alerts: pd.DataFrame
|
|
57
|
+
diagnostics: pd.DataFrame
|
|
58
|
+
scores: pd.DataFrame
|
|
59
|
+
screening: pd.DataFrame
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def detect_sequential(
|
|
63
|
+
series: pd.DataFrame,
|
|
64
|
+
*,
|
|
65
|
+
baseline_steps: int = 15,
|
|
66
|
+
z_threshold: float = 4.0,
|
|
67
|
+
sustain: int = 3,
|
|
68
|
+
min_prevalence_change: float = 0.10,
|
|
69
|
+
numerical_floor: float = 1e-9,
|
|
70
|
+
) -> DetectionResult:
|
|
71
|
+
"""Return the first sustained alert and all screening scores per series.
|
|
72
|
+
|
|
73
|
+
The first ``baseline_steps`` observed rows freeze the mean and the scale,
|
|
74
|
+
max(sample SD, median baseline SE, ``numerical_floor``). Binary series alert
|
|
75
|
+
only on rises of at least ``min_prevalence_change``. Missing baseline SE
|
|
76
|
+
makes a series unscorable; missing monitoring SE resets the streak. Streaks
|
|
77
|
+
count logged rows, ignoring step gaps.
|
|
78
|
+
"""
|
|
79
|
+
if type(baseline_steps) is not int or baseline_steps < 2:
|
|
80
|
+
raise ValueError("baseline_steps must be an integer of at least two")
|
|
81
|
+
if type(sustain) is not int or sustain < 1:
|
|
82
|
+
raise ValueError("sustain must be a positive integer")
|
|
83
|
+
for name, value in [
|
|
84
|
+
("z_threshold", z_threshold),
|
|
85
|
+
("numerical_floor", numerical_floor),
|
|
86
|
+
("min_prevalence_change", min_prevalence_change),
|
|
87
|
+
]:
|
|
88
|
+
if (
|
|
89
|
+
isinstance(value, bool)
|
|
90
|
+
or not isinstance(value, Real)
|
|
91
|
+
or not np.isfinite(value)
|
|
92
|
+
):
|
|
93
|
+
raise ValueError(f"{name} must be finite and numeric")
|
|
94
|
+
if z_threshold <= 0 or numerical_floor <= 0 or not 0 <= min_prevalence_change <= 1:
|
|
95
|
+
raise ValueError("invalid detector threshold or floor")
|
|
96
|
+
scores = validate_series(series)
|
|
97
|
+
scores["delta"] = np.nan
|
|
98
|
+
scores["z"] = np.nan
|
|
99
|
+
scores["direction"] = 0
|
|
100
|
+
scores["qualifies"] = False
|
|
101
|
+
scores["streak"] = 0
|
|
102
|
+
scores["status"] = "baseline"
|
|
103
|
+
alerts, diagnostics = [], []
|
|
104
|
+
for (run, feature), group in scores.groupby(["run", "feature"], sort=False):
|
|
105
|
+
group_alerts, diagnostic = _detect_group(
|
|
106
|
+
scores,
|
|
107
|
+
group,
|
|
108
|
+
run,
|
|
109
|
+
feature,
|
|
110
|
+
baseline_steps,
|
|
111
|
+
z_threshold,
|
|
112
|
+
sustain,
|
|
113
|
+
min_prevalence_change,
|
|
114
|
+
numerical_floor,
|
|
115
|
+
)
|
|
116
|
+
alerts.extend(group_alerts)
|
|
117
|
+
diagnostics.append(diagnostic)
|
|
118
|
+
|
|
119
|
+
diagnostic_table = pd.DataFrame(diagnostics, columns=_DIAGNOSTIC_COLUMNS)
|
|
120
|
+
alert_table = (
|
|
121
|
+
pd.DataFrame(alerts, columns=_ALERT_COLUMNS)
|
|
122
|
+
.sort_values(["run", "alert_step", "feature"])
|
|
123
|
+
.reset_index(drop=True)
|
|
124
|
+
)
|
|
125
|
+
if diagnostic_table.empty:
|
|
126
|
+
screening = pd.DataFrame(columns=_SCREENING_COLUMNS)
|
|
127
|
+
else:
|
|
128
|
+
screening = (
|
|
129
|
+
diagnostic_table.assign(
|
|
130
|
+
screened=diagnostic_table.n_scored.gt(0),
|
|
131
|
+
alerted=diagnostic_table.status.eq("alerted"),
|
|
132
|
+
)
|
|
133
|
+
.groupby("run", sort=True)
|
|
134
|
+
.agg(
|
|
135
|
+
n_series=("feature", "size"),
|
|
136
|
+
n_screened=("screened", "sum"),
|
|
137
|
+
n_alerted=("alerted", "sum"),
|
|
138
|
+
)
|
|
139
|
+
.reset_index()
|
|
140
|
+
)
|
|
141
|
+
screening["n_unscorable"] = screening.n_series - screening.n_screened
|
|
142
|
+
return DetectionResult(alert_table, diagnostic_table, scores, screening)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _monitoring_scores(
|
|
146
|
+
values, errors, mean, scale, kind, z_threshold, min_prevalence_change
|
|
147
|
+
):
|
|
148
|
+
valid = np.isfinite(errors)
|
|
149
|
+
try:
|
|
150
|
+
with np.errstate(over="raise", invalid="raise", divide="raise"):
|
|
151
|
+
delta = values - mean
|
|
152
|
+
z = np.full(len(delta), np.nan)
|
|
153
|
+
z[valid] = delta[valid] / scale
|
|
154
|
+
except FloatingPointError:
|
|
155
|
+
raise ValueError("numerical overflow computing screening scores") from None
|
|
156
|
+
if kind == "binary":
|
|
157
|
+
# Allow float64 roundoff so an exact minimum change qualifies.
|
|
158
|
+
effect_met = delta >= min_prevalence_change - 4 * np.finfo(float).eps
|
|
159
|
+
qualifies = valid & (z > z_threshold) & effect_met
|
|
160
|
+
else:
|
|
161
|
+
qualifies = valid & (np.abs(z) > z_threshold)
|
|
162
|
+
direction = np.where(qualifies, np.sign(np.nan_to_num(z, nan=0)), 0).astype(int)
|
|
163
|
+
return delta, z, qualifies, direction
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _alert_streaks(direction: np.ndarray, sustain: int):
|
|
167
|
+
streak, previous_direction, start = 0, 0, 0
|
|
168
|
+
streaks = np.zeros(len(direction), dtype=int)
|
|
169
|
+
first_alert = None
|
|
170
|
+
for offset, sign in enumerate(direction):
|
|
171
|
+
if sign == 0:
|
|
172
|
+
streak = 0
|
|
173
|
+
elif sign == previous_direction:
|
|
174
|
+
streak += 1
|
|
175
|
+
else:
|
|
176
|
+
streak, start = 1, offset
|
|
177
|
+
previous_direction = sign
|
|
178
|
+
streaks[offset] = streak
|
|
179
|
+
if streak >= sustain and first_alert is None:
|
|
180
|
+
first_alert = start, offset
|
|
181
|
+
return streaks, first_alert
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _detect_group(
|
|
185
|
+
scores,
|
|
186
|
+
group,
|
|
187
|
+
run,
|
|
188
|
+
feature,
|
|
189
|
+
baseline_steps,
|
|
190
|
+
z_threshold,
|
|
191
|
+
sustain,
|
|
192
|
+
min_prevalence_change,
|
|
193
|
+
numerical_floor,
|
|
194
|
+
):
|
|
195
|
+
indices = group.index.to_numpy()
|
|
196
|
+
steps = group.step.to_numpy()
|
|
197
|
+
x, errors = group["mean"].to_numpy(), group.se.to_numpy()
|
|
198
|
+
kind, tier = group.kind.iloc[0], group.tier.iloc[0]
|
|
199
|
+
n_base = min(baseline_steps, len(group))
|
|
200
|
+
common = {"run": run, "feature": feature, "tier": tier, "kind": kind}
|
|
201
|
+
diagnostic = {
|
|
202
|
+
**common,
|
|
203
|
+
"status": "insufficient_baseline",
|
|
204
|
+
"baseline_n": n_base,
|
|
205
|
+
"baseline_start": int(steps[0]),
|
|
206
|
+
"baseline_end": int(steps[n_base - 1]),
|
|
207
|
+
"baseline_mean": np.nan,
|
|
208
|
+
"baseline_sd": np.nan,
|
|
209
|
+
"baseline_se_floor": np.nan,
|
|
210
|
+
"numerical_floor": float(numerical_floor),
|
|
211
|
+
"scale": np.nan,
|
|
212
|
+
"scale_source": None,
|
|
213
|
+
"numerical_floor_dominant": False,
|
|
214
|
+
"n_scored": 0,
|
|
215
|
+
"n_unscorable": len(group) - n_base,
|
|
216
|
+
}
|
|
217
|
+
if len(group) < baseline_steps:
|
|
218
|
+
return [], diagnostic
|
|
219
|
+
baseline = _baseline_statistics(
|
|
220
|
+
x[:baseline_steps], errors[:baseline_steps], numerical_floor
|
|
221
|
+
)
|
|
222
|
+
diagnostic.update(baseline)
|
|
223
|
+
post_indices = indices[baseline_steps:]
|
|
224
|
+
if diagnostic["status"] == "insufficient_baseline_uncertainty":
|
|
225
|
+
scores.loc[post_indices, "status"] = diagnostic["status"]
|
|
226
|
+
return [], diagnostic
|
|
227
|
+
mean = baseline["baseline_mean"]
|
|
228
|
+
scale = baseline["scale"]
|
|
229
|
+
guard_dominates = baseline["numerical_floor_dominant"]
|
|
230
|
+
valid = np.isfinite(errors[baseline_steps:])
|
|
231
|
+
diagnostic.update(
|
|
232
|
+
n_scored=int(valid.sum()),
|
|
233
|
+
n_unscorable=int((~valid).sum()),
|
|
234
|
+
status="no_alert" if valid.any() else "no_scorable_observations",
|
|
235
|
+
)
|
|
236
|
+
delta, z, qualifies, direction = _monitoring_scores(
|
|
237
|
+
x[baseline_steps:],
|
|
238
|
+
errors[baseline_steps:],
|
|
239
|
+
mean,
|
|
240
|
+
scale,
|
|
241
|
+
kind,
|
|
242
|
+
z_threshold,
|
|
243
|
+
min_prevalence_change,
|
|
244
|
+
)
|
|
245
|
+
scores.loc[post_indices, "delta"] = delta
|
|
246
|
+
scores.loc[post_indices, "z"] = z
|
|
247
|
+
scores.loc[post_indices, "qualifies"] = qualifies
|
|
248
|
+
scores.loc[post_indices, "direction"] = direction
|
|
249
|
+
scores.loc[post_indices, "status"] = np.where(valid, "scored", "unavailable_se")
|
|
250
|
+
streaks, first_alert = _alert_streaks(direction, sustain)
|
|
251
|
+
alerts = []
|
|
252
|
+
if first_alert is not None:
|
|
253
|
+
start, offset = first_alert
|
|
254
|
+
row = group.iloc[baseline_steps + offset]
|
|
255
|
+
alerts.append(
|
|
256
|
+
{
|
|
257
|
+
**common,
|
|
258
|
+
"onset_step": int(steps[baseline_steps + start]),
|
|
259
|
+
"alert_step": int(row.step),
|
|
260
|
+
"direction": "up" if direction[offset] > 0 else "down",
|
|
261
|
+
"baseline_mean": mean,
|
|
262
|
+
"value_at_alert": float(row["mean"]),
|
|
263
|
+
"z_at_alert": float(z[offset]),
|
|
264
|
+
"delta_at_alert": float(delta[offset]),
|
|
265
|
+
"scale": scale,
|
|
266
|
+
"se_at_alert": float(row.se),
|
|
267
|
+
"n_prompts_at_alert": int(row.n_prompts),
|
|
268
|
+
"n_sampling_units_at_alert": int(row.n_sampling_units),
|
|
269
|
+
"n_rollouts_at_alert": int(row.n_rollouts),
|
|
270
|
+
"numerical_floor_dominant": guard_dominates,
|
|
271
|
+
}
|
|
272
|
+
)
|
|
273
|
+
diagnostic["status"] = "alerted"
|
|
274
|
+
scores.loc[post_indices, "streak"] = streaks
|
|
275
|
+
return alerts, diagnostic
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def _baseline_statistics(
|
|
279
|
+
values: np.ndarray, errors: np.ndarray, numerical_floor: float
|
|
280
|
+
) -> dict:
|
|
281
|
+
try:
|
|
282
|
+
with np.errstate(over="raise", invalid="raise", divide="raise"):
|
|
283
|
+
mean = float(values.mean())
|
|
284
|
+
sd = float(values.std(ddof=1))
|
|
285
|
+
except FloatingPointError:
|
|
286
|
+
raise ValueError("numerical overflow computing baseline statistics") from None
|
|
287
|
+
baseline = {"baseline_mean": mean, "baseline_sd": sd}
|
|
288
|
+
if not np.isfinite(errors).all():
|
|
289
|
+
return baseline | {"status": "insufficient_baseline_uncertainty"}
|
|
290
|
+
try:
|
|
291
|
+
with np.errstate(over="raise", invalid="raise"):
|
|
292
|
+
se_floor = float(np.median(errors))
|
|
293
|
+
except FloatingPointError:
|
|
294
|
+
raise ValueError("numerical overflow computing baseline SE floor") from None
|
|
295
|
+
scale = max(sd, se_floor, float(numerical_floor))
|
|
296
|
+
guard_dominates = max(sd, se_floor) <= numerical_floor
|
|
297
|
+
source = (
|
|
298
|
+
"numerical_floor"
|
|
299
|
+
if guard_dominates
|
|
300
|
+
else ("baseline_sd" if sd >= se_floor else "baseline_se")
|
|
301
|
+
)
|
|
302
|
+
return baseline | {
|
|
303
|
+
"baseline_se_floor": se_floor,
|
|
304
|
+
"scale": scale,
|
|
305
|
+
"scale_source": source,
|
|
306
|
+
"numerical_floor_dominant": guard_dominates,
|
|
307
|
+
}
|