rolloutdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rolloutdiff/__init__.py +130 -0
- rolloutdiff/_aria_replay.py +346 -0
- rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
- rolloutdiff/_dependence.py +195 -0
- rolloutdiff/_feature_values.py +79 -0
- rolloutdiff/_replay_container.py +568 -0
- rolloutdiff/_report_validation.py +758 -0
- rolloutdiff/_retrospective_policy.py +33 -0
- rolloutdiff/_sampling.py +127 -0
- rolloutdiff/_series_validation.py +67 -0
- rolloutdiff/_watch_report.py +436 -0
- rolloutdiff/_watch_storage.py +800 -0
- rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
- rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
- rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
- rolloutdiff/_web/assets/third-party-notices.txt +112 -0
- rolloutdiff/_web/index.html +14 -0
- rolloutdiff/adapters.py +264 -0
- rolloutdiff/app.py +956 -0
- rolloutdiff/app_evidence.py +678 -0
- rolloutdiff/benchmark_reference.py +458 -0
- rolloutdiff/cli.py +1292 -0
- rolloutdiff/comparison.py +572 -0
- rolloutdiff/comparison_report.py +982 -0
- rolloutdiff/corroboration.py +287 -0
- rolloutdiff/detect.py +307 -0
- rolloutdiff/events.py +173 -0
- rolloutdiff/evidence.py +285 -0
- rolloutdiff/exemplars.py +240 -0
- rolloutdiff/features.py +823 -0
- rolloutdiff/html_report.py +726 -0
- rolloutdiff/inspect_logs.py +383 -0
- rolloutdiff/inspection.py +207 -0
- rolloutdiff/integrations.py +552 -0
- rolloutdiff/investigation.py +1146 -0
- rolloutdiff/notifications.py +406 -0
- rolloutdiff/pipeline.py +809 -0
- rolloutdiff/producer.py +663 -0
- rolloutdiff/prompt_mix.py +258 -0
- rolloutdiff/ranking.py +388 -0
- rolloutdiff/replay.py +589 -0
- rolloutdiff/replication.py +572 -0
- rolloutdiff/report.py +838 -0
- rolloutdiff/retrospective.py +386 -0
- rolloutdiff/schema.py +109 -0
- rolloutdiff/sensitivity.py +202 -0
- rolloutdiff/series.py +430 -0
- rolloutdiff/tool_features.py +422 -0
- rolloutdiff/trajectories.py +236 -0
- rolloutdiff/watch-compatibility.json +13 -0
- rolloutdiff/watch.py +742 -0
- rolloutdiff-0.1.0.dist-info/METADATA +158 -0
- rolloutdiff-0.1.0.dist-info/RECORD +62 -0
- rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
- rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
- rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
- rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,572 @@
|
|
|
1
|
+
"""Deterministic sampling-unit split evidence for the first sequential alert."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import zlib
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from inspect import signature
|
|
8
|
+
|
|
9
|
+
import numpy as np
|
|
10
|
+
import pandas as pd
|
|
11
|
+
|
|
12
|
+
from ._feature_values import feature_values
|
|
13
|
+
from ._sampling import resolve_sampling_units, subset_sampling_units
|
|
14
|
+
from .detect import DetectionResult, detect_sequential
|
|
15
|
+
from .features import FeatureTables
|
|
16
|
+
from .series import aggregate
|
|
17
|
+
|
|
18
|
+
_HALF_COLUMNS = [
|
|
19
|
+
"half",
|
|
20
|
+
"status",
|
|
21
|
+
"alert_step",
|
|
22
|
+
"onset_step",
|
|
23
|
+
"direction",
|
|
24
|
+
"alert_position",
|
|
25
|
+
"n_prompts",
|
|
26
|
+
"n_sampling_units",
|
|
27
|
+
"n_rollouts",
|
|
28
|
+
"n_steps",
|
|
29
|
+
"n_scored",
|
|
30
|
+
"n_unscorable",
|
|
31
|
+
"missing_steps",
|
|
32
|
+
"reason",
|
|
33
|
+
"source_detect_status",
|
|
34
|
+
"baseline_mean",
|
|
35
|
+
"baseline_sd",
|
|
36
|
+
"baseline_se_floor",
|
|
37
|
+
"scale",
|
|
38
|
+
"numerical_floor_dominant",
|
|
39
|
+
]
|
|
40
|
+
_FIXED_COLUMNS = [
|
|
41
|
+
"baseline_expected",
|
|
42
|
+
"baseline_required",
|
|
43
|
+
"baseline_used",
|
|
44
|
+
"baseline_used_steps",
|
|
45
|
+
"baseline_excluded_steps",
|
|
46
|
+
"monitoring_expected",
|
|
47
|
+
"monitoring_available",
|
|
48
|
+
"unavailable_se_steps",
|
|
49
|
+
"first_possible_alert_step",
|
|
50
|
+
"observed_alert_step",
|
|
51
|
+
"observed_onset_step",
|
|
52
|
+
"observed_direction",
|
|
53
|
+
"first_alert_identified",
|
|
54
|
+
]
|
|
55
|
+
_ALIAS = "_evidence_value"
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@dataclass(frozen=True)
|
|
59
|
+
class ReplicationEvidence:
|
|
60
|
+
run: str
|
|
61
|
+
feature: str
|
|
62
|
+
status: str
|
|
63
|
+
replicates: bool | None
|
|
64
|
+
full: DetectionResult
|
|
65
|
+
halves: pd.DataFrame
|
|
66
|
+
assignments: pd.DataFrame
|
|
67
|
+
tolerance: int
|
|
68
|
+
horizon_complete: bool
|
|
69
|
+
cutoff_step: int | None
|
|
70
|
+
coverage_policy: str = "contiguous_prefix"
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _series(
|
|
74
|
+
frame: pd.DataFrame,
|
|
75
|
+
*,
|
|
76
|
+
feature: str,
|
|
77
|
+
kind: str,
|
|
78
|
+
tier: str,
|
|
79
|
+
sampling_units: pd.DataFrame,
|
|
80
|
+
) -> pd.DataFrame:
|
|
81
|
+
features = FeatureTables(
|
|
82
|
+
frame[["run", "sample_id", "_value"]].rename(columns={"_value": _ALIAS}),
|
|
83
|
+
pd.DataFrame(columns=["run", "sample_id", "feature"]),
|
|
84
|
+
pd.DataFrame([{"feature": _ALIAS, "tier": tier, "kind": kind}]),
|
|
85
|
+
)
|
|
86
|
+
result = aggregate(
|
|
87
|
+
frame.drop(columns="_value"), features, sampling_units=sampling_units
|
|
88
|
+
).series
|
|
89
|
+
return (
|
|
90
|
+
result.loc[result.feature.eq(_ALIAS)]
|
|
91
|
+
.assign(feature=feature)
|
|
92
|
+
.reset_index(drop=True)
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def split_half_replication(
|
|
97
|
+
rollouts: pd.DataFrame,
|
|
98
|
+
features: FeatureTables,
|
|
99
|
+
*,
|
|
100
|
+
run: str,
|
|
101
|
+
feature: str,
|
|
102
|
+
baseline_steps: int = 15,
|
|
103
|
+
z_threshold: float = 4.0,
|
|
104
|
+
sustain: int = 3,
|
|
105
|
+
min_prevalence_change: float = 0.10,
|
|
106
|
+
numerical_floor: float = 1e-9,
|
|
107
|
+
tolerance: int = 10,
|
|
108
|
+
sampling_units: pd.DataFrame | None = None,
|
|
109
|
+
coverage_policy: str = "contiguous_prefix",
|
|
110
|
+
) -> ReplicationEvidence:
|
|
111
|
+
"""Compare first full/half alerts using whole-unit CRC32 & 1 splits.
|
|
112
|
+
|
|
113
|
+
Replication needs all three first alerts in one direction within
|
|
114
|
+
``tolerance`` full-grid positions; observed disagreement fails regardless
|
|
115
|
+
of the other half. A scorable half without an alert stays
|
|
116
|
+
``pending`` until that horizon is complete, and an unavailable-SE row that
|
|
117
|
+
could hide an earlier compatible alert makes the half unavailable.
|
|
118
|
+
``contiguous_prefix`` stops a half at its first gap; ``fixed_window`` needs
|
|
119
|
+
max(2, ceil(``baseline_steps``/2)) finite-SE baseline rows and resumes after
|
|
120
|
+
gaps.
|
|
121
|
+
"""
|
|
122
|
+
_validate_coverage_policy(coverage_policy)
|
|
123
|
+
if type(tolerance) is not int or tolerance < 0:
|
|
124
|
+
raise ValueError("tolerance must be a nonnegative integer")
|
|
125
|
+
parameters = {
|
|
126
|
+
"baseline_steps": baseline_steps,
|
|
127
|
+
"z_threshold": z_threshold,
|
|
128
|
+
"sustain": sustain,
|
|
129
|
+
"min_prevalence_change": min_prevalence_change,
|
|
130
|
+
"numerical_floor": numerical_floor,
|
|
131
|
+
}
|
|
132
|
+
full, half_series, half_schedules, assignments = _prepare_halves(
|
|
133
|
+
rollouts,
|
|
134
|
+
features,
|
|
135
|
+
run=run,
|
|
136
|
+
feature=feature,
|
|
137
|
+
sampling_units=sampling_units,
|
|
138
|
+
parameters=parameters,
|
|
139
|
+
)
|
|
140
|
+
return _replication_prepared(
|
|
141
|
+
full,
|
|
142
|
+
half_series,
|
|
143
|
+
half_schedules,
|
|
144
|
+
assignments,
|
|
145
|
+
run=run,
|
|
146
|
+
feature=feature,
|
|
147
|
+
tolerance=tolerance,
|
|
148
|
+
parameters=parameters,
|
|
149
|
+
coverage_policy=coverage_policy,
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _prepare_halves(rollouts, features, *, run, feature, sampling_units, parameters):
|
|
154
|
+
units = resolve_sampling_units(rollouts, sampling_units)
|
|
155
|
+
frame, kind = feature_values(rollouts, features, run=run, feature=feature)
|
|
156
|
+
units = subset_sampling_units(units, frame)
|
|
157
|
+
tier = (
|
|
158
|
+
"structural"
|
|
159
|
+
if feature == "scalar:reward"
|
|
160
|
+
else features.catalog.set_index("feature").loc[feature, "tier"]
|
|
161
|
+
)
|
|
162
|
+
assignments = units.copy()
|
|
163
|
+
assignments["half"] = assignments.sampling_unit_id.map(
|
|
164
|
+
lambda unit: zlib.crc32(unit.encode("utf-8")) & 1
|
|
165
|
+
)
|
|
166
|
+
full = detect_sequential(
|
|
167
|
+
_series(frame, feature=feature, kind=kind, tier=tier, sampling_units=units),
|
|
168
|
+
**parameters,
|
|
169
|
+
)
|
|
170
|
+
half_series, half_schedules = {}, {}
|
|
171
|
+
for half in (0, 1):
|
|
172
|
+
prompts = assignments.loc[assignments.half.eq(half), "prompt_id"]
|
|
173
|
+
subset = frame.loc[frame.prompt_id.isin(prompts)]
|
|
174
|
+
half_units = subset_sampling_units(units, subset)
|
|
175
|
+
half_series[half] = _series(
|
|
176
|
+
subset, feature=feature, kind=kind, tier=tier, sampling_units=half_units
|
|
177
|
+
)
|
|
178
|
+
half_schedules[half] = (
|
|
179
|
+
subset.groupby(["step", "prompt_id"], sort=True)
|
|
180
|
+
.size()
|
|
181
|
+
.rename("n_rollouts")
|
|
182
|
+
.reset_index()
|
|
183
|
+
.merge(
|
|
184
|
+
half_units[["prompt_id", "sampling_unit_id"]],
|
|
185
|
+
on="prompt_id",
|
|
186
|
+
how="left",
|
|
187
|
+
validate="many_to_one",
|
|
188
|
+
)
|
|
189
|
+
)
|
|
190
|
+
return full, half_series, half_schedules, assignments
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _validate_coverage_policy(policy: str) -> None:
|
|
194
|
+
if policy not in ("contiguous_prefix", "fixed_window"):
|
|
195
|
+
raise ValueError("coverage_policy must be contiguous_prefix or fixed_window")
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _detector_parameters(parameters: dict) -> dict:
|
|
199
|
+
resolved = {
|
|
200
|
+
name: parameter.default
|
|
201
|
+
for name, parameter in signature(detect_sequential).parameters.items()
|
|
202
|
+
if name != "series"
|
|
203
|
+
}
|
|
204
|
+
return resolved | parameters
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _fixed_baseline_scores(series: pd.DataFrame, steps: list[int], parameters: dict):
|
|
208
|
+
"""Score a sparse fixed baseline; callers scan the full grid for streaks."""
|
|
209
|
+
baseline_steps = parameters["baseline_steps"]
|
|
210
|
+
baseline_grid, monitor_grid = steps[:baseline_steps], steps[baseline_steps:]
|
|
211
|
+
prepared = series.loc[series.step.isin(steps)]
|
|
212
|
+
# Empty aggregates carry an object-typed SE column; no unit means no evidence.
|
|
213
|
+
finite = np.isfinite(prepared.se.to_numpy(dtype=float))
|
|
214
|
+
baseline = prepared.loc[prepared.step.isin(baseline_grid) & finite]
|
|
215
|
+
monitoring = prepared.loc[prepared.step.isin(monitor_grid)]
|
|
216
|
+
used = set(baseline.step)
|
|
217
|
+
available = set(prepared.loc[finite, "step"])
|
|
218
|
+
coverage = {
|
|
219
|
+
"baseline_expected": baseline_steps,
|
|
220
|
+
"baseline_required": max(2, (baseline_steps + 1) // 2),
|
|
221
|
+
"baseline_used": len(baseline),
|
|
222
|
+
"baseline_used_steps": [int(step) for step in baseline_grid if step in used],
|
|
223
|
+
"baseline_excluded_steps": [
|
|
224
|
+
int(step) for step in baseline_grid if step not in used
|
|
225
|
+
],
|
|
226
|
+
"monitoring_expected": len(monitor_grid),
|
|
227
|
+
"monitoring_available": sum(step in available for step in monitor_grid),
|
|
228
|
+
"unavailable_se_steps": [int(step) for step in prepared.loc[~finite, "step"]],
|
|
229
|
+
"n_scored": 0,
|
|
230
|
+
"n_unscorable": len(monitor_grid),
|
|
231
|
+
}
|
|
232
|
+
for name in (
|
|
233
|
+
"baseline_mean",
|
|
234
|
+
"baseline_sd",
|
|
235
|
+
"baseline_se_floor",
|
|
236
|
+
"scale",
|
|
237
|
+
"numerical_floor_dominant",
|
|
238
|
+
):
|
|
239
|
+
coverage[name] = None
|
|
240
|
+
if len(baseline) < coverage["baseline_required"]:
|
|
241
|
+
return coverage, None
|
|
242
|
+
result = detect_sequential(
|
|
243
|
+
pd.concat([baseline, monitoring], ignore_index=True),
|
|
244
|
+
**(parameters | {"baseline_steps": len(baseline)}),
|
|
245
|
+
)
|
|
246
|
+
diagnostic = result.diagnostics.iloc[0]
|
|
247
|
+
for name in (
|
|
248
|
+
"baseline_mean",
|
|
249
|
+
"baseline_sd",
|
|
250
|
+
"baseline_se_floor",
|
|
251
|
+
"scale",
|
|
252
|
+
"numerical_floor_dominant",
|
|
253
|
+
"n_scored",
|
|
254
|
+
):
|
|
255
|
+
coverage[name] = diagnostic[name]
|
|
256
|
+
coverage["n_unscorable"] = len(monitor_grid) - coverage["n_scored"]
|
|
257
|
+
return coverage, result
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _fixed_window_half(
|
|
261
|
+
half: int,
|
|
262
|
+
series: pd.DataFrame,
|
|
263
|
+
schedule: pd.DataFrame,
|
|
264
|
+
steps: list[int],
|
|
265
|
+
positions: dict,
|
|
266
|
+
parameters: dict,
|
|
267
|
+
horizon_complete: bool,
|
|
268
|
+
) -> dict:
|
|
269
|
+
"""Scan the full grid, distinguishing observed from possible first alerts."""
|
|
270
|
+
resolved = _detector_parameters(parameters)
|
|
271
|
+
coverage, result = _fixed_baseline_scores(series, steps, resolved)
|
|
272
|
+
monitor_grid = steps[resolved["baseline_steps"] :]
|
|
273
|
+
observed = set(schedule.step)
|
|
274
|
+
row = {
|
|
275
|
+
"half": half,
|
|
276
|
+
"status": "unavailable",
|
|
277
|
+
"alert_step": None,
|
|
278
|
+
"onset_step": None,
|
|
279
|
+
"direction": None,
|
|
280
|
+
"alert_position": None,
|
|
281
|
+
"n_prompts": int(schedule.prompt_id.nunique()),
|
|
282
|
+
"n_sampling_units": int(schedule.sampling_unit_id.nunique()),
|
|
283
|
+
"n_rollouts": int(schedule.n_rollouts.sum()),
|
|
284
|
+
"n_steps": int(schedule.step.nunique()),
|
|
285
|
+
"missing_steps": [int(step) for step in steps if step not in observed],
|
|
286
|
+
"reason": "insufficient_fixed_baseline",
|
|
287
|
+
"source_detect_status": "insufficient_fixed_baseline",
|
|
288
|
+
"first_possible_alert_step": None,
|
|
289
|
+
"observed_alert_step": None,
|
|
290
|
+
"observed_onset_step": None,
|
|
291
|
+
"observed_direction": None,
|
|
292
|
+
"first_alert_identified": False,
|
|
293
|
+
**coverage,
|
|
294
|
+
}
|
|
295
|
+
if result is None:
|
|
296
|
+
return row
|
|
297
|
+
signs = {
|
|
298
|
+
int(point.step): int(point.direction)
|
|
299
|
+
for point in result.scores.loc[
|
|
300
|
+
result.scores.step.isin(monitor_grid)
|
|
301
|
+
].itertuples()
|
|
302
|
+
if np.isfinite(point.se)
|
|
303
|
+
}
|
|
304
|
+
sustain = resolved["sustain"]
|
|
305
|
+
allowed_signs = (1,) if series.kind.iloc[0] == "binary" else (-1, 1)
|
|
306
|
+
row.update(_scan_first_alerts(signs, monitor_grid, sustain, allowed_signs))
|
|
307
|
+
|
|
308
|
+
actual, possible = row["observed_alert_step"], row["first_possible_alert_step"]
|
|
309
|
+
row["source_detect_status"] = (
|
|
310
|
+
"alerted"
|
|
311
|
+
if actual is not None
|
|
312
|
+
else ("no_alert" if row["n_scored"] else "no_scorable_observations")
|
|
313
|
+
)
|
|
314
|
+
if actual is not None and possible == actual:
|
|
315
|
+
row.update(
|
|
316
|
+
status="alerted",
|
|
317
|
+
reason="first_alert_observed",
|
|
318
|
+
alert_step=actual,
|
|
319
|
+
onset_step=row["observed_onset_step"],
|
|
320
|
+
direction=row["observed_direction"],
|
|
321
|
+
alert_position=positions[actual],
|
|
322
|
+
first_alert_identified=True,
|
|
323
|
+
)
|
|
324
|
+
elif possible is not None:
|
|
325
|
+
row["reason"] = "ambiguous_first_alert"
|
|
326
|
+
elif horizon_complete:
|
|
327
|
+
row.update(status="no_alert", reason="no_alert_within_complete_horizon")
|
|
328
|
+
else:
|
|
329
|
+
row.update(status="pending", reason="incomplete_comparison_horizon")
|
|
330
|
+
return row
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def _replication_prepared(
|
|
334
|
+
full: DetectionResult,
|
|
335
|
+
half_series: dict[int, pd.DataFrame],
|
|
336
|
+
half_schedules: dict[int, pd.DataFrame],
|
|
337
|
+
assignments: pd.DataFrame,
|
|
338
|
+
*,
|
|
339
|
+
run: str,
|
|
340
|
+
feature: str,
|
|
341
|
+
tolerance: int,
|
|
342
|
+
parameters: dict,
|
|
343
|
+
coverage_policy: str = "contiguous_prefix",
|
|
344
|
+
) -> ReplicationEvidence:
|
|
345
|
+
"""Compare first alerts using preaggregated half series."""
|
|
346
|
+
_validate_coverage_policy(coverage_policy)
|
|
347
|
+
columns = _HALF_COLUMNS + (
|
|
348
|
+
_FIXED_COLUMNS if coverage_policy == "fixed_window" else []
|
|
349
|
+
)
|
|
350
|
+
if full.alerts.empty:
|
|
351
|
+
status = (
|
|
352
|
+
"no_full_alert"
|
|
353
|
+
if full.diagnostics.n_scored.sum() > 0
|
|
354
|
+
else "full_unscorable"
|
|
355
|
+
)
|
|
356
|
+
return ReplicationEvidence(
|
|
357
|
+
run,
|
|
358
|
+
feature,
|
|
359
|
+
status,
|
|
360
|
+
None,
|
|
361
|
+
full,
|
|
362
|
+
pd.DataFrame(columns=columns),
|
|
363
|
+
assignments,
|
|
364
|
+
tolerance,
|
|
365
|
+
False,
|
|
366
|
+
None,
|
|
367
|
+
coverage_policy,
|
|
368
|
+
)
|
|
369
|
+
|
|
370
|
+
alert = full.alerts.iloc[0]
|
|
371
|
+
steps = full.scores.step.tolist()
|
|
372
|
+
positions = {step: position for position, step in enumerate(steps)}
|
|
373
|
+
full_position = positions[alert.alert_step]
|
|
374
|
+
horizon_complete = full_position + tolerance < len(steps)
|
|
375
|
+
cutoff_position = min(full_position + tolerance, len(steps) - 1)
|
|
376
|
+
required_steps = steps[: cutoff_position + 1]
|
|
377
|
+
cutoff_step = int(required_steps[-1])
|
|
378
|
+
rows = []
|
|
379
|
+
for half in (0, 1):
|
|
380
|
+
subset = half_schedules[half].loc[
|
|
381
|
+
half_schedules[half].step.isin(required_steps)
|
|
382
|
+
]
|
|
383
|
+
if coverage_policy == "fixed_window":
|
|
384
|
+
rows.append(
|
|
385
|
+
_fixed_window_half(
|
|
386
|
+
half,
|
|
387
|
+
half_series[half],
|
|
388
|
+
subset,
|
|
389
|
+
required_steps,
|
|
390
|
+
positions,
|
|
391
|
+
parameters,
|
|
392
|
+
horizon_complete,
|
|
393
|
+
)
|
|
394
|
+
)
|
|
395
|
+
continue
|
|
396
|
+
rows.append(
|
|
397
|
+
_contiguous_half(
|
|
398
|
+
half,
|
|
399
|
+
half_series[half],
|
|
400
|
+
subset,
|
|
401
|
+
required_steps,
|
|
402
|
+
positions,
|
|
403
|
+
parameters,
|
|
404
|
+
horizon_complete,
|
|
405
|
+
)
|
|
406
|
+
)
|
|
407
|
+
|
|
408
|
+
halves = pd.DataFrame(rows, columns=columns)
|
|
409
|
+
# Keep integer step identities exact when alerts are missing.
|
|
410
|
+
step_columns = ["alert_step", "onset_step"]
|
|
411
|
+
if coverage_policy == "fixed_window":
|
|
412
|
+
step_columns += [
|
|
413
|
+
"first_possible_alert_step",
|
|
414
|
+
"observed_alert_step",
|
|
415
|
+
"observed_onset_step",
|
|
416
|
+
]
|
|
417
|
+
for column in step_columns:
|
|
418
|
+
halves[column] = pd.Series([row[column] for row in rows], dtype=object)
|
|
419
|
+
alerted = halves.loc[halves.status.eq("alerted")]
|
|
420
|
+
alert_positions = [full_position, *alerted.alert_position.tolist()]
|
|
421
|
+
known_failure = (
|
|
422
|
+
halves.status.eq("no_alert").any()
|
|
423
|
+
or alerted.direction.ne(alert.direction).any()
|
|
424
|
+
or max(alert_positions) - min(alert_positions) > tolerance
|
|
425
|
+
)
|
|
426
|
+
if known_failure:
|
|
427
|
+
status, replicates = "not_replicated", False
|
|
428
|
+
elif len(alerted) == 2:
|
|
429
|
+
status, replicates = "replicated", True
|
|
430
|
+
elif halves.status.eq("unavailable").any():
|
|
431
|
+
status, replicates = "unavailable", None
|
|
432
|
+
else:
|
|
433
|
+
status, replicates = "pending", None
|
|
434
|
+
return ReplicationEvidence(
|
|
435
|
+
run,
|
|
436
|
+
feature,
|
|
437
|
+
status,
|
|
438
|
+
replicates,
|
|
439
|
+
full,
|
|
440
|
+
halves,
|
|
441
|
+
assignments,
|
|
442
|
+
tolerance,
|
|
443
|
+
horizon_complete,
|
|
444
|
+
cutoff_step,
|
|
445
|
+
coverage_policy,
|
|
446
|
+
)
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def _scan_first_alerts(
|
|
450
|
+
signs: dict, monitor_grid: list[int], sustain: int, allowed_signs: tuple[int, ...]
|
|
451
|
+
) -> dict:
|
|
452
|
+
"""Track witnessed alerts and earlier alerts that missing rows could hide."""
|
|
453
|
+
witnesses = {
|
|
454
|
+
"first_possible_alert_step": None,
|
|
455
|
+
"observed_alert_step": None,
|
|
456
|
+
"observed_onset_step": None,
|
|
457
|
+
"observed_direction": None,
|
|
458
|
+
}
|
|
459
|
+
possible_streaks = dict.fromkeys(allowed_signs, 0)
|
|
460
|
+
actual_streak, previous_sign, onset = 0, 0, None
|
|
461
|
+
for step in monitor_grid:
|
|
462
|
+
sign = signs.get(step) # None is unavailable, zero is observed nonqualifying.
|
|
463
|
+
for candidate_sign in allowed_signs:
|
|
464
|
+
possible_streaks[candidate_sign] = (
|
|
465
|
+
possible_streaks[candidate_sign] + 1
|
|
466
|
+
if sign is None or sign == candidate_sign
|
|
467
|
+
else 0
|
|
468
|
+
)
|
|
469
|
+
if (
|
|
470
|
+
possible_streaks[candidate_sign] >= sustain
|
|
471
|
+
and witnesses["first_possible_alert_step"] is None
|
|
472
|
+
):
|
|
473
|
+
witnesses["first_possible_alert_step"] = int(step)
|
|
474
|
+
if sign is None or sign == 0:
|
|
475
|
+
actual_streak, previous_sign = 0, 0
|
|
476
|
+
elif sign == previous_sign:
|
|
477
|
+
actual_streak += 1
|
|
478
|
+
else:
|
|
479
|
+
actual_streak, previous_sign, onset = 1, sign, step
|
|
480
|
+
if actual_streak >= sustain and witnesses["observed_alert_step"] is None:
|
|
481
|
+
witnesses.update(
|
|
482
|
+
observed_alert_step=int(step),
|
|
483
|
+
observed_onset_step=int(onset),
|
|
484
|
+
observed_direction="up" if sign > 0 else "down",
|
|
485
|
+
)
|
|
486
|
+
|
|
487
|
+
return witnesses
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
def _contiguous_half(
|
|
491
|
+
half, series, subset, required_steps, positions, parameters, horizon_complete
|
|
492
|
+
):
|
|
493
|
+
observed = set(subset.step)
|
|
494
|
+
missing = [int(step) for step in required_steps if step not in observed]
|
|
495
|
+
prefix_end = positions[missing[0]] if missing else len(required_steps)
|
|
496
|
+
prefix = subset.loc[subset.step.isin(required_steps[:prefix_end])]
|
|
497
|
+
prepared = series.loc[series.step.isin(required_steps[:prefix_end])]
|
|
498
|
+
result = detect_sequential(prepared, **parameters)
|
|
499
|
+
diagnostic = result.diagnostics.iloc[0] if not result.diagnostics.empty else None
|
|
500
|
+
source_status = diagnostic.status if diagnostic is not None else "no_observations"
|
|
501
|
+
n_scored = int(diagnostic.n_scored) if diagnostic is not None else 0
|
|
502
|
+
n_unscorable = int(diagnostic.n_unscorable) if diagnostic is not None else 0
|
|
503
|
+
row = {
|
|
504
|
+
"half": half,
|
|
505
|
+
"status": "unavailable",
|
|
506
|
+
"alert_step": None,
|
|
507
|
+
"onset_step": None,
|
|
508
|
+
"direction": None,
|
|
509
|
+
"alert_position": None,
|
|
510
|
+
"n_prompts": int(prefix.prompt_id.nunique()),
|
|
511
|
+
"n_sampling_units": int(prefix.sampling_unit_id.nunique()),
|
|
512
|
+
"n_rollouts": int(prefix.n_rollouts.sum()),
|
|
513
|
+
"n_steps": int(prefix.step.nunique()),
|
|
514
|
+
"n_scored": n_scored,
|
|
515
|
+
"n_unscorable": n_unscorable,
|
|
516
|
+
"missing_steps": missing,
|
|
517
|
+
"reason": None,
|
|
518
|
+
"source_detect_status": source_status,
|
|
519
|
+
}
|
|
520
|
+
for name in (
|
|
521
|
+
"baseline_mean",
|
|
522
|
+
"baseline_sd",
|
|
523
|
+
"baseline_se_floor",
|
|
524
|
+
"scale",
|
|
525
|
+
"numerical_floor_dominant",
|
|
526
|
+
):
|
|
527
|
+
row[name] = diagnostic[name] if diagnostic is not None else None
|
|
528
|
+
if not result.alerts.empty:
|
|
529
|
+
half_alert = result.alerts.iloc[0]
|
|
530
|
+
# An unavailable-SE row before the observed alert could have completed
|
|
531
|
+
# an earlier compatible streak, so the first alert is then unknown.
|
|
532
|
+
resolved = _detector_parameters(parameters)
|
|
533
|
+
monitor_grid = required_steps[resolved["baseline_steps"] : prefix_end]
|
|
534
|
+
signs = {
|
|
535
|
+
int(point.step): int(point.direction)
|
|
536
|
+
for point in result.scores.loc[
|
|
537
|
+
result.scores.step.isin(monitor_grid)
|
|
538
|
+
].itertuples()
|
|
539
|
+
if np.isfinite(point.se)
|
|
540
|
+
}
|
|
541
|
+
allowed_signs = (1,) if series.kind.iloc[0] == "binary" else (-1, 1)
|
|
542
|
+
possible = _scan_first_alerts(
|
|
543
|
+
signs, monitor_grid, resolved["sustain"], allowed_signs
|
|
544
|
+
)["first_possible_alert_step"]
|
|
545
|
+
if possible != int(half_alert.alert_step):
|
|
546
|
+
row["reason"] = "ambiguous_first_alert"
|
|
547
|
+
else:
|
|
548
|
+
row.update(
|
|
549
|
+
status="alerted",
|
|
550
|
+
alert_step=int(half_alert.alert_step),
|
|
551
|
+
onset_step=int(half_alert.onset_step),
|
|
552
|
+
direction=half_alert.direction,
|
|
553
|
+
alert_position=positions[half_alert.alert_step],
|
|
554
|
+
reason="first_alert_observed",
|
|
555
|
+
)
|
|
556
|
+
elif missing:
|
|
557
|
+
row["reason"] = "missing_step_grid"
|
|
558
|
+
elif source_status in (
|
|
559
|
+
"insufficient_baseline",
|
|
560
|
+
"insufficient_baseline_uncertainty",
|
|
561
|
+
"no_observations",
|
|
562
|
+
):
|
|
563
|
+
row["reason"] = source_status
|
|
564
|
+
elif n_unscorable:
|
|
565
|
+
row["reason"] = "unavailable_monitoring_uncertainty"
|
|
566
|
+
elif not horizon_complete:
|
|
567
|
+
row.update(status="pending", reason="incomplete_comparison_horizon")
|
|
568
|
+
elif n_scored:
|
|
569
|
+
row.update(status="no_alert", reason="no_alert_within_complete_horizon")
|
|
570
|
+
else:
|
|
571
|
+
row["reason"] = source_status
|
|
572
|
+
return row
|