rolloutdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rolloutdiff/__init__.py +130 -0
- rolloutdiff/_aria_replay.py +346 -0
- rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
- rolloutdiff/_dependence.py +195 -0
- rolloutdiff/_feature_values.py +79 -0
- rolloutdiff/_replay_container.py +568 -0
- rolloutdiff/_report_validation.py +758 -0
- rolloutdiff/_retrospective_policy.py +33 -0
- rolloutdiff/_sampling.py +127 -0
- rolloutdiff/_series_validation.py +67 -0
- rolloutdiff/_watch_report.py +436 -0
- rolloutdiff/_watch_storage.py +800 -0
- rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
- rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
- rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
- rolloutdiff/_web/assets/third-party-notices.txt +112 -0
- rolloutdiff/_web/index.html +14 -0
- rolloutdiff/adapters.py +264 -0
- rolloutdiff/app.py +956 -0
- rolloutdiff/app_evidence.py +678 -0
- rolloutdiff/benchmark_reference.py +458 -0
- rolloutdiff/cli.py +1292 -0
- rolloutdiff/comparison.py +572 -0
- rolloutdiff/comparison_report.py +982 -0
- rolloutdiff/corroboration.py +287 -0
- rolloutdiff/detect.py +307 -0
- rolloutdiff/events.py +173 -0
- rolloutdiff/evidence.py +285 -0
- rolloutdiff/exemplars.py +240 -0
- rolloutdiff/features.py +823 -0
- rolloutdiff/html_report.py +726 -0
- rolloutdiff/inspect_logs.py +383 -0
- rolloutdiff/inspection.py +207 -0
- rolloutdiff/integrations.py +552 -0
- rolloutdiff/investigation.py +1146 -0
- rolloutdiff/notifications.py +406 -0
- rolloutdiff/pipeline.py +809 -0
- rolloutdiff/producer.py +663 -0
- rolloutdiff/prompt_mix.py +258 -0
- rolloutdiff/ranking.py +388 -0
- rolloutdiff/replay.py +589 -0
- rolloutdiff/replication.py +572 -0
- rolloutdiff/report.py +838 -0
- rolloutdiff/retrospective.py +386 -0
- rolloutdiff/schema.py +109 -0
- rolloutdiff/sensitivity.py +202 -0
- rolloutdiff/series.py +430 -0
- rolloutdiff/tool_features.py +422 -0
- rolloutdiff/trajectories.py +236 -0
- rolloutdiff/watch-compatibility.json +13 -0
- rolloutdiff/watch.py +742 -0
- rolloutdiff-0.1.0.dist-info/METADATA +158 -0
- rolloutdiff-0.1.0.dist-info/RECORD +62 -0
- rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
- rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
- rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
- rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,386 @@
|
|
|
1
|
+
"""Exact L2 mean-shift candidate with an optional covariance confirmation."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from numbers import Real
|
|
7
|
+
|
|
8
|
+
import numpy as np
|
|
9
|
+
import pandas as pd
|
|
10
|
+
|
|
11
|
+
from ._dependence import confirm_ar1, quasi_binomial_errors
|
|
12
|
+
from ._series_validation import validate_series
|
|
13
|
+
|
|
14
|
+
_COMMON = ["run", "feature", "tier", "kind"]
|
|
15
|
+
_SEGMENT = [
|
|
16
|
+
"previous_step",
|
|
17
|
+
"n_before",
|
|
18
|
+
"n_after",
|
|
19
|
+
"mean_before",
|
|
20
|
+
"mean_after",
|
|
21
|
+
"delta",
|
|
22
|
+
"rms_se_before",
|
|
23
|
+
"rms_se_after",
|
|
24
|
+
]
|
|
25
|
+
_COSTS = [
|
|
26
|
+
"normalization_scale",
|
|
27
|
+
"sse_no_change_scaled",
|
|
28
|
+
"sse_split_scaled",
|
|
29
|
+
"gain_scaled",
|
|
30
|
+
"penalty_scaled",
|
|
31
|
+
"residual_variance_scaled",
|
|
32
|
+
"se_variance_floor_scaled",
|
|
33
|
+
"variance_scaled",
|
|
34
|
+
"variance_source",
|
|
35
|
+
]
|
|
36
|
+
_DEPENDENCE = [
|
|
37
|
+
"dependence",
|
|
38
|
+
"independent_selected",
|
|
39
|
+
"independent_reason",
|
|
40
|
+
"covariance_status",
|
|
41
|
+
"covariance_measurement",
|
|
42
|
+
"covariance_gain",
|
|
43
|
+
"covariance_penalty",
|
|
44
|
+
"covariance_null_rho",
|
|
45
|
+
"covariance_alternative_rho",
|
|
46
|
+
"covariance_null_tau_variance_scaled",
|
|
47
|
+
"covariance_alternative_tau_variance_scaled",
|
|
48
|
+
"covariance_null_rho_bound",
|
|
49
|
+
"covariance_alternative_rho_bound",
|
|
50
|
+
"covariance_null_variance_bound",
|
|
51
|
+
"covariance_alternative_variance_bound",
|
|
52
|
+
"covariance_delta_scaled",
|
|
53
|
+
"covariance_grid_fits",
|
|
54
|
+
]
|
|
55
|
+
_DIAGNOSTICS = [
|
|
56
|
+
*_COMMON,
|
|
57
|
+
"status",
|
|
58
|
+
"reason",
|
|
59
|
+
"n_observations",
|
|
60
|
+
"n_candidates",
|
|
61
|
+
"n_missing_se",
|
|
62
|
+
"min_segment_steps",
|
|
63
|
+
"penalty_multiplier",
|
|
64
|
+
"min_effect",
|
|
65
|
+
"candidate_step",
|
|
66
|
+
*_SEGMENT,
|
|
67
|
+
*_COSTS,
|
|
68
|
+
*_DEPENDENCE,
|
|
69
|
+
]
|
|
70
|
+
_CHANGES = [*_COMMON, "changepoint_step", *_SEGMENT, "direction"]
|
|
71
|
+
_SCREENING = ["run", "n_series", "n_screened", "n_changes", "n_unscorable"]
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@dataclass(frozen=True)
|
|
75
|
+
class RetrospectiveResult:
|
|
76
|
+
changepoints: pd.DataFrame
|
|
77
|
+
diagnostics: pd.DataFrame
|
|
78
|
+
screening: pd.DataFrame
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _moments(values: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
|
|
82
|
+
"""Stable prefix means and residual sums, avoiding sum(x²)-sum(x)²/n."""
|
|
83
|
+
means, sums = np.zeros(len(values) + 1), np.zeros(len(values) + 1)
|
|
84
|
+
for count, value in enumerate(values, 1):
|
|
85
|
+
delta = value - means[count - 1]
|
|
86
|
+
means[count] = means[count - 1] + delta / count
|
|
87
|
+
sums[count] = max(0.0, sums[count - 1] + delta * (value - means[count]))
|
|
88
|
+
return means, sums
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _rms(values: np.ndarray) -> float:
|
|
92
|
+
scale = float(values.max())
|
|
93
|
+
return float(scale * np.sqrt(np.mean((values / scale) ** 2))) if scale else 0.0
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _fit(
|
|
97
|
+
x: np.ndarray, se: np.ndarray, minimum: int, multiplier: float
|
|
98
|
+
) -> tuple[int, dict]:
|
|
99
|
+
# Normalize squared quantities to keep the variance floor independent
|
|
100
|
+
# of the feature's physical units.
|
|
101
|
+
with np.errstate(over="raise", invalid="raise", divide="raise"):
|
|
102
|
+
centered = x - x[0]
|
|
103
|
+
scale = max(float(np.abs(centered).max()), float(se.max()))
|
|
104
|
+
y = centered / scale if scale else np.zeros(len(x))
|
|
105
|
+
errors = se / scale if scale else np.zeros(len(x))
|
|
106
|
+
means, sums = _moments(y)
|
|
107
|
+
reverse_means, reverse_sums = _moments(y[::-1])
|
|
108
|
+
n = len(y)
|
|
109
|
+
splits = np.arange(minimum, n - minimum + 1)
|
|
110
|
+
costs = sums[splits] + reverse_sums[n - splits]
|
|
111
|
+
best = float(costs.min())
|
|
112
|
+
# Resolve numerical ties toward the earliest boundary with a tolerance
|
|
113
|
+
# relative to the normalized full-run cost.
|
|
114
|
+
tolerance = 8 * np.finfo(float).eps * sums[n]
|
|
115
|
+
k = int(splits[np.flatnonzero(costs <= best + tolerance)[0]])
|
|
116
|
+
split_cost = float(sums[k] + reverse_sums[n - k])
|
|
117
|
+
null_cost = float(sums[n])
|
|
118
|
+
residual_variance = split_cost / (n - 2)
|
|
119
|
+
se_variance = float(np.mean(errors**2))
|
|
120
|
+
variance = max(residual_variance, se_variance)
|
|
121
|
+
penalty = multiplier * (np.log(n) * variance)
|
|
122
|
+
source = (
|
|
123
|
+
"zero"
|
|
124
|
+
if variance == 0
|
|
125
|
+
else (
|
|
126
|
+
"across_prompt_se" if se_variance >= residual_variance else "residual"
|
|
127
|
+
)
|
|
128
|
+
)
|
|
129
|
+
left, right = means[k], reverse_means[n - k]
|
|
130
|
+
numbers = {
|
|
131
|
+
"normalization_scale": scale,
|
|
132
|
+
"mean_before": float(x[0] + left * scale),
|
|
133
|
+
"mean_after": float(x[0] + right * scale),
|
|
134
|
+
"delta": float((right - left) * scale),
|
|
135
|
+
"rms_se_before": _rms(se[:k]),
|
|
136
|
+
"rms_se_after": _rms(se[k:]),
|
|
137
|
+
"sse_no_change_scaled": null_cost,
|
|
138
|
+
"sse_split_scaled": split_cost,
|
|
139
|
+
"gain_scaled": max(0.0, null_cost - split_cost),
|
|
140
|
+
"penalty_scaled": float(penalty),
|
|
141
|
+
"residual_variance_scaled": residual_variance,
|
|
142
|
+
"se_variance_floor_scaled": se_variance,
|
|
143
|
+
"variance_scaled": variance,
|
|
144
|
+
}
|
|
145
|
+
if right != left and numbers["delta"] == 0:
|
|
146
|
+
raise FloatingPointError("effect underflow")
|
|
147
|
+
if not all(np.isfinite(value) for value in numbers.values()):
|
|
148
|
+
raise FloatingPointError
|
|
149
|
+
return k, {**numbers, "variance_source": source}
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def detect_retrospective(
|
|
153
|
+
series: pd.DataFrame,
|
|
154
|
+
*,
|
|
155
|
+
min_segment_steps: int = 5,
|
|
156
|
+
penalty_multiplier: float = 3.0,
|
|
157
|
+
min_prevalence_change: float = 0.10,
|
|
158
|
+
min_continuous_change: float = 0.0,
|
|
159
|
+
dependence: str = "ar1",
|
|
160
|
+
) -> RetrospectiveResult:
|
|
161
|
+
"""Select at most one full-run mean shift per run/feature.
|
|
162
|
+
|
|
163
|
+
The best two-segment L2 split (``changepoint_step`` starts the right
|
|
164
|
+
segment) is selected when its absolute effect reaches the kind's minimum
|
|
165
|
+
change and SSE0 - SSE1 > ``penalty_multiplier`` * log(n) * variance, where
|
|
166
|
+
variance = max(SSE1/(n-2), mean(SE**2)). Missing SE or too few rows is
|
|
167
|
+
unscorable.
|
|
168
|
+
|
|
169
|
+
``dependence="ar1"`` keeps a selection only if, with a stationary AR(1)
|
|
170
|
+
covariance profiled under each mean model, twice the log-likelihood gain
|
|
171
|
+
exceeds ``penalty_multiplier`` * log(n) and the GLS shift keeps its
|
|
172
|
+
direction; exact piecewise-constant zero-SE splits stay selected as
|
|
173
|
+
``exact_zero_variance_unconfirmed``. AR lags count logged rows, ignoring
|
|
174
|
+
step gaps. ``*_scaled`` diagnostics divide by ``normalization_scale``
|
|
175
|
+
(squared for costs and variances); other effects keep input units.
|
|
176
|
+
"""
|
|
177
|
+
if dependence not in ("ar1", "independent"):
|
|
178
|
+
raise ValueError("dependence must be ar1 or independent")
|
|
179
|
+
if type(min_segment_steps) is not int or min_segment_steps < 2:
|
|
180
|
+
raise ValueError("min_segment_steps must be an integer of at least two")
|
|
181
|
+
for name, value in [
|
|
182
|
+
("penalty_multiplier", penalty_multiplier),
|
|
183
|
+
("min_prevalence_change", min_prevalence_change),
|
|
184
|
+
("min_continuous_change", min_continuous_change),
|
|
185
|
+
]:
|
|
186
|
+
if (
|
|
187
|
+
isinstance(value, bool)
|
|
188
|
+
or not isinstance(value, Real)
|
|
189
|
+
or not np.isfinite(value)
|
|
190
|
+
):
|
|
191
|
+
raise ValueError(f"{name} must be finite and numeric")
|
|
192
|
+
if (
|
|
193
|
+
penalty_multiplier <= 0
|
|
194
|
+
or not 0 <= min_prevalence_change <= 1
|
|
195
|
+
or min_continuous_change < 0
|
|
196
|
+
):
|
|
197
|
+
raise ValueError("invalid penalty or minimum effect")
|
|
198
|
+
frame = validate_series(series)
|
|
199
|
+
diagnostics, changes = [], []
|
|
200
|
+
for (run, feature), group in frame.groupby(["run", "feature"], sort=False):
|
|
201
|
+
row, change = _detect_group(
|
|
202
|
+
group,
|
|
203
|
+
run,
|
|
204
|
+
feature,
|
|
205
|
+
min_segment_steps,
|
|
206
|
+
penalty_multiplier,
|
|
207
|
+
min_prevalence_change,
|
|
208
|
+
min_continuous_change,
|
|
209
|
+
dependence,
|
|
210
|
+
)
|
|
211
|
+
diagnostics.append(row)
|
|
212
|
+
if change is not None:
|
|
213
|
+
changes.append(change)
|
|
214
|
+
|
|
215
|
+
diagnostic_table = pd.DataFrame(diagnostics, columns=_DIAGNOSTICS)
|
|
216
|
+
# Keep integer step identities exact when candidates are missing.
|
|
217
|
+
for column in ("candidate_step", "previous_step"):
|
|
218
|
+
diagnostic_table[column] = pd.Series(
|
|
219
|
+
[row[column] for row in diagnostics], dtype=object
|
|
220
|
+
)
|
|
221
|
+
change_table = (
|
|
222
|
+
pd.DataFrame(changes, columns=_CHANGES)
|
|
223
|
+
.sort_values(["run", "changepoint_step", "feature"])
|
|
224
|
+
.reset_index(drop=True)
|
|
225
|
+
)
|
|
226
|
+
if diagnostic_table.empty:
|
|
227
|
+
screening = pd.DataFrame(columns=_SCREENING)
|
|
228
|
+
else:
|
|
229
|
+
screening = (
|
|
230
|
+
diagnostic_table.assign(
|
|
231
|
+
screened=diagnostic_table.status.isin(["change", "no_change"]),
|
|
232
|
+
changed=diagnostic_table.status.eq("change"),
|
|
233
|
+
)
|
|
234
|
+
.groupby("run", sort=True)
|
|
235
|
+
.agg(
|
|
236
|
+
n_series=("feature", "size"),
|
|
237
|
+
n_screened=("screened", "sum"),
|
|
238
|
+
n_changes=("changed", "sum"),
|
|
239
|
+
)
|
|
240
|
+
.reset_index()
|
|
241
|
+
)
|
|
242
|
+
screening["n_unscorable"] = screening.n_series - screening.n_screened
|
|
243
|
+
return RetrospectiveResult(change_table, diagnostic_table, screening)
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def _select_candidate(row: dict, x: np.ndarray, kind: str, effect: float) -> None:
|
|
247
|
+
# Allow float64 roundoff so an effect exactly at the cutoff qualifies.
|
|
248
|
+
tolerance = (
|
|
249
|
+
4
|
|
250
|
+
* np.finfo(float).eps
|
|
251
|
+
* (1 if kind == "binary" else max(abs(row["delta"]), effect))
|
|
252
|
+
)
|
|
253
|
+
if np.all(x == x[0]):
|
|
254
|
+
row.update(status="no_change", reason="constant")
|
|
255
|
+
elif abs(row["delta"]) < effect - tolerance:
|
|
256
|
+
row.update(status="no_change", reason="below_effect")
|
|
257
|
+
elif row["gain_scaled"] <= row["penalty_scaled"]:
|
|
258
|
+
row.update(status="no_change", reason="penalty")
|
|
259
|
+
else:
|
|
260
|
+
row.update(status="change", reason="penalized_fit")
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _confirm_candidate(row, x, se, units, k, kind, penalty_multiplier, dependence):
|
|
264
|
+
row.update(
|
|
265
|
+
independent_selected=row["status"] == "change",
|
|
266
|
+
independent_reason=row["reason"],
|
|
267
|
+
covariance_status="not_requested"
|
|
268
|
+
if dependence == "independent"
|
|
269
|
+
else "not_needed",
|
|
270
|
+
)
|
|
271
|
+
if not row["independent_selected"] or dependence != "ar1":
|
|
272
|
+
return
|
|
273
|
+
if np.all(se == 0) and row["sse_split_scaled"] == 0:
|
|
274
|
+
# An exact descriptive step has singular Gaussian likelihood.
|
|
275
|
+
row["covariance_status"] = "exact_zero_variance_unconfirmed"
|
|
276
|
+
return
|
|
277
|
+
try:
|
|
278
|
+
with np.errstate(over="raise", invalid="raise", divide="raise"):
|
|
279
|
+
scale = row["normalization_scale"]
|
|
280
|
+
normalized = (x - x[0]) / scale
|
|
281
|
+
errors = se / scale
|
|
282
|
+
if np.any((se > 0) & (errors == 0)):
|
|
283
|
+
raise FloatingPointError("measurement SE underflow")
|
|
284
|
+
null_errors, row["covariance_measurement"] = errors, "step_se"
|
|
285
|
+
if kind == "binary":
|
|
286
|
+
constant = np.full(len(x), x.mean())
|
|
287
|
+
modelled = quasi_binomial_errors(x, errors, units, constant)
|
|
288
|
+
if modelled is not None:
|
|
289
|
+
segments = np.where(
|
|
290
|
+
np.arange(len(x)) < k, row["mean_before"], row["mean_after"]
|
|
291
|
+
)
|
|
292
|
+
null_errors, row["covariance_measurement"] = (
|
|
293
|
+
modelled,
|
|
294
|
+
"quasi_binomial_se",
|
|
295
|
+
)
|
|
296
|
+
errors = quasi_binomial_errors(x, errors, units, segments)
|
|
297
|
+
row.update(
|
|
298
|
+
confirm_ar1(
|
|
299
|
+
normalized,
|
|
300
|
+
null_errors,
|
|
301
|
+
k,
|
|
302
|
+
float(penalty_multiplier),
|
|
303
|
+
alternative_errors=errors,
|
|
304
|
+
)
|
|
305
|
+
)
|
|
306
|
+
except FloatingPointError:
|
|
307
|
+
raise ValueError(
|
|
308
|
+
"numerical overflow or underflow fitting retrospective covariance"
|
|
309
|
+
) from None
|
|
310
|
+
if row["covariance_status"] != "confirmed":
|
|
311
|
+
row.update(status="no_change", reason="covariance_not_confirmed")
|
|
312
|
+
elif np.sign(row["covariance_delta_scaled"]) != np.sign(row["delta"]):
|
|
313
|
+
row.update(
|
|
314
|
+
status="no_change",
|
|
315
|
+
reason="covariance_direction_mismatch",
|
|
316
|
+
covariance_status="direction_mismatch",
|
|
317
|
+
)
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def _detect_group(
|
|
321
|
+
group,
|
|
322
|
+
run,
|
|
323
|
+
feature,
|
|
324
|
+
min_segment_steps,
|
|
325
|
+
penalty_multiplier,
|
|
326
|
+
min_prevalence_change,
|
|
327
|
+
min_continuous_change,
|
|
328
|
+
dependence,
|
|
329
|
+
):
|
|
330
|
+
kind, tier = group.kind.iloc[0], group.tier.iloc[0]
|
|
331
|
+
x, se, steps = group["mean"].to_numpy(), group.se.to_numpy(), group.step.to_numpy()
|
|
332
|
+
units = group.n_sampling_units.to_numpy(dtype=float)
|
|
333
|
+
n = len(x)
|
|
334
|
+
effect = float(min_prevalence_change if kind == "binary" else min_continuous_change)
|
|
335
|
+
row = {name: np.nan for name in _DIAGNOSTICS}
|
|
336
|
+
row.update(
|
|
337
|
+
run=run,
|
|
338
|
+
feature=feature,
|
|
339
|
+
tier=tier,
|
|
340
|
+
kind=kind,
|
|
341
|
+
status="insufficient_data",
|
|
342
|
+
reason="too_few_observations",
|
|
343
|
+
n_observations=n,
|
|
344
|
+
n_candidates=0,
|
|
345
|
+
n_missing_se=int(np.isnan(se).sum()),
|
|
346
|
+
min_segment_steps=min_segment_steps,
|
|
347
|
+
penalty_multiplier=float(penalty_multiplier),
|
|
348
|
+
min_effect=effect,
|
|
349
|
+
candidate_step=None,
|
|
350
|
+
previous_step=None,
|
|
351
|
+
variance_source=None,
|
|
352
|
+
dependence=dependence,
|
|
353
|
+
independent_selected=None,
|
|
354
|
+
independent_reason=None,
|
|
355
|
+
covariance_status="unscorable",
|
|
356
|
+
covariance_grid_fits=0,
|
|
357
|
+
)
|
|
358
|
+
if n < 2 * min_segment_steps:
|
|
359
|
+
return row, None
|
|
360
|
+
if row["n_missing_se"]:
|
|
361
|
+
row.update(status="unavailable_uncertainty", reason="missing_se")
|
|
362
|
+
return row, None
|
|
363
|
+
try:
|
|
364
|
+
k, fitted = _fit(x, se, min_segment_steps, float(penalty_multiplier))
|
|
365
|
+
except FloatingPointError:
|
|
366
|
+
raise ValueError(
|
|
367
|
+
"numerical overflow or underflow fitting retrospective segmentation"
|
|
368
|
+
) from None
|
|
369
|
+
row.update(
|
|
370
|
+
fitted,
|
|
371
|
+
candidate_step=int(steps[k]),
|
|
372
|
+
previous_step=int(steps[k - 1]),
|
|
373
|
+
n_before=k,
|
|
374
|
+
n_after=n - k,
|
|
375
|
+
n_candidates=n - 2 * min_segment_steps + 1,
|
|
376
|
+
)
|
|
377
|
+
_select_candidate(row, x, kind, effect)
|
|
378
|
+
_confirm_candidate(row, x, se, units, k, kind, penalty_multiplier, dependence)
|
|
379
|
+
change = None
|
|
380
|
+
if row["status"] == "change":
|
|
381
|
+
change = {name: row[name] for name in [*_COMMON, *_SEGMENT]}
|
|
382
|
+
change.update(
|
|
383
|
+
changepoint_step=int(steps[k]),
|
|
384
|
+
direction="up" if row["delta"] > 0 else "down",
|
|
385
|
+
)
|
|
386
|
+
return row, change
|
rolloutdiff/schema.py
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""Canonical rollout records, excluding benchmark labels."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
import pandas as pd
|
|
9
|
+
from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Rollout(BaseModel):
|
|
13
|
+
"""One completion keyed by (run, sample_id), with unchanged text.
|
|
14
|
+
|
|
15
|
+
Prompt IDs identify exact content within runs; messages retain roles/metadata.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
model_config = ConfigDict(
|
|
19
|
+
extra="forbid", strict=True, frozen=True, allow_inf_nan=False
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
run: str = Field(min_length=1)
|
|
23
|
+
step: int = Field(ge=0)
|
|
24
|
+
sample_id: str = Field(min_length=1)
|
|
25
|
+
prompt_id: str = Field(min_length=1)
|
|
26
|
+
prompt: str
|
|
27
|
+
response: str
|
|
28
|
+
reward: float
|
|
29
|
+
prompt_messages: list[dict[str, Any]] | None = None
|
|
30
|
+
response_messages: list[dict[str, Any]] | None = None
|
|
31
|
+
tool_calls: list[dict[str, Any]] | None = None
|
|
32
|
+
|
|
33
|
+
@field_validator("run", "sample_id", "prompt_id")
|
|
34
|
+
@classmethod
|
|
35
|
+
def nonblank_identifier(cls, value: str) -> str:
|
|
36
|
+
if not value.strip():
|
|
37
|
+
raise ValueError("identifier must not be blank")
|
|
38
|
+
return value
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
CANONICAL_COLUMNS = list(Rollout.model_fields)
|
|
42
|
+
REQUIRED_COLUMNS = [
|
|
43
|
+
name for name, field in Rollout.model_fields.items() if field.is_required()
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class _ComparisonRollout(Rollout):
|
|
48
|
+
step: int | None = Field(ge=0)
|
|
49
|
+
reward: float | None
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def validate_rollouts(
|
|
53
|
+
table: pd.DataFrame, *, allow_unavailable: bool = False
|
|
54
|
+
) -> pd.DataFrame:
|
|
55
|
+
"""Validate canonical rows and reset the positional index.
|
|
56
|
+
|
|
57
|
+
Comparisons may opt into null step/reward values. Analysis and watch require
|
|
58
|
+
observed steps and finite rewards.
|
|
59
|
+
"""
|
|
60
|
+
if type(allow_unavailable) is not bool:
|
|
61
|
+
raise ValueError("allow_unavailable must be Boolean")
|
|
62
|
+
if not table.columns.is_unique:
|
|
63
|
+
raise ValueError("duplicate rollout columns")
|
|
64
|
+
missing = set(REQUIRED_COLUMNS) - set(table.columns)
|
|
65
|
+
extra = set(table.columns) - set(CANONICAL_COLUMNS)
|
|
66
|
+
if missing or extra:
|
|
67
|
+
raise ValueError(
|
|
68
|
+
f"invalid rollout columns: missing={sorted(missing)}, extra={sorted(extra)}"
|
|
69
|
+
)
|
|
70
|
+
rows = []
|
|
71
|
+
model = _ComparisonRollout if allow_unavailable else Rollout
|
|
72
|
+
for position, row in enumerate(table.to_dict("records")):
|
|
73
|
+
# pandas materializes absent optional values as NaN in mixed tables.
|
|
74
|
+
for name in ("prompt_messages", "response_messages", "tool_calls"):
|
|
75
|
+
value = row.get(name)
|
|
76
|
+
if value is None or (isinstance(value, float) and pd.isna(value)):
|
|
77
|
+
row[name] = None
|
|
78
|
+
if allow_unavailable:
|
|
79
|
+
for name in ("step", "reward"):
|
|
80
|
+
if row.get(name) is pd.NA or (
|
|
81
|
+
isinstance(row.get(name), float) and pd.isna(row[name])
|
|
82
|
+
):
|
|
83
|
+
row[name] = None
|
|
84
|
+
try:
|
|
85
|
+
rows.append(model.model_validate(row).model_dump())
|
|
86
|
+
except ValidationError as exc:
|
|
87
|
+
fields = ", ".join(
|
|
88
|
+
".".join(map(str, e["loc"])) for e in exc.errors(include_input=False)
|
|
89
|
+
)
|
|
90
|
+
raise ValueError(f"invalid rollout at row {position}: {fields}") from None
|
|
91
|
+
out = pd.DataFrame(rows, columns=CANONICAL_COLUMNS)
|
|
92
|
+
if out.duplicated(["run", "sample_id"]).any():
|
|
93
|
+
raise ValueError("duplicate (run, sample_id) rollout identity")
|
|
94
|
+
if not out.empty:
|
|
95
|
+
prompt_keys = out[["run", "prompt_id", "prompt"]].copy()
|
|
96
|
+
prompt_keys["messages"] = out.prompt_messages.map(
|
|
97
|
+
lambda value: json.dumps(value, sort_keys=True, ensure_ascii=False)
|
|
98
|
+
)
|
|
99
|
+
distinct = prompt_keys.drop_duplicates().groupby(["run", "prompt_id"]).size()
|
|
100
|
+
if (distinct > 1).any():
|
|
101
|
+
raise ValueError(
|
|
102
|
+
"prompt_id refers to different prompt content within a run"
|
|
103
|
+
)
|
|
104
|
+
for name, dtype in (("step", "int64"), ("reward", "float64")):
|
|
105
|
+
if allow_unavailable and any(row[name] is None for row in rows):
|
|
106
|
+
out[name] = pd.Series([row[name] for row in rows], dtype=object)
|
|
107
|
+
else:
|
|
108
|
+
out[name] = out[name].astype(dtype)
|
|
109
|
+
return out
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
"""Fixed, experimental sensitivity policy over the retained feature catalog."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from copy import deepcopy
|
|
6
|
+
from numbers import Real
|
|
7
|
+
|
|
8
|
+
import numpy as np
|
|
9
|
+
import pandas as pd
|
|
10
|
+
|
|
11
|
+
from .detect import DetectionResult, detect_sequential
|
|
12
|
+
|
|
13
|
+
LENIENT_ID = "lenient-structural-binary-v1-reconstructed"
|
|
14
|
+
LENIENT_SEQUENTIAL = {
|
|
15
|
+
"baseline_steps": 15,
|
|
16
|
+
"z_threshold": 4.0,
|
|
17
|
+
"sustain": 3,
|
|
18
|
+
"min_prevalence_change": 0.0,
|
|
19
|
+
"numerical_floor": 1e-9,
|
|
20
|
+
}
|
|
21
|
+
_DEFAULT_SEQUENTIAL = {**LENIENT_SEQUENTIAL, "min_prevalence_change": 0.10}
|
|
22
|
+
_DEFAULT_RETROSPECTIVE = {
|
|
23
|
+
"min_segment_steps": 5,
|
|
24
|
+
"penalty_multiplier": 3.0,
|
|
25
|
+
"min_prevalence_change": 0.10,
|
|
26
|
+
"min_continuous_change": 0.0,
|
|
27
|
+
"dependence": "ar1",
|
|
28
|
+
}
|
|
29
|
+
_DEFAULT_VOCABULARY = {"max_sparse_per_tier": 2000, "min_prevalence": 0.05}
|
|
30
|
+
_DEFAULT_EVIDENCE = {
|
|
31
|
+
"replication_tolerance": 10,
|
|
32
|
+
"replication_coverage_policy": "contiguous_prefix",
|
|
33
|
+
"corroboration_radius": 10,
|
|
34
|
+
"persistence_effect_fraction": 0.5,
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _require_matching(supplied: dict, expected: dict, name: str) -> None:
|
|
39
|
+
for key, value in supplied.items():
|
|
40
|
+
if key not in expected:
|
|
41
|
+
raise ValueError(f"{name}.{key} is not part of the lenient profile")
|
|
42
|
+
target = expected[key]
|
|
43
|
+
if type(target) is int:
|
|
44
|
+
matches = type(value) is int and value == target
|
|
45
|
+
elif isinstance(target, float):
|
|
46
|
+
matches = (
|
|
47
|
+
not isinstance(value, bool)
|
|
48
|
+
and isinstance(value, Real)
|
|
49
|
+
and np.isfinite(value)
|
|
50
|
+
and value == target
|
|
51
|
+
)
|
|
52
|
+
else:
|
|
53
|
+
matches = type(value) is type(target) and value == target
|
|
54
|
+
if not matches:
|
|
55
|
+
raise ValueError(
|
|
56
|
+
f"{name}.{key} must equal {target!r} for the lenient profile"
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def validate_sensitivity(
|
|
61
|
+
sensitivity: str,
|
|
62
|
+
*,
|
|
63
|
+
vocabulary: dict,
|
|
64
|
+
sequential: dict,
|
|
65
|
+
retrospective: dict,
|
|
66
|
+
evidence: dict,
|
|
67
|
+
) -> None:
|
|
68
|
+
if not isinstance(sensitivity, str) or sensitivity not in {"default", "lenient"}:
|
|
69
|
+
raise ValueError("sensitivity must be 'default' or 'lenient'")
|
|
70
|
+
if sensitivity == "default":
|
|
71
|
+
return
|
|
72
|
+
for name, supplied, expected in (
|
|
73
|
+
("vocabulary", vocabulary, _DEFAULT_VOCABULARY),
|
|
74
|
+
("sequential_options", sequential, LENIENT_SEQUENTIAL),
|
|
75
|
+
("retrospective_options", retrospective, _DEFAULT_RETROSPECTIVE),
|
|
76
|
+
("evidence", evidence, _DEFAULT_EVIDENCE),
|
|
77
|
+
):
|
|
78
|
+
_require_matching(supplied, expected, name)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def eligible(frame: pd.DataFrame) -> pd.Series:
|
|
82
|
+
return frame.tier.eq("structural") & frame.kind.eq("binary")
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _screening(diagnostics: pd.DataFrame, *, exclusions: bool = False) -> pd.DataFrame:
|
|
86
|
+
columns = ["run", "n_series", "n_screened", "n_alerted", "n_unscorable"]
|
|
87
|
+
if exclusions:
|
|
88
|
+
columns.append("n_excluded")
|
|
89
|
+
if diagnostics.empty:
|
|
90
|
+
return pd.DataFrame(columns=columns)
|
|
91
|
+
counts = (
|
|
92
|
+
diagnostics.assign(
|
|
93
|
+
screened=diagnostics.n_scored.gt(0),
|
|
94
|
+
alerted=diagnostics.status.eq("alerted"),
|
|
95
|
+
excluded=diagnostics.status.eq("profile_excluded"),
|
|
96
|
+
)
|
|
97
|
+
.groupby("run", sort=True)
|
|
98
|
+
.agg(
|
|
99
|
+
n_series=("feature", "size"),
|
|
100
|
+
n_screened=("screened", "sum"),
|
|
101
|
+
n_alerted=("alerted", "sum"),
|
|
102
|
+
n_excluded=("excluded", "sum"),
|
|
103
|
+
)
|
|
104
|
+
.reset_index()
|
|
105
|
+
)
|
|
106
|
+
counts["n_unscorable"] = counts.n_series - counts.n_screened - counts.n_excluded
|
|
107
|
+
return counts[columns]
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def select_lenient(
|
|
111
|
+
series: pd.DataFrame, default: DetectionResult
|
|
112
|
+
) -> tuple[DetectionResult, dict]:
|
|
113
|
+
"""Replace eligible screening and retain default continuous context."""
|
|
114
|
+
experiment = detect_sequential(series.loc[eligible(series)], **LENIENT_SEQUENTIAL)
|
|
115
|
+
diagnostics = default.diagnostics.copy()
|
|
116
|
+
scores = default.scores.copy()
|
|
117
|
+
for target, replacement in (
|
|
118
|
+
(diagnostics, experiment.diagnostics),
|
|
119
|
+
(scores, experiment.scores),
|
|
120
|
+
):
|
|
121
|
+
selected = eligible(target)
|
|
122
|
+
# Both detector calls preserve the validated series/group order.
|
|
123
|
+
if selected.any():
|
|
124
|
+
for column in target:
|
|
125
|
+
target.loc[selected, column] = replacement[column].to_numpy()
|
|
126
|
+
excluded = target.kind.eq("binary") & ~selected
|
|
127
|
+
target.loc[excluded, "status"] = "profile_excluded"
|
|
128
|
+
excluded_diagnostics = diagnostics.status.eq("profile_excluded")
|
|
129
|
+
diagnostics.loc[excluded_diagnostics, ["n_scored", "n_unscorable"]] = 0
|
|
130
|
+
excluded_scores = scores.status.eq("profile_excluded")
|
|
131
|
+
scores.loc[excluded_scores, ["delta", "z"]] = np.nan
|
|
132
|
+
scores.loc[excluded_scores, ["direction", "streak"]] = 0
|
|
133
|
+
scores.loc[excluded_scores, "qualifies"] = False
|
|
134
|
+
context_alerts = default.alerts.loc[default.alerts.kind.eq("continuous")]
|
|
135
|
+
alert_parts = [
|
|
136
|
+
part for part in (experiment.alerts, context_alerts) if not part.empty
|
|
137
|
+
]
|
|
138
|
+
alerts = (
|
|
139
|
+
pd.concat(alert_parts, ignore_index=True)
|
|
140
|
+
.sort_values(["run", "alert_step", "feature"])
|
|
141
|
+
.reset_index(drop=True)
|
|
142
|
+
if alert_parts
|
|
143
|
+
else default.alerts.iloc[:0].copy()
|
|
144
|
+
)
|
|
145
|
+
counts = {"default_full": default.screening.to_dict("records")}
|
|
146
|
+
for name, screening in (
|
|
147
|
+
(
|
|
148
|
+
"default_eligible",
|
|
149
|
+
_screening(default.diagnostics.loc[eligible(default.diagnostics)]),
|
|
150
|
+
),
|
|
151
|
+
("experimental_eligible", experiment.screening),
|
|
152
|
+
):
|
|
153
|
+
rows = {row["run"]: row for row in screening.to_dict("records")}
|
|
154
|
+
counts[name] = [
|
|
155
|
+
rows.get(
|
|
156
|
+
run,
|
|
157
|
+
{
|
|
158
|
+
"run": run,
|
|
159
|
+
"n_series": 0,
|
|
160
|
+
"n_screened": 0,
|
|
161
|
+
"n_alerted": 0,
|
|
162
|
+
"n_unscorable": 0,
|
|
163
|
+
},
|
|
164
|
+
)
|
|
165
|
+
for run in default.screening.run
|
|
166
|
+
]
|
|
167
|
+
return (
|
|
168
|
+
DetectionResult(
|
|
169
|
+
alerts, diagnostics, scores, _screening(diagnostics, exclusions=True)
|
|
170
|
+
),
|
|
171
|
+
counts,
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def profile_metadata(
|
|
176
|
+
counts: dict, *, explicit_sampling_units: bool, retrospective_settings: dict | None
|
|
177
|
+
) -> dict:
|
|
178
|
+
return {
|
|
179
|
+
"id": LENIENT_ID,
|
|
180
|
+
"selector": "lenient",
|
|
181
|
+
"identity": "new reconstruction on current semantics; not the unsaved original policy",
|
|
182
|
+
"experimental": True,
|
|
183
|
+
"scope": "offline_full_run",
|
|
184
|
+
"source_checkpoint": "f233cb2bd0a6a3a9c1b5f80878ebc7c1e92381f2",
|
|
185
|
+
"provenance": {
|
|
186
|
+
"decision_sha256": "3dd87c8305ac7a5606f4cf9d35b7dd97c82c9c141362b459f6c372dc8fb6098a"
|
|
187
|
+
},
|
|
188
|
+
"eligibility": {
|
|
189
|
+
"tier": "structural",
|
|
190
|
+
"kind": "binary",
|
|
191
|
+
"selection": "current retained catalog intersection",
|
|
192
|
+
},
|
|
193
|
+
"effective_settings": {
|
|
194
|
+
"vocabulary": deepcopy(_DEFAULT_VOCABULARY),
|
|
195
|
+
"sequential": deepcopy(LENIENT_SEQUENTIAL),
|
|
196
|
+
"context_sequential": deepcopy(_DEFAULT_SEQUENTIAL),
|
|
197
|
+
"retrospective": deepcopy(retrospective_settings),
|
|
198
|
+
"evidence": deepcopy(_DEFAULT_EVIDENCE),
|
|
199
|
+
"sampling_units": "explicit" if explicit_sampling_units else "exact_prompt",
|
|
200
|
+
},
|
|
201
|
+
"counts": deepcopy(counts),
|
|
202
|
+
}
|