rolloutdiff 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. rolloutdiff/__init__.py +130 -0
  2. rolloutdiff/_aria_replay.py +346 -0
  3. rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
  4. rolloutdiff/_dependence.py +195 -0
  5. rolloutdiff/_feature_values.py +79 -0
  6. rolloutdiff/_replay_container.py +568 -0
  7. rolloutdiff/_report_validation.py +758 -0
  8. rolloutdiff/_retrospective_policy.py +33 -0
  9. rolloutdiff/_sampling.py +127 -0
  10. rolloutdiff/_series_validation.py +67 -0
  11. rolloutdiff/_watch_report.py +436 -0
  12. rolloutdiff/_watch_storage.py +800 -0
  13. rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
  14. rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
  15. rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
  16. rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
  17. rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
  18. rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
  19. rolloutdiff/_web/assets/third-party-notices.txt +112 -0
  20. rolloutdiff/_web/index.html +14 -0
  21. rolloutdiff/adapters.py +264 -0
  22. rolloutdiff/app.py +956 -0
  23. rolloutdiff/app_evidence.py +678 -0
  24. rolloutdiff/benchmark_reference.py +458 -0
  25. rolloutdiff/cli.py +1292 -0
  26. rolloutdiff/comparison.py +572 -0
  27. rolloutdiff/comparison_report.py +982 -0
  28. rolloutdiff/corroboration.py +287 -0
  29. rolloutdiff/detect.py +307 -0
  30. rolloutdiff/events.py +173 -0
  31. rolloutdiff/evidence.py +285 -0
  32. rolloutdiff/exemplars.py +240 -0
  33. rolloutdiff/features.py +823 -0
  34. rolloutdiff/html_report.py +726 -0
  35. rolloutdiff/inspect_logs.py +383 -0
  36. rolloutdiff/inspection.py +207 -0
  37. rolloutdiff/integrations.py +552 -0
  38. rolloutdiff/investigation.py +1146 -0
  39. rolloutdiff/notifications.py +406 -0
  40. rolloutdiff/pipeline.py +809 -0
  41. rolloutdiff/producer.py +663 -0
  42. rolloutdiff/prompt_mix.py +258 -0
  43. rolloutdiff/ranking.py +388 -0
  44. rolloutdiff/replay.py +589 -0
  45. rolloutdiff/replication.py +572 -0
  46. rolloutdiff/report.py +838 -0
  47. rolloutdiff/retrospective.py +386 -0
  48. rolloutdiff/schema.py +109 -0
  49. rolloutdiff/sensitivity.py +202 -0
  50. rolloutdiff/series.py +430 -0
  51. rolloutdiff/tool_features.py +422 -0
  52. rolloutdiff/trajectories.py +236 -0
  53. rolloutdiff/watch-compatibility.json +13 -0
  54. rolloutdiff/watch.py +742 -0
  55. rolloutdiff-0.1.0.dist-info/METADATA +158 -0
  56. rolloutdiff-0.1.0.dist-info/RECORD +62 -0
  57. rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
  58. rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
  59. rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
  60. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
  61. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
  62. rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,386 @@
1
+ """Exact L2 mean-shift candidate with an optional covariance confirmation."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ from numbers import Real
7
+
8
+ import numpy as np
9
+ import pandas as pd
10
+
11
+ from ._dependence import confirm_ar1, quasi_binomial_errors
12
+ from ._series_validation import validate_series
13
+
14
+ _COMMON = ["run", "feature", "tier", "kind"]
15
+ _SEGMENT = [
16
+ "previous_step",
17
+ "n_before",
18
+ "n_after",
19
+ "mean_before",
20
+ "mean_after",
21
+ "delta",
22
+ "rms_se_before",
23
+ "rms_se_after",
24
+ ]
25
+ _COSTS = [
26
+ "normalization_scale",
27
+ "sse_no_change_scaled",
28
+ "sse_split_scaled",
29
+ "gain_scaled",
30
+ "penalty_scaled",
31
+ "residual_variance_scaled",
32
+ "se_variance_floor_scaled",
33
+ "variance_scaled",
34
+ "variance_source",
35
+ ]
36
+ _DEPENDENCE = [
37
+ "dependence",
38
+ "independent_selected",
39
+ "independent_reason",
40
+ "covariance_status",
41
+ "covariance_measurement",
42
+ "covariance_gain",
43
+ "covariance_penalty",
44
+ "covariance_null_rho",
45
+ "covariance_alternative_rho",
46
+ "covariance_null_tau_variance_scaled",
47
+ "covariance_alternative_tau_variance_scaled",
48
+ "covariance_null_rho_bound",
49
+ "covariance_alternative_rho_bound",
50
+ "covariance_null_variance_bound",
51
+ "covariance_alternative_variance_bound",
52
+ "covariance_delta_scaled",
53
+ "covariance_grid_fits",
54
+ ]
55
+ _DIAGNOSTICS = [
56
+ *_COMMON,
57
+ "status",
58
+ "reason",
59
+ "n_observations",
60
+ "n_candidates",
61
+ "n_missing_se",
62
+ "min_segment_steps",
63
+ "penalty_multiplier",
64
+ "min_effect",
65
+ "candidate_step",
66
+ *_SEGMENT,
67
+ *_COSTS,
68
+ *_DEPENDENCE,
69
+ ]
70
+ _CHANGES = [*_COMMON, "changepoint_step", *_SEGMENT, "direction"]
71
+ _SCREENING = ["run", "n_series", "n_screened", "n_changes", "n_unscorable"]
72
+
73
+
74
+ @dataclass(frozen=True)
75
+ class RetrospectiveResult:
76
+ changepoints: pd.DataFrame
77
+ diagnostics: pd.DataFrame
78
+ screening: pd.DataFrame
79
+
80
+
81
+ def _moments(values: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
82
+ """Stable prefix means and residual sums, avoiding sum(x²)-sum(x)²/n."""
83
+ means, sums = np.zeros(len(values) + 1), np.zeros(len(values) + 1)
84
+ for count, value in enumerate(values, 1):
85
+ delta = value - means[count - 1]
86
+ means[count] = means[count - 1] + delta / count
87
+ sums[count] = max(0.0, sums[count - 1] + delta * (value - means[count]))
88
+ return means, sums
89
+
90
+
91
+ def _rms(values: np.ndarray) -> float:
92
+ scale = float(values.max())
93
+ return float(scale * np.sqrt(np.mean((values / scale) ** 2))) if scale else 0.0
94
+
95
+
96
+ def _fit(
97
+ x: np.ndarray, se: np.ndarray, minimum: int, multiplier: float
98
+ ) -> tuple[int, dict]:
99
+ # Normalize squared quantities to keep the variance floor independent
100
+ # of the feature's physical units.
101
+ with np.errstate(over="raise", invalid="raise", divide="raise"):
102
+ centered = x - x[0]
103
+ scale = max(float(np.abs(centered).max()), float(se.max()))
104
+ y = centered / scale if scale else np.zeros(len(x))
105
+ errors = se / scale if scale else np.zeros(len(x))
106
+ means, sums = _moments(y)
107
+ reverse_means, reverse_sums = _moments(y[::-1])
108
+ n = len(y)
109
+ splits = np.arange(minimum, n - minimum + 1)
110
+ costs = sums[splits] + reverse_sums[n - splits]
111
+ best = float(costs.min())
112
+ # Resolve numerical ties toward the earliest boundary with a tolerance
113
+ # relative to the normalized full-run cost.
114
+ tolerance = 8 * np.finfo(float).eps * sums[n]
115
+ k = int(splits[np.flatnonzero(costs <= best + tolerance)[0]])
116
+ split_cost = float(sums[k] + reverse_sums[n - k])
117
+ null_cost = float(sums[n])
118
+ residual_variance = split_cost / (n - 2)
119
+ se_variance = float(np.mean(errors**2))
120
+ variance = max(residual_variance, se_variance)
121
+ penalty = multiplier * (np.log(n) * variance)
122
+ source = (
123
+ "zero"
124
+ if variance == 0
125
+ else (
126
+ "across_prompt_se" if se_variance >= residual_variance else "residual"
127
+ )
128
+ )
129
+ left, right = means[k], reverse_means[n - k]
130
+ numbers = {
131
+ "normalization_scale": scale,
132
+ "mean_before": float(x[0] + left * scale),
133
+ "mean_after": float(x[0] + right * scale),
134
+ "delta": float((right - left) * scale),
135
+ "rms_se_before": _rms(se[:k]),
136
+ "rms_se_after": _rms(se[k:]),
137
+ "sse_no_change_scaled": null_cost,
138
+ "sse_split_scaled": split_cost,
139
+ "gain_scaled": max(0.0, null_cost - split_cost),
140
+ "penalty_scaled": float(penalty),
141
+ "residual_variance_scaled": residual_variance,
142
+ "se_variance_floor_scaled": se_variance,
143
+ "variance_scaled": variance,
144
+ }
145
+ if right != left and numbers["delta"] == 0:
146
+ raise FloatingPointError("effect underflow")
147
+ if not all(np.isfinite(value) for value in numbers.values()):
148
+ raise FloatingPointError
149
+ return k, {**numbers, "variance_source": source}
150
+
151
+
152
+ def detect_retrospective(
153
+ series: pd.DataFrame,
154
+ *,
155
+ min_segment_steps: int = 5,
156
+ penalty_multiplier: float = 3.0,
157
+ min_prevalence_change: float = 0.10,
158
+ min_continuous_change: float = 0.0,
159
+ dependence: str = "ar1",
160
+ ) -> RetrospectiveResult:
161
+ """Select at most one full-run mean shift per run/feature.
162
+
163
+ The best two-segment L2 split (``changepoint_step`` starts the right
164
+ segment) is selected when its absolute effect reaches the kind's minimum
165
+ change and SSE0 - SSE1 > ``penalty_multiplier`` * log(n) * variance, where
166
+ variance = max(SSE1/(n-2), mean(SE**2)). Missing SE or too few rows is
167
+ unscorable.
168
+
169
+ ``dependence="ar1"`` keeps a selection only if, with a stationary AR(1)
170
+ covariance profiled under each mean model, twice the log-likelihood gain
171
+ exceeds ``penalty_multiplier`` * log(n) and the GLS shift keeps its
172
+ direction; exact piecewise-constant zero-SE splits stay selected as
173
+ ``exact_zero_variance_unconfirmed``. AR lags count logged rows, ignoring
174
+ step gaps. ``*_scaled`` diagnostics divide by ``normalization_scale``
175
+ (squared for costs and variances); other effects keep input units.
176
+ """
177
+ if dependence not in ("ar1", "independent"):
178
+ raise ValueError("dependence must be ar1 or independent")
179
+ if type(min_segment_steps) is not int or min_segment_steps < 2:
180
+ raise ValueError("min_segment_steps must be an integer of at least two")
181
+ for name, value in [
182
+ ("penalty_multiplier", penalty_multiplier),
183
+ ("min_prevalence_change", min_prevalence_change),
184
+ ("min_continuous_change", min_continuous_change),
185
+ ]:
186
+ if (
187
+ isinstance(value, bool)
188
+ or not isinstance(value, Real)
189
+ or not np.isfinite(value)
190
+ ):
191
+ raise ValueError(f"{name} must be finite and numeric")
192
+ if (
193
+ penalty_multiplier <= 0
194
+ or not 0 <= min_prevalence_change <= 1
195
+ or min_continuous_change < 0
196
+ ):
197
+ raise ValueError("invalid penalty or minimum effect")
198
+ frame = validate_series(series)
199
+ diagnostics, changes = [], []
200
+ for (run, feature), group in frame.groupby(["run", "feature"], sort=False):
201
+ row, change = _detect_group(
202
+ group,
203
+ run,
204
+ feature,
205
+ min_segment_steps,
206
+ penalty_multiplier,
207
+ min_prevalence_change,
208
+ min_continuous_change,
209
+ dependence,
210
+ )
211
+ diagnostics.append(row)
212
+ if change is not None:
213
+ changes.append(change)
214
+
215
+ diagnostic_table = pd.DataFrame(diagnostics, columns=_DIAGNOSTICS)
216
+ # Keep integer step identities exact when candidates are missing.
217
+ for column in ("candidate_step", "previous_step"):
218
+ diagnostic_table[column] = pd.Series(
219
+ [row[column] for row in diagnostics], dtype=object
220
+ )
221
+ change_table = (
222
+ pd.DataFrame(changes, columns=_CHANGES)
223
+ .sort_values(["run", "changepoint_step", "feature"])
224
+ .reset_index(drop=True)
225
+ )
226
+ if diagnostic_table.empty:
227
+ screening = pd.DataFrame(columns=_SCREENING)
228
+ else:
229
+ screening = (
230
+ diagnostic_table.assign(
231
+ screened=diagnostic_table.status.isin(["change", "no_change"]),
232
+ changed=diagnostic_table.status.eq("change"),
233
+ )
234
+ .groupby("run", sort=True)
235
+ .agg(
236
+ n_series=("feature", "size"),
237
+ n_screened=("screened", "sum"),
238
+ n_changes=("changed", "sum"),
239
+ )
240
+ .reset_index()
241
+ )
242
+ screening["n_unscorable"] = screening.n_series - screening.n_screened
243
+ return RetrospectiveResult(change_table, diagnostic_table, screening)
244
+
245
+
246
+ def _select_candidate(row: dict, x: np.ndarray, kind: str, effect: float) -> None:
247
+ # Allow float64 roundoff so an effect exactly at the cutoff qualifies.
248
+ tolerance = (
249
+ 4
250
+ * np.finfo(float).eps
251
+ * (1 if kind == "binary" else max(abs(row["delta"]), effect))
252
+ )
253
+ if np.all(x == x[0]):
254
+ row.update(status="no_change", reason="constant")
255
+ elif abs(row["delta"]) < effect - tolerance:
256
+ row.update(status="no_change", reason="below_effect")
257
+ elif row["gain_scaled"] <= row["penalty_scaled"]:
258
+ row.update(status="no_change", reason="penalty")
259
+ else:
260
+ row.update(status="change", reason="penalized_fit")
261
+
262
+
263
+ def _confirm_candidate(row, x, se, units, k, kind, penalty_multiplier, dependence):
264
+ row.update(
265
+ independent_selected=row["status"] == "change",
266
+ independent_reason=row["reason"],
267
+ covariance_status="not_requested"
268
+ if dependence == "independent"
269
+ else "not_needed",
270
+ )
271
+ if not row["independent_selected"] or dependence != "ar1":
272
+ return
273
+ if np.all(se == 0) and row["sse_split_scaled"] == 0:
274
+ # An exact descriptive step has singular Gaussian likelihood.
275
+ row["covariance_status"] = "exact_zero_variance_unconfirmed"
276
+ return
277
+ try:
278
+ with np.errstate(over="raise", invalid="raise", divide="raise"):
279
+ scale = row["normalization_scale"]
280
+ normalized = (x - x[0]) / scale
281
+ errors = se / scale
282
+ if np.any((se > 0) & (errors == 0)):
283
+ raise FloatingPointError("measurement SE underflow")
284
+ null_errors, row["covariance_measurement"] = errors, "step_se"
285
+ if kind == "binary":
286
+ constant = np.full(len(x), x.mean())
287
+ modelled = quasi_binomial_errors(x, errors, units, constant)
288
+ if modelled is not None:
289
+ segments = np.where(
290
+ np.arange(len(x)) < k, row["mean_before"], row["mean_after"]
291
+ )
292
+ null_errors, row["covariance_measurement"] = (
293
+ modelled,
294
+ "quasi_binomial_se",
295
+ )
296
+ errors = quasi_binomial_errors(x, errors, units, segments)
297
+ row.update(
298
+ confirm_ar1(
299
+ normalized,
300
+ null_errors,
301
+ k,
302
+ float(penalty_multiplier),
303
+ alternative_errors=errors,
304
+ )
305
+ )
306
+ except FloatingPointError:
307
+ raise ValueError(
308
+ "numerical overflow or underflow fitting retrospective covariance"
309
+ ) from None
310
+ if row["covariance_status"] != "confirmed":
311
+ row.update(status="no_change", reason="covariance_not_confirmed")
312
+ elif np.sign(row["covariance_delta_scaled"]) != np.sign(row["delta"]):
313
+ row.update(
314
+ status="no_change",
315
+ reason="covariance_direction_mismatch",
316
+ covariance_status="direction_mismatch",
317
+ )
318
+
319
+
320
+ def _detect_group(
321
+ group,
322
+ run,
323
+ feature,
324
+ min_segment_steps,
325
+ penalty_multiplier,
326
+ min_prevalence_change,
327
+ min_continuous_change,
328
+ dependence,
329
+ ):
330
+ kind, tier = group.kind.iloc[0], group.tier.iloc[0]
331
+ x, se, steps = group["mean"].to_numpy(), group.se.to_numpy(), group.step.to_numpy()
332
+ units = group.n_sampling_units.to_numpy(dtype=float)
333
+ n = len(x)
334
+ effect = float(min_prevalence_change if kind == "binary" else min_continuous_change)
335
+ row = {name: np.nan for name in _DIAGNOSTICS}
336
+ row.update(
337
+ run=run,
338
+ feature=feature,
339
+ tier=tier,
340
+ kind=kind,
341
+ status="insufficient_data",
342
+ reason="too_few_observations",
343
+ n_observations=n,
344
+ n_candidates=0,
345
+ n_missing_se=int(np.isnan(se).sum()),
346
+ min_segment_steps=min_segment_steps,
347
+ penalty_multiplier=float(penalty_multiplier),
348
+ min_effect=effect,
349
+ candidate_step=None,
350
+ previous_step=None,
351
+ variance_source=None,
352
+ dependence=dependence,
353
+ independent_selected=None,
354
+ independent_reason=None,
355
+ covariance_status="unscorable",
356
+ covariance_grid_fits=0,
357
+ )
358
+ if n < 2 * min_segment_steps:
359
+ return row, None
360
+ if row["n_missing_se"]:
361
+ row.update(status="unavailable_uncertainty", reason="missing_se")
362
+ return row, None
363
+ try:
364
+ k, fitted = _fit(x, se, min_segment_steps, float(penalty_multiplier))
365
+ except FloatingPointError:
366
+ raise ValueError(
367
+ "numerical overflow or underflow fitting retrospective segmentation"
368
+ ) from None
369
+ row.update(
370
+ fitted,
371
+ candidate_step=int(steps[k]),
372
+ previous_step=int(steps[k - 1]),
373
+ n_before=k,
374
+ n_after=n - k,
375
+ n_candidates=n - 2 * min_segment_steps + 1,
376
+ )
377
+ _select_candidate(row, x, kind, effect)
378
+ _confirm_candidate(row, x, se, units, k, kind, penalty_multiplier, dependence)
379
+ change = None
380
+ if row["status"] == "change":
381
+ change = {name: row[name] for name in [*_COMMON, *_SEGMENT]}
382
+ change.update(
383
+ changepoint_step=int(steps[k]),
384
+ direction="up" if row["delta"] > 0 else "down",
385
+ )
386
+ return row, change
rolloutdiff/schema.py ADDED
@@ -0,0 +1,109 @@
1
+ """Canonical rollout records, excluding benchmark labels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from typing import Any
7
+
8
+ import pandas as pd
9
+ from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator
10
+
11
+
12
+ class Rollout(BaseModel):
13
+ """One completion keyed by (run, sample_id), with unchanged text.
14
+
15
+ Prompt IDs identify exact content within runs; messages retain roles/metadata.
16
+ """
17
+
18
+ model_config = ConfigDict(
19
+ extra="forbid", strict=True, frozen=True, allow_inf_nan=False
20
+ )
21
+
22
+ run: str = Field(min_length=1)
23
+ step: int = Field(ge=0)
24
+ sample_id: str = Field(min_length=1)
25
+ prompt_id: str = Field(min_length=1)
26
+ prompt: str
27
+ response: str
28
+ reward: float
29
+ prompt_messages: list[dict[str, Any]] | None = None
30
+ response_messages: list[dict[str, Any]] | None = None
31
+ tool_calls: list[dict[str, Any]] | None = None
32
+
33
+ @field_validator("run", "sample_id", "prompt_id")
34
+ @classmethod
35
+ def nonblank_identifier(cls, value: str) -> str:
36
+ if not value.strip():
37
+ raise ValueError("identifier must not be blank")
38
+ return value
39
+
40
+
41
+ CANONICAL_COLUMNS = list(Rollout.model_fields)
42
+ REQUIRED_COLUMNS = [
43
+ name for name, field in Rollout.model_fields.items() if field.is_required()
44
+ ]
45
+
46
+
47
+ class _ComparisonRollout(Rollout):
48
+ step: int | None = Field(ge=0)
49
+ reward: float | None
50
+
51
+
52
+ def validate_rollouts(
53
+ table: pd.DataFrame, *, allow_unavailable: bool = False
54
+ ) -> pd.DataFrame:
55
+ """Validate canonical rows and reset the positional index.
56
+
57
+ Comparisons may opt into null step/reward values. Analysis and watch require
58
+ observed steps and finite rewards.
59
+ """
60
+ if type(allow_unavailable) is not bool:
61
+ raise ValueError("allow_unavailable must be Boolean")
62
+ if not table.columns.is_unique:
63
+ raise ValueError("duplicate rollout columns")
64
+ missing = set(REQUIRED_COLUMNS) - set(table.columns)
65
+ extra = set(table.columns) - set(CANONICAL_COLUMNS)
66
+ if missing or extra:
67
+ raise ValueError(
68
+ f"invalid rollout columns: missing={sorted(missing)}, extra={sorted(extra)}"
69
+ )
70
+ rows = []
71
+ model = _ComparisonRollout if allow_unavailable else Rollout
72
+ for position, row in enumerate(table.to_dict("records")):
73
+ # pandas materializes absent optional values as NaN in mixed tables.
74
+ for name in ("prompt_messages", "response_messages", "tool_calls"):
75
+ value = row.get(name)
76
+ if value is None or (isinstance(value, float) and pd.isna(value)):
77
+ row[name] = None
78
+ if allow_unavailable:
79
+ for name in ("step", "reward"):
80
+ if row.get(name) is pd.NA or (
81
+ isinstance(row.get(name), float) and pd.isna(row[name])
82
+ ):
83
+ row[name] = None
84
+ try:
85
+ rows.append(model.model_validate(row).model_dump())
86
+ except ValidationError as exc:
87
+ fields = ", ".join(
88
+ ".".join(map(str, e["loc"])) for e in exc.errors(include_input=False)
89
+ )
90
+ raise ValueError(f"invalid rollout at row {position}: {fields}") from None
91
+ out = pd.DataFrame(rows, columns=CANONICAL_COLUMNS)
92
+ if out.duplicated(["run", "sample_id"]).any():
93
+ raise ValueError("duplicate (run, sample_id) rollout identity")
94
+ if not out.empty:
95
+ prompt_keys = out[["run", "prompt_id", "prompt"]].copy()
96
+ prompt_keys["messages"] = out.prompt_messages.map(
97
+ lambda value: json.dumps(value, sort_keys=True, ensure_ascii=False)
98
+ )
99
+ distinct = prompt_keys.drop_duplicates().groupby(["run", "prompt_id"]).size()
100
+ if (distinct > 1).any():
101
+ raise ValueError(
102
+ "prompt_id refers to different prompt content within a run"
103
+ )
104
+ for name, dtype in (("step", "int64"), ("reward", "float64")):
105
+ if allow_unavailable and any(row[name] is None for row in rows):
106
+ out[name] = pd.Series([row[name] for row in rows], dtype=object)
107
+ else:
108
+ out[name] = out[name].astype(dtype)
109
+ return out
@@ -0,0 +1,202 @@
1
+ """Fixed, experimental sensitivity policy over the retained feature catalog."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from copy import deepcopy
6
+ from numbers import Real
7
+
8
+ import numpy as np
9
+ import pandas as pd
10
+
11
+ from .detect import DetectionResult, detect_sequential
12
+
13
+ LENIENT_ID = "lenient-structural-binary-v1-reconstructed"
14
+ LENIENT_SEQUENTIAL = {
15
+ "baseline_steps": 15,
16
+ "z_threshold": 4.0,
17
+ "sustain": 3,
18
+ "min_prevalence_change": 0.0,
19
+ "numerical_floor": 1e-9,
20
+ }
21
+ _DEFAULT_SEQUENTIAL = {**LENIENT_SEQUENTIAL, "min_prevalence_change": 0.10}
22
+ _DEFAULT_RETROSPECTIVE = {
23
+ "min_segment_steps": 5,
24
+ "penalty_multiplier": 3.0,
25
+ "min_prevalence_change": 0.10,
26
+ "min_continuous_change": 0.0,
27
+ "dependence": "ar1",
28
+ }
29
+ _DEFAULT_VOCABULARY = {"max_sparse_per_tier": 2000, "min_prevalence": 0.05}
30
+ _DEFAULT_EVIDENCE = {
31
+ "replication_tolerance": 10,
32
+ "replication_coverage_policy": "contiguous_prefix",
33
+ "corroboration_radius": 10,
34
+ "persistence_effect_fraction": 0.5,
35
+ }
36
+
37
+
38
+ def _require_matching(supplied: dict, expected: dict, name: str) -> None:
39
+ for key, value in supplied.items():
40
+ if key not in expected:
41
+ raise ValueError(f"{name}.{key} is not part of the lenient profile")
42
+ target = expected[key]
43
+ if type(target) is int:
44
+ matches = type(value) is int and value == target
45
+ elif isinstance(target, float):
46
+ matches = (
47
+ not isinstance(value, bool)
48
+ and isinstance(value, Real)
49
+ and np.isfinite(value)
50
+ and value == target
51
+ )
52
+ else:
53
+ matches = type(value) is type(target) and value == target
54
+ if not matches:
55
+ raise ValueError(
56
+ f"{name}.{key} must equal {target!r} for the lenient profile"
57
+ )
58
+
59
+
60
+ def validate_sensitivity(
61
+ sensitivity: str,
62
+ *,
63
+ vocabulary: dict,
64
+ sequential: dict,
65
+ retrospective: dict,
66
+ evidence: dict,
67
+ ) -> None:
68
+ if not isinstance(sensitivity, str) or sensitivity not in {"default", "lenient"}:
69
+ raise ValueError("sensitivity must be 'default' or 'lenient'")
70
+ if sensitivity == "default":
71
+ return
72
+ for name, supplied, expected in (
73
+ ("vocabulary", vocabulary, _DEFAULT_VOCABULARY),
74
+ ("sequential_options", sequential, LENIENT_SEQUENTIAL),
75
+ ("retrospective_options", retrospective, _DEFAULT_RETROSPECTIVE),
76
+ ("evidence", evidence, _DEFAULT_EVIDENCE),
77
+ ):
78
+ _require_matching(supplied, expected, name)
79
+
80
+
81
+ def eligible(frame: pd.DataFrame) -> pd.Series:
82
+ return frame.tier.eq("structural") & frame.kind.eq("binary")
83
+
84
+
85
+ def _screening(diagnostics: pd.DataFrame, *, exclusions: bool = False) -> pd.DataFrame:
86
+ columns = ["run", "n_series", "n_screened", "n_alerted", "n_unscorable"]
87
+ if exclusions:
88
+ columns.append("n_excluded")
89
+ if diagnostics.empty:
90
+ return pd.DataFrame(columns=columns)
91
+ counts = (
92
+ diagnostics.assign(
93
+ screened=diagnostics.n_scored.gt(0),
94
+ alerted=diagnostics.status.eq("alerted"),
95
+ excluded=diagnostics.status.eq("profile_excluded"),
96
+ )
97
+ .groupby("run", sort=True)
98
+ .agg(
99
+ n_series=("feature", "size"),
100
+ n_screened=("screened", "sum"),
101
+ n_alerted=("alerted", "sum"),
102
+ n_excluded=("excluded", "sum"),
103
+ )
104
+ .reset_index()
105
+ )
106
+ counts["n_unscorable"] = counts.n_series - counts.n_screened - counts.n_excluded
107
+ return counts[columns]
108
+
109
+
110
+ def select_lenient(
111
+ series: pd.DataFrame, default: DetectionResult
112
+ ) -> tuple[DetectionResult, dict]:
113
+ """Replace eligible screening and retain default continuous context."""
114
+ experiment = detect_sequential(series.loc[eligible(series)], **LENIENT_SEQUENTIAL)
115
+ diagnostics = default.diagnostics.copy()
116
+ scores = default.scores.copy()
117
+ for target, replacement in (
118
+ (diagnostics, experiment.diagnostics),
119
+ (scores, experiment.scores),
120
+ ):
121
+ selected = eligible(target)
122
+ # Both detector calls preserve the validated series/group order.
123
+ if selected.any():
124
+ for column in target:
125
+ target.loc[selected, column] = replacement[column].to_numpy()
126
+ excluded = target.kind.eq("binary") & ~selected
127
+ target.loc[excluded, "status"] = "profile_excluded"
128
+ excluded_diagnostics = diagnostics.status.eq("profile_excluded")
129
+ diagnostics.loc[excluded_diagnostics, ["n_scored", "n_unscorable"]] = 0
130
+ excluded_scores = scores.status.eq("profile_excluded")
131
+ scores.loc[excluded_scores, ["delta", "z"]] = np.nan
132
+ scores.loc[excluded_scores, ["direction", "streak"]] = 0
133
+ scores.loc[excluded_scores, "qualifies"] = False
134
+ context_alerts = default.alerts.loc[default.alerts.kind.eq("continuous")]
135
+ alert_parts = [
136
+ part for part in (experiment.alerts, context_alerts) if not part.empty
137
+ ]
138
+ alerts = (
139
+ pd.concat(alert_parts, ignore_index=True)
140
+ .sort_values(["run", "alert_step", "feature"])
141
+ .reset_index(drop=True)
142
+ if alert_parts
143
+ else default.alerts.iloc[:0].copy()
144
+ )
145
+ counts = {"default_full": default.screening.to_dict("records")}
146
+ for name, screening in (
147
+ (
148
+ "default_eligible",
149
+ _screening(default.diagnostics.loc[eligible(default.diagnostics)]),
150
+ ),
151
+ ("experimental_eligible", experiment.screening),
152
+ ):
153
+ rows = {row["run"]: row for row in screening.to_dict("records")}
154
+ counts[name] = [
155
+ rows.get(
156
+ run,
157
+ {
158
+ "run": run,
159
+ "n_series": 0,
160
+ "n_screened": 0,
161
+ "n_alerted": 0,
162
+ "n_unscorable": 0,
163
+ },
164
+ )
165
+ for run in default.screening.run
166
+ ]
167
+ return (
168
+ DetectionResult(
169
+ alerts, diagnostics, scores, _screening(diagnostics, exclusions=True)
170
+ ),
171
+ counts,
172
+ )
173
+
174
+
175
+ def profile_metadata(
176
+ counts: dict, *, explicit_sampling_units: bool, retrospective_settings: dict | None
177
+ ) -> dict:
178
+ return {
179
+ "id": LENIENT_ID,
180
+ "selector": "lenient",
181
+ "identity": "new reconstruction on current semantics; not the unsaved original policy",
182
+ "experimental": True,
183
+ "scope": "offline_full_run",
184
+ "source_checkpoint": "f233cb2bd0a6a3a9c1b5f80878ebc7c1e92381f2",
185
+ "provenance": {
186
+ "decision_sha256": "3dd87c8305ac7a5606f4cf9d35b7dd97c82c9c141362b459f6c372dc8fb6098a"
187
+ },
188
+ "eligibility": {
189
+ "tier": "structural",
190
+ "kind": "binary",
191
+ "selection": "current retained catalog intersection",
192
+ },
193
+ "effective_settings": {
194
+ "vocabulary": deepcopy(_DEFAULT_VOCABULARY),
195
+ "sequential": deepcopy(LENIENT_SEQUENTIAL),
196
+ "context_sequential": deepcopy(_DEFAULT_SEQUENTIAL),
197
+ "retrospective": deepcopy(retrospective_settings),
198
+ "evidence": deepcopy(_DEFAULT_EVIDENCE),
199
+ "sampling_units": "explicit" if explicit_sampling_units else "exact_prompt",
200
+ },
201
+ "counts": deepcopy(counts),
202
+ }