rolloutdiff 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. rolloutdiff/__init__.py +130 -0
  2. rolloutdiff/_aria_replay.py +346 -0
  3. rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
  4. rolloutdiff/_dependence.py +195 -0
  5. rolloutdiff/_feature_values.py +79 -0
  6. rolloutdiff/_replay_container.py +568 -0
  7. rolloutdiff/_report_validation.py +758 -0
  8. rolloutdiff/_retrospective_policy.py +33 -0
  9. rolloutdiff/_sampling.py +127 -0
  10. rolloutdiff/_series_validation.py +67 -0
  11. rolloutdiff/_watch_report.py +436 -0
  12. rolloutdiff/_watch_storage.py +800 -0
  13. rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
  14. rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
  15. rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
  16. rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
  17. rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
  18. rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
  19. rolloutdiff/_web/assets/third-party-notices.txt +112 -0
  20. rolloutdiff/_web/index.html +14 -0
  21. rolloutdiff/adapters.py +264 -0
  22. rolloutdiff/app.py +956 -0
  23. rolloutdiff/app_evidence.py +678 -0
  24. rolloutdiff/benchmark_reference.py +458 -0
  25. rolloutdiff/cli.py +1292 -0
  26. rolloutdiff/comparison.py +572 -0
  27. rolloutdiff/comparison_report.py +982 -0
  28. rolloutdiff/corroboration.py +287 -0
  29. rolloutdiff/detect.py +307 -0
  30. rolloutdiff/events.py +173 -0
  31. rolloutdiff/evidence.py +285 -0
  32. rolloutdiff/exemplars.py +240 -0
  33. rolloutdiff/features.py +823 -0
  34. rolloutdiff/html_report.py +726 -0
  35. rolloutdiff/inspect_logs.py +383 -0
  36. rolloutdiff/inspection.py +207 -0
  37. rolloutdiff/integrations.py +552 -0
  38. rolloutdiff/investigation.py +1146 -0
  39. rolloutdiff/notifications.py +406 -0
  40. rolloutdiff/pipeline.py +809 -0
  41. rolloutdiff/producer.py +663 -0
  42. rolloutdiff/prompt_mix.py +258 -0
  43. rolloutdiff/ranking.py +388 -0
  44. rolloutdiff/replay.py +589 -0
  45. rolloutdiff/replication.py +572 -0
  46. rolloutdiff/report.py +838 -0
  47. rolloutdiff/retrospective.py +386 -0
  48. rolloutdiff/schema.py +109 -0
  49. rolloutdiff/sensitivity.py +202 -0
  50. rolloutdiff/series.py +430 -0
  51. rolloutdiff/tool_features.py +422 -0
  52. rolloutdiff/trajectories.py +236 -0
  53. rolloutdiff/watch-compatibility.json +13 -0
  54. rolloutdiff/watch.py +742 -0
  55. rolloutdiff-0.1.0.dist-info/METADATA +158 -0
  56. rolloutdiff-0.1.0.dist-info/RECORD +62 -0
  57. rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
  58. rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
  59. rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
  60. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
  61. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
  62. rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,809 @@
1
+ """Analyze every retained feature with shared aggregation and evidence policies."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import inspect
7
+ import platform
8
+ import zlib
9
+ from collections.abc import Mapping
10
+ from copy import deepcopy
11
+ from dataclasses import dataclass, field, replace
12
+ from importlib.metadata import PackageNotFoundError, version
13
+ from numbers import Real
14
+ from pathlib import Path
15
+
16
+ import numpy as np
17
+ import pandas as pd
18
+
19
+ from ._feature_values import _FeatureValueIndex
20
+ from ._retrospective_policy import disabled_result, retrospective_metadata
21
+ from ._sampling import resolve_sampling_units, subset_sampling_units
22
+ from .corroboration import _corroboration_prepared
23
+ from .detect import DetectionResult, detect_sequential
24
+ from .events import _event_table, _group_events_prepared, _member_table
25
+ from .evidence import _persistence_prepared, _reward_association_prepared
26
+ from .exemplars import _exemplar_table, _select_exemplars_prepared
27
+ from .features import FeatureTables, featurize
28
+ from .prompt_mix import _prompt_mix_prepared
29
+ from .ranking import _ranking_table, rank_events
30
+ from .replication import _replication_prepared, _validate_coverage_policy
31
+ from .retrospective import RetrospectiveResult, detect_retrospective
32
+ from .schema import validate_rollouts
33
+ from .sensitivity import (
34
+ LENIENT_ID,
35
+ LENIENT_SEQUENTIAL,
36
+ profile_metadata,
37
+ select_lenient,
38
+ validate_sensitivity,
39
+ )
40
+ from .series import Aggregation, aggregate
41
+
42
+ _EVIDENCE_COLUMNS = [
43
+ "run",
44
+ "feature",
45
+ "tier",
46
+ "kind",
47
+ "sequential_status",
48
+ "scale_source",
49
+ "numerical_floor_dominant",
50
+ "retrospective_status",
51
+ "onset_step",
52
+ "alert_step",
53
+ "changepoint_step",
54
+ "replication_status",
55
+ "replication_coverage_policy",
56
+ "replicates",
57
+ "replication_horizon_complete",
58
+ "replication_cutoff_step",
59
+ "corroboration_status",
60
+ "corroborates",
61
+ "corroboration_reason",
62
+ "corroboration_window_start",
63
+ "corroboration_window_end",
64
+ "corroboration_horizon_complete",
65
+ "persistence_status",
66
+ "persistence_fraction",
67
+ "persistence_coverage",
68
+ "persistence_n_observations",
69
+ "persistence_n_evaluable",
70
+ "persistence_n_retained",
71
+ "persistence_n_unavailable",
72
+ "reward_status",
73
+ "reward_mean_difference",
74
+ "reward_n_groups",
75
+ "reward_n_mixed_groups",
76
+ "reward_n_rollouts",
77
+ "reward_n_included_rollouts",
78
+ "reward_window_start",
79
+ "reward_window_end",
80
+ "promptmix_status",
81
+ "prompt_before_start",
82
+ "prompt_before_end",
83
+ "prompt_after_start",
84
+ "prompt_after_end",
85
+ "prompt_positive_prompts",
86
+ "prompt_total_prompts",
87
+ "prompt_shared_prompts",
88
+ "prompt_shared_sampling_units",
89
+ "prompt_shared_change",
90
+ "prompt_shared_se",
91
+ "prompt_length_distance",
92
+ "prompt_length_delta",
93
+ ]
94
+
95
+
96
+ @dataclass(frozen=True)
97
+ class AnalysisResult:
98
+ features: FeatureTables
99
+ aggregation: Aggregation
100
+ sequential: DetectionResult
101
+ retrospective: RetrospectiveResult
102
+ evidence: pd.DataFrame
103
+ half_assignments: pd.DataFrame
104
+ replication_halves: pd.DataFrame
105
+ corroboration_halves: pd.DataFrame
106
+ persistence_steps: pd.DataFrame
107
+ reward_groups: pd.DataFrame
108
+ prompt_values: pd.DataFrame
109
+ prompt_shifts: pd.DataFrame
110
+ exemplars: pd.DataFrame = field(default_factory=_exemplar_table)
111
+ events: pd.DataFrame = field(default_factory=_event_table)
112
+ event_members: pd.DataFrame = field(default_factory=_member_table)
113
+ event_ranking: pd.DataFrame = field(default_factory=_ranking_table)
114
+ benchmark_metadata: dict | None = None
115
+ sensitivity_metadata: dict | None = None
116
+ retrospective_metadata: dict | None = None
117
+
118
+
119
+ def _effective_options(function, supplied: dict) -> dict:
120
+ parameters = inspect.signature(function).parameters
121
+ values = {
122
+ key: parameter.default
123
+ for key, parameter in parameters.items()
124
+ if parameter.default is not inspect.Parameter.empty
125
+ }
126
+ values.update(supplied)
127
+ return {
128
+ key: value.item() if isinstance(value, np.generic) else value
129
+ for key, value in values.items()
130
+ }
131
+
132
+
133
+ def _benchmark_metadata(vocabulary, sequential, retrospective, sampling_units):
134
+ modules = (
135
+ "pipeline",
136
+ "features",
137
+ "series",
138
+ "detect",
139
+ "retrospective",
140
+ "_retrospective_policy",
141
+ "_sampling",
142
+ "_dependence",
143
+ "_series_validation",
144
+ "_feature_values",
145
+ "schema",
146
+ "evidence",
147
+ "replication",
148
+ "corroboration",
149
+ "events",
150
+ "exemplars",
151
+ "ranking",
152
+ "prompt_mix",
153
+ )
154
+ return {
155
+ "settings": {
156
+ "vocabulary": deepcopy(vocabulary),
157
+ "sequential": _effective_options(detect_sequential, sequential),
158
+ "retrospective": deepcopy(retrospective),
159
+ "sampling_units": "explicit"
160
+ if sampling_units is not None
161
+ else "exact_prompt",
162
+ },
163
+ "analytical_code_sha256s": {
164
+ f"{name}.py": hashlib.sha256(
165
+ Path(__file__).with_name(f"{name}.py").read_bytes()
166
+ ).hexdigest()
167
+ for name in modules
168
+ },
169
+ "runtime": {
170
+ "python": platform.python_version(),
171
+ **{name: version(name) for name in ("numpy", "pandas", "pyarrow")},
172
+ },
173
+ }
174
+
175
+
176
+ def _options(value: Mapping | None, name: str) -> dict:
177
+ if value is None:
178
+ return {}
179
+ if not isinstance(value, Mapping):
180
+ raise ValueError(f"{name} must be a mapping of detector keyword arguments")
181
+ return dict(value)
182
+
183
+
184
+ def _groups(frame: pd.DataFrame) -> dict:
185
+ return dict(tuple(frame.groupby(["run", "feature"], sort=False)))
186
+
187
+
188
+ def _subset_features(features: FeatureTables, rollouts: pd.DataFrame) -> FeatureTables:
189
+ keys = ["run", "sample_id"]
190
+ selected = pd.MultiIndex.from_frame(rollouts[keys])
191
+ return FeatureTables(
192
+ features.dense.loc[
193
+ pd.MultiIndex.from_frame(features.dense[keys]).isin(selected)
194
+ ],
195
+ features.sparse.loc[
196
+ pd.MultiIndex.from_frame(features.sparse[keys]).isin(selected)
197
+ ],
198
+ features.catalog,
199
+ )
200
+
201
+
202
+ def _details(parts: list[pd.DataFrame]) -> pd.DataFrame:
203
+ if not parts:
204
+ return pd.DataFrame(columns=["run", "feature"])
205
+ columns = list(dict.fromkeys(column for part in parts for column in part.columns))
206
+ nonempty = [part for part in parts if not part.empty]
207
+ if not nonempty:
208
+ return parts[0].iloc[:0].reindex(columns=columns).copy()
209
+ # Series concat preserves all-NA object dtypes and exact optional integer IDs.
210
+ data = {}
211
+ for column in columns:
212
+ column_parts = [
213
+ part[column]
214
+ if column in part
215
+ else pd.Series([pd.NA] * len(part), dtype=object)
216
+ for part in nonempty
217
+ ]
218
+ data[column] = pd.concat(column_parts, ignore_index=True)
219
+ return pd.DataFrame(data)
220
+
221
+
222
+ def _aggregate_halves(
223
+ rollouts: pd.DataFrame, features: FeatureTables, units: pd.DataFrame
224
+ ) -> tuple[pd.DataFrame, dict, dict]:
225
+ assignments = units.copy()
226
+ assignments["half"] = assignments.sampling_unit_id.map(
227
+ lambda unit: zlib.crc32(unit.encode("utf-8")) & 1
228
+ )
229
+ half_ids = (
230
+ assignments.set_index(["run", "prompt_id"])
231
+ .half.reindex(pd.MultiIndex.from_frame(rollouts[["run", "prompt_id"]]))
232
+ .to_numpy()
233
+ )
234
+ half_series, half_schedules = {}, {}
235
+ for half in (0, 1):
236
+ subset = rollouts.loc[half_ids == half]
237
+ result = aggregate(
238
+ subset,
239
+ _subset_features(features, subset),
240
+ sampling_units=subset_sampling_units(units, subset),
241
+ )
242
+ half_series[half] = _groups(result.series)
243
+ half_schedules[half] = dict(
244
+ tuple(result.prompt_schedule.groupby("run", sort=False))
245
+ )
246
+
247
+ return assignments, half_series, half_schedules
248
+
249
+
250
+ def _fixed_alert_evidence(
251
+ diagnostic,
252
+ selected_full: DetectionResult,
253
+ half_series: dict,
254
+ half_schedules: dict,
255
+ assignments: pd.DataFrame,
256
+ values: _FeatureValueIndex,
257
+ units: pd.DataFrame,
258
+ *,
259
+ replication_tolerance: int,
260
+ replication_coverage_policy: str,
261
+ corroboration_radius: int,
262
+ sequential_parameters: dict,
263
+ ) -> tuple[dict, dict[str, pd.DataFrame], bool]:
264
+ run, feature = diagnostic.run, diagnostic.feature
265
+ alerts = selected_full.alerts
266
+ alert = alerts.iloc[0]
267
+ onset, alert_step = int(alert.onset_step), int(alert.alert_step)
268
+ before = (int(diagnostic.baseline_start), int(diagnostic.baseline_end))
269
+ after = (onset, alert_step)
270
+ rep = _replication_prepared(
271
+ selected_full,
272
+ half_series,
273
+ half_schedules,
274
+ assignments,
275
+ run=run,
276
+ feature=feature,
277
+ tolerance=replication_tolerance,
278
+ parameters=sequential_parameters,
279
+ coverage_policy=replication_coverage_policy,
280
+ )
281
+ corroboration = _corroboration_prepared(
282
+ selected_full,
283
+ half_series,
284
+ half_schedules,
285
+ assignments,
286
+ run=run,
287
+ feature=feature,
288
+ radius=corroboration_radius,
289
+ parameters=sequential_parameters,
290
+ )
291
+ frame, kind = values.values(run, feature)
292
+ association = _reward_association_prepared(
293
+ frame, kind, run=run, feature=feature, window=after
294
+ )
295
+ mix = _prompt_mix_prepared(
296
+ frame,
297
+ kind,
298
+ run=run,
299
+ feature=feature,
300
+ before=before,
301
+ after=after,
302
+ sampling_units=units,
303
+ )
304
+ length = mix.prompt_shift.loc[mix.prompt_shift.descriptor.eq("prompt_chars")].iloc[
305
+ 0
306
+ ]
307
+ if not mix.shared_prompts:
308
+ mix_status = "no_shared_prompts"
309
+ elif mix.shared_se is None:
310
+ mix_status = "shared_uncertainty_unavailable"
311
+ else:
312
+ mix_status = "available"
313
+ row = {"onset_step": onset, "alert_step": alert_step}
314
+ row.update(
315
+ replication_status=rep.status,
316
+ replicates=rep.replicates,
317
+ replication_horizon_complete=rep.horizon_complete,
318
+ replication_cutoff_step=rep.cutoff_step,
319
+ )
320
+ row.update(
321
+ corroboration_status=corroboration.status,
322
+ corroborates=corroboration.corroborates,
323
+ corroboration_reason=corroboration.reason,
324
+ corroboration_window_start=corroboration.window_start_step,
325
+ corroboration_window_end=corroboration.window_end_step,
326
+ corroboration_horizon_complete=corroboration.horizon_complete,
327
+ )
328
+ row.update(
329
+ reward_status=association.status,
330
+ reward_mean_difference=association.mean_difference,
331
+ reward_n_groups=association.n_groups,
332
+ reward_n_mixed_groups=association.n_mixed_groups,
333
+ reward_n_rollouts=association.n_rollouts,
334
+ reward_n_included_rollouts=association.n_included_rollouts,
335
+ reward_window_start=after[0],
336
+ reward_window_end=after[1],
337
+ )
338
+ row.update(
339
+ promptmix_status=mix_status,
340
+ prompt_before_start=before[0],
341
+ prompt_before_end=before[1],
342
+ prompt_after_start=after[0],
343
+ prompt_after_end=after[1],
344
+ prompt_positive_prompts=mix.positive_prompts,
345
+ prompt_total_prompts=mix.total_prompts,
346
+ prompt_shared_prompts=mix.shared_prompts,
347
+ prompt_shared_sampling_units=mix.shared_sampling_units,
348
+ prompt_shared_change=mix.shared_change,
349
+ prompt_shared_se=mix.shared_se,
350
+ prompt_length_distance=length.distance,
351
+ prompt_length_delta=length.delta,
352
+ )
353
+ details = {
354
+ "replication_halves": rep.halves.assign(run=run, feature=feature),
355
+ "corroboration_halves": corroboration.halves.assign(run=run, feature=feature),
356
+ "reward_groups": association.groups.assign(
357
+ feature=feature, window_start=after[0], window_end=after[1]
358
+ ),
359
+ }
360
+ for name, detail in (
361
+ ("prompt_values", mix.per_prompt),
362
+ ("prompt_shifts", mix.prompt_shift),
363
+ ):
364
+ details[name] = detail.assign(
365
+ run=run,
366
+ feature=feature,
367
+ before_start=before[0],
368
+ before_end=before[1],
369
+ after_start=after[0],
370
+ after_end=after[1],
371
+ )
372
+ return row, details, rep.horizon_complete and corroboration.horizon_complete
373
+
374
+
375
+ def _copy_fixed_evidence(row, details):
376
+ copied = {name: frame.copy(deep=True) for name, frame in details.items()}
377
+ # pandas deep copies retain lists stored in object cells.
378
+ for frame in copied.values():
379
+ for name in frame.select_dtypes(include="object"):
380
+ frame[name] = pd.Series(
381
+ [deepcopy(value) for value in frame[name]],
382
+ index=frame.index,
383
+ dtype=object,
384
+ )
385
+ return deepcopy(row), copied
386
+
387
+
388
+ def _alert_evidence(
389
+ diagnostic,
390
+ selected_full,
391
+ half_series,
392
+ half_schedules,
393
+ assignments,
394
+ values,
395
+ units,
396
+ *,
397
+ replication_tolerance,
398
+ replication_coverage_policy,
399
+ corroboration_radius,
400
+ persistence_effect_fraction,
401
+ sequential_parameters,
402
+ fixed_cache=None,
403
+ cache_context=None,
404
+ ):
405
+ alert = selected_full.alerts.iloc[0]
406
+ run, feature = diagnostic.run, diagnostic.feature
407
+ key = (run, feature)
408
+ steps = selected_full.scores.step.tolist()
409
+ end = steps.index(alert.alert_step) + max(
410
+ replication_tolerance, corroboration_radius
411
+ )
412
+ signature = (
413
+ cache_context,
414
+ tuple(alert.items()),
415
+ tuple(
416
+ (name, getattr(diagnostic, name))
417
+ for name in diagnostic._fields
418
+ if name.startswith("baseline_")
419
+ or name in {"numerical_floor", "scale", "scale_source", "kind"}
420
+ ),
421
+ tuple(steps[: end + 1]),
422
+ replication_tolerance,
423
+ replication_coverage_policy,
424
+ corroboration_radius,
425
+ tuple(sorted(sequential_parameters.items())),
426
+ )
427
+ cached = fixed_cache.get(key) if fixed_cache is not None else None
428
+ if cached is not None and cached[0] == signature:
429
+ row, details = _copy_fixed_evidence(cached[1], cached[2])
430
+ else:
431
+ row, details, complete = _fixed_alert_evidence(
432
+ diagnostic,
433
+ selected_full,
434
+ half_series,
435
+ half_schedules,
436
+ assignments,
437
+ values,
438
+ units,
439
+ replication_tolerance=replication_tolerance,
440
+ replication_coverage_policy=replication_coverage_policy,
441
+ corroboration_radius=corroboration_radius,
442
+ sequential_parameters=sequential_parameters,
443
+ )
444
+ if fixed_cache is not None:
445
+ fixed_cache.pop(key, None)
446
+ if complete:
447
+ fixed_cache[key] = (signature, *_copy_fixed_evidence(row, details))
448
+ retained = _persistence_prepared(
449
+ selected_full.scores,
450
+ run=run,
451
+ feature=feature,
452
+ after_step=int(alert.alert_step),
453
+ reference_mean=float(alert.baseline_mean),
454
+ change=float(alert.delta_at_alert),
455
+ effect_fraction=persistence_effect_fraction,
456
+ )
457
+ row.update(
458
+ persistence_status=retained.status,
459
+ persistence_fraction=retained.fraction_retained,
460
+ persistence_coverage=retained.coverage,
461
+ persistence_n_observations=retained.n_observations,
462
+ persistence_n_evaluable=retained.n_evaluable,
463
+ persistence_n_retained=retained.n_retained,
464
+ persistence_n_unavailable=retained.n_unavailable,
465
+ )
466
+ details["persistence_steps"] = retained.steps.assign(
467
+ after_step=int(alert.alert_step),
468
+ reference_mean=retained.reference_mean,
469
+ reference_change=retained.change,
470
+ effect_fraction=retained.effect_fraction,
471
+ threshold=retained.threshold,
472
+ )
473
+ return row, details
474
+
475
+
476
+ def _analysis_options(
477
+ *,
478
+ sequential_options,
479
+ retrospective,
480
+ retrospective_options,
481
+ replication_tolerance,
482
+ replication_coverage_policy,
483
+ corroboration_radius,
484
+ persistence_effect_fraction,
485
+ ) -> tuple[dict, dict]:
486
+ if type(retrospective) is not bool:
487
+ raise ValueError("retrospective must be a boolean")
488
+ if not retrospective and retrospective_options is not None:
489
+ raise ValueError("retrospective_options require retrospective=True")
490
+ _validate_coverage_policy(replication_coverage_policy)
491
+ if type(corroboration_radius) is not int or corroboration_radius < 0:
492
+ raise ValueError("corroboration_radius must be a nonnegative integer")
493
+ if type(replication_tolerance) is not int or replication_tolerance < 0:
494
+ raise ValueError("replication_tolerance must be a nonnegative integer")
495
+ if (
496
+ isinstance(persistence_effect_fraction, bool)
497
+ or not isinstance(persistence_effect_fraction, Real)
498
+ or not np.isfinite(persistence_effect_fraction)
499
+ or not 0 < persistence_effect_fraction <= 1
500
+ ):
501
+ raise ValueError("persistence_effect_fraction must be in (0, 1]")
502
+ sequential_parameters = _options(sequential_options, "sequential_options")
503
+ retrospective_parameters = _options(retrospective_options, "retrospective_options")
504
+ return sequential_parameters, retrospective_parameters
505
+
506
+
507
+ def analyze_rollouts(
508
+ rollouts: pd.DataFrame,
509
+ *,
510
+ max_sparse_per_tier: int = 2000,
511
+ min_prevalence: float = 0.05,
512
+ sequential_options: Mapping | None = None,
513
+ retrospective: bool = False,
514
+ retrospective_options: Mapping | None = None,
515
+ replication_tolerance: int = 10,
516
+ replication_coverage_policy: str = "contiguous_prefix",
517
+ corroboration_radius: int = 10,
518
+ persistence_effect_fraction: float = 0.5,
519
+ sampling_units: pd.DataFrame | None = None,
520
+ sensitivity: str = "default",
521
+ ) -> AnalysisResult:
522
+ """Screen all retained features and reward, then attach sequential evidence.
523
+
524
+ Select vocabulary once for every run and both CRC32 whole-unit halves.
525
+ Anchor reward and prompt mix to onset through alert, with initial-baseline
526
+ prompt comparison and post-alert persistence. Pass retrospective=True for
527
+ experimental full-run localization.
528
+ """
529
+ sequential_parameters, retrospective_parameters = _analysis_options(
530
+ sequential_options=sequential_options,
531
+ retrospective=retrospective,
532
+ retrospective_options=retrospective_options,
533
+ replication_tolerance=replication_tolerance,
534
+ replication_coverage_policy=replication_coverage_policy,
535
+ corroboration_radius=corroboration_radius,
536
+ persistence_effect_fraction=persistence_effect_fraction,
537
+ )
538
+ validate_sensitivity(
539
+ sensitivity,
540
+ vocabulary={
541
+ "max_sparse_per_tier": max_sparse_per_tier,
542
+ "min_prevalence": min_prevalence,
543
+ },
544
+ sequential=sequential_parameters,
545
+ retrospective=retrospective_parameters,
546
+ evidence={
547
+ "replication_tolerance": replication_tolerance,
548
+ "replication_coverage_policy": replication_coverage_policy,
549
+ "corroboration_radius": corroboration_radius,
550
+ "persistence_effect_fraction": persistence_effect_fraction,
551
+ },
552
+ )
553
+ rollouts = validate_rollouts(rollouts)
554
+ units = resolve_sampling_units(rollouts, sampling_units)
555
+ features = featurize(
556
+ rollouts, max_sparse_per_tier=max_sparse_per_tier, min_prevalence=min_prevalence
557
+ )
558
+ result = _analyze_prepared(
559
+ rollouts,
560
+ features,
561
+ sequential_options={} if sensitivity == "lenient" else sequential_parameters,
562
+ retrospective=retrospective,
563
+ retrospective_options=retrospective_parameters if retrospective else None,
564
+ replication_tolerance=replication_tolerance,
565
+ replication_coverage_policy=replication_coverage_policy,
566
+ corroboration_radius=corroboration_radius,
567
+ persistence_effect_fraction=persistence_effect_fraction,
568
+ sampling_units=units,
569
+ _sensitivity=sensitivity,
570
+ )
571
+ try:
572
+ metadata = _benchmark_metadata(
573
+ {
574
+ "max_sparse_per_tier": max_sparse_per_tier,
575
+ "min_prevalence": min_prevalence,
576
+ },
577
+ LENIENT_SEQUENTIAL if sensitivity == "lenient" else sequential_parameters,
578
+ result.retrospective_metadata["effective_settings"],
579
+ sampling_units,
580
+ )
581
+ if sensitivity == "lenient":
582
+ metadata["sensitivity"] = LENIENT_ID
583
+ metadata["analytical_code_sha256s"]["sensitivity.py"] = hashlib.sha256(
584
+ Path(__file__).with_name("sensitivity.py").read_bytes()
585
+ ).hexdigest()
586
+ except (OSError, PackageNotFoundError):
587
+ metadata = {"sensitivity": LENIENT_ID} if sensitivity == "lenient" else None
588
+ sensitivity_metadata = result.sensitivity_metadata
589
+ if sensitivity_metadata is not None:
590
+ sensitivity_metadata = profile_metadata(
591
+ sensitivity_metadata["counts"],
592
+ retrospective_settings=result.retrospective_metadata["effective_settings"],
593
+ explicit_sampling_units=sampling_units is not None,
594
+ )
595
+ return replace(
596
+ result, benchmark_metadata=metadata, sensitivity_metadata=sensitivity_metadata
597
+ )
598
+
599
+
600
+ def _analyze_prepared(
601
+ rollouts: pd.DataFrame,
602
+ features: FeatureTables,
603
+ *,
604
+ sequential_options: Mapping | None = None,
605
+ retrospective: bool = False,
606
+ retrospective_options: Mapping | None = None,
607
+ replication_tolerance: int = 10,
608
+ replication_coverage_policy: str = "contiguous_prefix",
609
+ corroboration_radius: int = 10,
610
+ persistence_effect_fraction: float = 0.5,
611
+ sampling_units: pd.DataFrame | None = None,
612
+ _fixed_evidence_cache: dict | None = None,
613
+ _fixed_evidence_context: str | None = None,
614
+ _sensitivity: str = "default",
615
+ ) -> AnalysisResult:
616
+ """Analyze retained features without extracting responses again.
617
+
618
+ Fixed evidence reuse requires immutable, append-only watch prefixes under
619
+ the supplied configuration context. Ordinary analysis does not use it.
620
+ """
621
+ sequential_parameters, retrospective_parameters = _analysis_options(
622
+ sequential_options=sequential_options,
623
+ retrospective=retrospective,
624
+ retrospective_options=retrospective_options,
625
+ replication_tolerance=replication_tolerance,
626
+ replication_coverage_policy=replication_coverage_policy,
627
+ corroboration_radius=corroboration_radius,
628
+ persistence_effect_fraction=persistence_effect_fraction,
629
+ )
630
+ if _fixed_evidence_cache is not None and (
631
+ type(_fixed_evidence_cache) is not dict
632
+ or not isinstance(_fixed_evidence_context, str)
633
+ or not _fixed_evidence_context
634
+ ):
635
+ raise ValueError("fixed evidence reuse requires a watch configuration context")
636
+ rollouts = validate_rollouts(rollouts)
637
+ units = resolve_sampling_units(rollouts, sampling_units)
638
+ aggregation = aggregate(rollouts, features, sampling_units=units)
639
+ sequential = detect_sequential(aggregation.series, **sequential_parameters)
640
+ if retrospective:
641
+ retrospective_result = detect_retrospective(
642
+ aggregation.series, **retrospective_parameters
643
+ )
644
+ settings = _effective_options(detect_retrospective, retrospective_parameters)
645
+ else:
646
+ retrospective_result = disabled_result()
647
+ settings = None
648
+ sensitivity_metadata = None
649
+ if _sensitivity == "lenient":
650
+ sequential, counts = select_lenient(aggregation.series, sequential)
651
+ sensitivity_metadata = profile_metadata(
652
+ counts, explicit_sampling_units=True, retrospective_settings=settings
653
+ )
654
+
655
+ half_features = features
656
+ if _fixed_evidence_cache is not None:
657
+ # Only alerts use sparse half-series. Keep dense validation unchanged.
658
+ needed = (set(features.dense.columns) - {"run", "sample_id"}) | set(
659
+ sequential.alerts.feature
660
+ )
661
+ half_features = FeatureTables(
662
+ dense=features.dense,
663
+ sparse=features.sparse.loc[features.sparse.feature.isin(needed)],
664
+ catalog=features.catalog.loc[features.catalog.feature.isin(needed)],
665
+ )
666
+ assignments, half_series, half_schedules = _aggregate_halves(
667
+ rollouts, half_features, units
668
+ )
669
+
670
+ full_scores = _groups(sequential.scores)
671
+ full_alerts = _groups(sequential.alerts)
672
+ if _fixed_evidence_cache is not None:
673
+ for key in set(_fixed_evidence_cache) - set(full_alerts):
674
+ del _fixed_evidence_cache[key]
675
+ retro_diagnostics = retrospective_result.diagnostics.set_index(["run", "feature"])
676
+ changes = retrospective_result.changepoints.set_index(["run", "feature"])
677
+ assignments_by_run = dict(tuple(assignments.groupby("run", sort=False)))
678
+ units_by_run = dict(tuple(units.groupby("run", sort=False)))
679
+ values = _FeatureValueIndex(rollouts, features)
680
+ rows = []
681
+ detail_parts = {
682
+ "replication_halves": [],
683
+ "corroboration_halves": [],
684
+ "persistence_steps": [],
685
+ "reward_groups": [],
686
+ "prompt_values": [],
687
+ "prompt_shifts": [],
688
+ }
689
+ empty_series = aggregation.series.iloc[:0]
690
+ empty_schedule = aggregation.prompt_schedule.iloc[:0]
691
+ for diagnostic in sequential.diagnostics.itertuples(index=False):
692
+ run, feature = diagnostic.run, diagnostic.feature
693
+ key = (run, feature)
694
+ row = dict.fromkeys(_EVIDENCE_COLUMNS)
695
+ row.update(
696
+ run=run,
697
+ feature=feature,
698
+ tier=diagnostic.tier,
699
+ kind=diagnostic.kind,
700
+ sequential_status=diagnostic.status,
701
+ scale_source=diagnostic.scale_source,
702
+ numerical_floor_dominant=bool(diagnostic.numerical_floor_dominant),
703
+ retrospective_status=retro_diagnostics.loc[key, "status"]
704
+ if retrospective
705
+ else "disabled",
706
+ replication_coverage_policy=replication_coverage_policy,
707
+ )
708
+ if key in changes.index:
709
+ row["changepoint_step"] = int(changes.loc[key, "changepoint_step"])
710
+ if key not in full_alerts:
711
+ if diagnostic.status == "profile_excluded":
712
+ row.update(
713
+ replication_status="not_applicable_profile_excluded",
714
+ corroboration_status="unavailable",
715
+ corroboration_reason="profile_excluded",
716
+ persistence_status="not_applicable_profile_excluded",
717
+ reward_status="not_applicable_profile_excluded",
718
+ promptmix_status="not_applicable_profile_excluded",
719
+ )
720
+ rows.append(row)
721
+ continue
722
+ unscorable = diagnostic.n_scored == 0
723
+ unavailable = (
724
+ "not_applicable_full_unscorable"
725
+ if unscorable
726
+ else "not_applicable_no_sequential_alert"
727
+ )
728
+ row.update(
729
+ replication_status="full_unscorable" if unscorable else unavailable,
730
+ corroboration_status="unavailable",
731
+ corroboration_reason="full_unscorable"
732
+ if unscorable
733
+ else "no_full_alert",
734
+ persistence_status=unavailable,
735
+ reward_status=unavailable,
736
+ promptmix_status=unavailable,
737
+ )
738
+ rows.append(row)
739
+ continue
740
+
741
+ selected_full = DetectionResult(
742
+ full_alerts[key],
743
+ pd.DataFrame([diagnostic._asdict()]),
744
+ full_scores[key],
745
+ sequential.screening.iloc[:0],
746
+ )
747
+ alert_row, details = _alert_evidence(
748
+ diagnostic,
749
+ selected_full,
750
+ {half: half_series[half].get(key, empty_series) for half in (0, 1)},
751
+ {half: half_schedules[half].get(run, empty_schedule) for half in (0, 1)},
752
+ assignments_by_run[run],
753
+ values,
754
+ units_by_run[run],
755
+ replication_tolerance=replication_tolerance,
756
+ replication_coverage_policy=replication_coverage_policy,
757
+ corroboration_radius=corroboration_radius,
758
+ persistence_effect_fraction=persistence_effect_fraction,
759
+ sequential_parameters=LENIENT_SEQUENTIAL
760
+ if _sensitivity == "lenient"
761
+ and diagnostic.tier == "structural"
762
+ and diagnostic.kind == "binary"
763
+ else sequential_parameters,
764
+ fixed_cache=_fixed_evidence_cache,
765
+ cache_context=_fixed_evidence_context,
766
+ )
767
+ row.update(alert_row)
768
+ rows.append(row)
769
+ for name, detail in details.items():
770
+ detail_parts[name].append(detail)
771
+
772
+ evidence = pd.DataFrame(rows, columns=_EVIDENCE_COLUMNS)
773
+ # Keep integer step IDs exact when identities are missing.
774
+ for name in [
775
+ "onset_step",
776
+ "alert_step",
777
+ "changepoint_step",
778
+ "replication_cutoff_step",
779
+ "corroboration_window_start",
780
+ "corroboration_window_end",
781
+ "reward_window_start",
782
+ "reward_window_end",
783
+ "prompt_before_start",
784
+ "prompt_before_end",
785
+ "prompt_after_start",
786
+ "prompt_after_end",
787
+ ]:
788
+ evidence[name] = pd.Series([row[name] for row in rows], dtype=object)
789
+ evidence["replicates"] = pd.array(evidence.replicates, dtype="boolean")
790
+ evidence["corroborates"] = pd.array(evidence.corroborates, dtype="boolean")
791
+ evidence["numerical_floor_dominant"] = evidence.numerical_floor_dominant.astype(
792
+ bool
793
+ )
794
+ grouped = _group_events_prepared(values, sequential.alerts)
795
+ return AnalysisResult(
796
+ features=features,
797
+ aggregation=aggregation,
798
+ sequential=sequential,
799
+ retrospective=retrospective_result,
800
+ retrospective_metadata=retrospective_metadata(settings),
801
+ evidence=evidence,
802
+ half_assignments=assignments,
803
+ exemplars=_select_exemplars_prepared(values, sequential.alerts),
804
+ events=grouped.events,
805
+ event_members=grouped.members,
806
+ event_ranking=rank_events(grouped.events, grouped.members, evidence),
807
+ sensitivity_metadata=sensitivity_metadata,
808
+ **{name: _details(parts) for name, parts in detail_parts.items()},
809
+ )