rolloutdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rolloutdiff/__init__.py +130 -0
- rolloutdiff/_aria_replay.py +346 -0
- rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
- rolloutdiff/_dependence.py +195 -0
- rolloutdiff/_feature_values.py +79 -0
- rolloutdiff/_replay_container.py +568 -0
- rolloutdiff/_report_validation.py +758 -0
- rolloutdiff/_retrospective_policy.py +33 -0
- rolloutdiff/_sampling.py +127 -0
- rolloutdiff/_series_validation.py +67 -0
- rolloutdiff/_watch_report.py +436 -0
- rolloutdiff/_watch_storage.py +800 -0
- rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
- rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
- rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
- rolloutdiff/_web/assets/third-party-notices.txt +112 -0
- rolloutdiff/_web/index.html +14 -0
- rolloutdiff/adapters.py +264 -0
- rolloutdiff/app.py +956 -0
- rolloutdiff/app_evidence.py +678 -0
- rolloutdiff/benchmark_reference.py +458 -0
- rolloutdiff/cli.py +1292 -0
- rolloutdiff/comparison.py +572 -0
- rolloutdiff/comparison_report.py +982 -0
- rolloutdiff/corroboration.py +287 -0
- rolloutdiff/detect.py +307 -0
- rolloutdiff/events.py +173 -0
- rolloutdiff/evidence.py +285 -0
- rolloutdiff/exemplars.py +240 -0
- rolloutdiff/features.py +823 -0
- rolloutdiff/html_report.py +726 -0
- rolloutdiff/inspect_logs.py +383 -0
- rolloutdiff/inspection.py +207 -0
- rolloutdiff/integrations.py +552 -0
- rolloutdiff/investigation.py +1146 -0
- rolloutdiff/notifications.py +406 -0
- rolloutdiff/pipeline.py +809 -0
- rolloutdiff/producer.py +663 -0
- rolloutdiff/prompt_mix.py +258 -0
- rolloutdiff/ranking.py +388 -0
- rolloutdiff/replay.py +589 -0
- rolloutdiff/replication.py +572 -0
- rolloutdiff/report.py +838 -0
- rolloutdiff/retrospective.py +386 -0
- rolloutdiff/schema.py +109 -0
- rolloutdiff/sensitivity.py +202 -0
- rolloutdiff/series.py +430 -0
- rolloutdiff/tool_features.py +422 -0
- rolloutdiff/trajectories.py +236 -0
- rolloutdiff/watch-compatibility.json +13 -0
- rolloutdiff/watch.py +742 -0
- rolloutdiff-0.1.0.dist-info/METADATA +158 -0
- rolloutdiff-0.1.0.dist-info/RECORD +62 -0
- rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
- rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
- rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
- rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
rolloutdiff/pipeline.py
ADDED
|
@@ -0,0 +1,809 @@
|
|
|
1
|
+
"""Analyze every retained feature with shared aggregation and evidence policies."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import inspect
|
|
7
|
+
import platform
|
|
8
|
+
import zlib
|
|
9
|
+
from collections.abc import Mapping
|
|
10
|
+
from copy import deepcopy
|
|
11
|
+
from dataclasses import dataclass, field, replace
|
|
12
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
13
|
+
from numbers import Real
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
import numpy as np
|
|
17
|
+
import pandas as pd
|
|
18
|
+
|
|
19
|
+
from ._feature_values import _FeatureValueIndex
|
|
20
|
+
from ._retrospective_policy import disabled_result, retrospective_metadata
|
|
21
|
+
from ._sampling import resolve_sampling_units, subset_sampling_units
|
|
22
|
+
from .corroboration import _corroboration_prepared
|
|
23
|
+
from .detect import DetectionResult, detect_sequential
|
|
24
|
+
from .events import _event_table, _group_events_prepared, _member_table
|
|
25
|
+
from .evidence import _persistence_prepared, _reward_association_prepared
|
|
26
|
+
from .exemplars import _exemplar_table, _select_exemplars_prepared
|
|
27
|
+
from .features import FeatureTables, featurize
|
|
28
|
+
from .prompt_mix import _prompt_mix_prepared
|
|
29
|
+
from .ranking import _ranking_table, rank_events
|
|
30
|
+
from .replication import _replication_prepared, _validate_coverage_policy
|
|
31
|
+
from .retrospective import RetrospectiveResult, detect_retrospective
|
|
32
|
+
from .schema import validate_rollouts
|
|
33
|
+
from .sensitivity import (
|
|
34
|
+
LENIENT_ID,
|
|
35
|
+
LENIENT_SEQUENTIAL,
|
|
36
|
+
profile_metadata,
|
|
37
|
+
select_lenient,
|
|
38
|
+
validate_sensitivity,
|
|
39
|
+
)
|
|
40
|
+
from .series import Aggregation, aggregate
|
|
41
|
+
|
|
42
|
+
_EVIDENCE_COLUMNS = [
|
|
43
|
+
"run",
|
|
44
|
+
"feature",
|
|
45
|
+
"tier",
|
|
46
|
+
"kind",
|
|
47
|
+
"sequential_status",
|
|
48
|
+
"scale_source",
|
|
49
|
+
"numerical_floor_dominant",
|
|
50
|
+
"retrospective_status",
|
|
51
|
+
"onset_step",
|
|
52
|
+
"alert_step",
|
|
53
|
+
"changepoint_step",
|
|
54
|
+
"replication_status",
|
|
55
|
+
"replication_coverage_policy",
|
|
56
|
+
"replicates",
|
|
57
|
+
"replication_horizon_complete",
|
|
58
|
+
"replication_cutoff_step",
|
|
59
|
+
"corroboration_status",
|
|
60
|
+
"corroborates",
|
|
61
|
+
"corroboration_reason",
|
|
62
|
+
"corroboration_window_start",
|
|
63
|
+
"corroboration_window_end",
|
|
64
|
+
"corroboration_horizon_complete",
|
|
65
|
+
"persistence_status",
|
|
66
|
+
"persistence_fraction",
|
|
67
|
+
"persistence_coverage",
|
|
68
|
+
"persistence_n_observations",
|
|
69
|
+
"persistence_n_evaluable",
|
|
70
|
+
"persistence_n_retained",
|
|
71
|
+
"persistence_n_unavailable",
|
|
72
|
+
"reward_status",
|
|
73
|
+
"reward_mean_difference",
|
|
74
|
+
"reward_n_groups",
|
|
75
|
+
"reward_n_mixed_groups",
|
|
76
|
+
"reward_n_rollouts",
|
|
77
|
+
"reward_n_included_rollouts",
|
|
78
|
+
"reward_window_start",
|
|
79
|
+
"reward_window_end",
|
|
80
|
+
"promptmix_status",
|
|
81
|
+
"prompt_before_start",
|
|
82
|
+
"prompt_before_end",
|
|
83
|
+
"prompt_after_start",
|
|
84
|
+
"prompt_after_end",
|
|
85
|
+
"prompt_positive_prompts",
|
|
86
|
+
"prompt_total_prompts",
|
|
87
|
+
"prompt_shared_prompts",
|
|
88
|
+
"prompt_shared_sampling_units",
|
|
89
|
+
"prompt_shared_change",
|
|
90
|
+
"prompt_shared_se",
|
|
91
|
+
"prompt_length_distance",
|
|
92
|
+
"prompt_length_delta",
|
|
93
|
+
]
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@dataclass(frozen=True)
|
|
97
|
+
class AnalysisResult:
|
|
98
|
+
features: FeatureTables
|
|
99
|
+
aggregation: Aggregation
|
|
100
|
+
sequential: DetectionResult
|
|
101
|
+
retrospective: RetrospectiveResult
|
|
102
|
+
evidence: pd.DataFrame
|
|
103
|
+
half_assignments: pd.DataFrame
|
|
104
|
+
replication_halves: pd.DataFrame
|
|
105
|
+
corroboration_halves: pd.DataFrame
|
|
106
|
+
persistence_steps: pd.DataFrame
|
|
107
|
+
reward_groups: pd.DataFrame
|
|
108
|
+
prompt_values: pd.DataFrame
|
|
109
|
+
prompt_shifts: pd.DataFrame
|
|
110
|
+
exemplars: pd.DataFrame = field(default_factory=_exemplar_table)
|
|
111
|
+
events: pd.DataFrame = field(default_factory=_event_table)
|
|
112
|
+
event_members: pd.DataFrame = field(default_factory=_member_table)
|
|
113
|
+
event_ranking: pd.DataFrame = field(default_factory=_ranking_table)
|
|
114
|
+
benchmark_metadata: dict | None = None
|
|
115
|
+
sensitivity_metadata: dict | None = None
|
|
116
|
+
retrospective_metadata: dict | None = None
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _effective_options(function, supplied: dict) -> dict:
|
|
120
|
+
parameters = inspect.signature(function).parameters
|
|
121
|
+
values = {
|
|
122
|
+
key: parameter.default
|
|
123
|
+
for key, parameter in parameters.items()
|
|
124
|
+
if parameter.default is not inspect.Parameter.empty
|
|
125
|
+
}
|
|
126
|
+
values.update(supplied)
|
|
127
|
+
return {
|
|
128
|
+
key: value.item() if isinstance(value, np.generic) else value
|
|
129
|
+
for key, value in values.items()
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _benchmark_metadata(vocabulary, sequential, retrospective, sampling_units):
|
|
134
|
+
modules = (
|
|
135
|
+
"pipeline",
|
|
136
|
+
"features",
|
|
137
|
+
"series",
|
|
138
|
+
"detect",
|
|
139
|
+
"retrospective",
|
|
140
|
+
"_retrospective_policy",
|
|
141
|
+
"_sampling",
|
|
142
|
+
"_dependence",
|
|
143
|
+
"_series_validation",
|
|
144
|
+
"_feature_values",
|
|
145
|
+
"schema",
|
|
146
|
+
"evidence",
|
|
147
|
+
"replication",
|
|
148
|
+
"corroboration",
|
|
149
|
+
"events",
|
|
150
|
+
"exemplars",
|
|
151
|
+
"ranking",
|
|
152
|
+
"prompt_mix",
|
|
153
|
+
)
|
|
154
|
+
return {
|
|
155
|
+
"settings": {
|
|
156
|
+
"vocabulary": deepcopy(vocabulary),
|
|
157
|
+
"sequential": _effective_options(detect_sequential, sequential),
|
|
158
|
+
"retrospective": deepcopy(retrospective),
|
|
159
|
+
"sampling_units": "explicit"
|
|
160
|
+
if sampling_units is not None
|
|
161
|
+
else "exact_prompt",
|
|
162
|
+
},
|
|
163
|
+
"analytical_code_sha256s": {
|
|
164
|
+
f"{name}.py": hashlib.sha256(
|
|
165
|
+
Path(__file__).with_name(f"{name}.py").read_bytes()
|
|
166
|
+
).hexdigest()
|
|
167
|
+
for name in modules
|
|
168
|
+
},
|
|
169
|
+
"runtime": {
|
|
170
|
+
"python": platform.python_version(),
|
|
171
|
+
**{name: version(name) for name in ("numpy", "pandas", "pyarrow")},
|
|
172
|
+
},
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _options(value: Mapping | None, name: str) -> dict:
|
|
177
|
+
if value is None:
|
|
178
|
+
return {}
|
|
179
|
+
if not isinstance(value, Mapping):
|
|
180
|
+
raise ValueError(f"{name} must be a mapping of detector keyword arguments")
|
|
181
|
+
return dict(value)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _groups(frame: pd.DataFrame) -> dict:
|
|
185
|
+
return dict(tuple(frame.groupby(["run", "feature"], sort=False)))
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _subset_features(features: FeatureTables, rollouts: pd.DataFrame) -> FeatureTables:
|
|
189
|
+
keys = ["run", "sample_id"]
|
|
190
|
+
selected = pd.MultiIndex.from_frame(rollouts[keys])
|
|
191
|
+
return FeatureTables(
|
|
192
|
+
features.dense.loc[
|
|
193
|
+
pd.MultiIndex.from_frame(features.dense[keys]).isin(selected)
|
|
194
|
+
],
|
|
195
|
+
features.sparse.loc[
|
|
196
|
+
pd.MultiIndex.from_frame(features.sparse[keys]).isin(selected)
|
|
197
|
+
],
|
|
198
|
+
features.catalog,
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _details(parts: list[pd.DataFrame]) -> pd.DataFrame:
|
|
203
|
+
if not parts:
|
|
204
|
+
return pd.DataFrame(columns=["run", "feature"])
|
|
205
|
+
columns = list(dict.fromkeys(column for part in parts for column in part.columns))
|
|
206
|
+
nonempty = [part for part in parts if not part.empty]
|
|
207
|
+
if not nonempty:
|
|
208
|
+
return parts[0].iloc[:0].reindex(columns=columns).copy()
|
|
209
|
+
# Series concat preserves all-NA object dtypes and exact optional integer IDs.
|
|
210
|
+
data = {}
|
|
211
|
+
for column in columns:
|
|
212
|
+
column_parts = [
|
|
213
|
+
part[column]
|
|
214
|
+
if column in part
|
|
215
|
+
else pd.Series([pd.NA] * len(part), dtype=object)
|
|
216
|
+
for part in nonempty
|
|
217
|
+
]
|
|
218
|
+
data[column] = pd.concat(column_parts, ignore_index=True)
|
|
219
|
+
return pd.DataFrame(data)
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _aggregate_halves(
|
|
223
|
+
rollouts: pd.DataFrame, features: FeatureTables, units: pd.DataFrame
|
|
224
|
+
) -> tuple[pd.DataFrame, dict, dict]:
|
|
225
|
+
assignments = units.copy()
|
|
226
|
+
assignments["half"] = assignments.sampling_unit_id.map(
|
|
227
|
+
lambda unit: zlib.crc32(unit.encode("utf-8")) & 1
|
|
228
|
+
)
|
|
229
|
+
half_ids = (
|
|
230
|
+
assignments.set_index(["run", "prompt_id"])
|
|
231
|
+
.half.reindex(pd.MultiIndex.from_frame(rollouts[["run", "prompt_id"]]))
|
|
232
|
+
.to_numpy()
|
|
233
|
+
)
|
|
234
|
+
half_series, half_schedules = {}, {}
|
|
235
|
+
for half in (0, 1):
|
|
236
|
+
subset = rollouts.loc[half_ids == half]
|
|
237
|
+
result = aggregate(
|
|
238
|
+
subset,
|
|
239
|
+
_subset_features(features, subset),
|
|
240
|
+
sampling_units=subset_sampling_units(units, subset),
|
|
241
|
+
)
|
|
242
|
+
half_series[half] = _groups(result.series)
|
|
243
|
+
half_schedules[half] = dict(
|
|
244
|
+
tuple(result.prompt_schedule.groupby("run", sort=False))
|
|
245
|
+
)
|
|
246
|
+
|
|
247
|
+
return assignments, half_series, half_schedules
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _fixed_alert_evidence(
|
|
251
|
+
diagnostic,
|
|
252
|
+
selected_full: DetectionResult,
|
|
253
|
+
half_series: dict,
|
|
254
|
+
half_schedules: dict,
|
|
255
|
+
assignments: pd.DataFrame,
|
|
256
|
+
values: _FeatureValueIndex,
|
|
257
|
+
units: pd.DataFrame,
|
|
258
|
+
*,
|
|
259
|
+
replication_tolerance: int,
|
|
260
|
+
replication_coverage_policy: str,
|
|
261
|
+
corroboration_radius: int,
|
|
262
|
+
sequential_parameters: dict,
|
|
263
|
+
) -> tuple[dict, dict[str, pd.DataFrame], bool]:
|
|
264
|
+
run, feature = diagnostic.run, diagnostic.feature
|
|
265
|
+
alerts = selected_full.alerts
|
|
266
|
+
alert = alerts.iloc[0]
|
|
267
|
+
onset, alert_step = int(alert.onset_step), int(alert.alert_step)
|
|
268
|
+
before = (int(diagnostic.baseline_start), int(diagnostic.baseline_end))
|
|
269
|
+
after = (onset, alert_step)
|
|
270
|
+
rep = _replication_prepared(
|
|
271
|
+
selected_full,
|
|
272
|
+
half_series,
|
|
273
|
+
half_schedules,
|
|
274
|
+
assignments,
|
|
275
|
+
run=run,
|
|
276
|
+
feature=feature,
|
|
277
|
+
tolerance=replication_tolerance,
|
|
278
|
+
parameters=sequential_parameters,
|
|
279
|
+
coverage_policy=replication_coverage_policy,
|
|
280
|
+
)
|
|
281
|
+
corroboration = _corroboration_prepared(
|
|
282
|
+
selected_full,
|
|
283
|
+
half_series,
|
|
284
|
+
half_schedules,
|
|
285
|
+
assignments,
|
|
286
|
+
run=run,
|
|
287
|
+
feature=feature,
|
|
288
|
+
radius=corroboration_radius,
|
|
289
|
+
parameters=sequential_parameters,
|
|
290
|
+
)
|
|
291
|
+
frame, kind = values.values(run, feature)
|
|
292
|
+
association = _reward_association_prepared(
|
|
293
|
+
frame, kind, run=run, feature=feature, window=after
|
|
294
|
+
)
|
|
295
|
+
mix = _prompt_mix_prepared(
|
|
296
|
+
frame,
|
|
297
|
+
kind,
|
|
298
|
+
run=run,
|
|
299
|
+
feature=feature,
|
|
300
|
+
before=before,
|
|
301
|
+
after=after,
|
|
302
|
+
sampling_units=units,
|
|
303
|
+
)
|
|
304
|
+
length = mix.prompt_shift.loc[mix.prompt_shift.descriptor.eq("prompt_chars")].iloc[
|
|
305
|
+
0
|
|
306
|
+
]
|
|
307
|
+
if not mix.shared_prompts:
|
|
308
|
+
mix_status = "no_shared_prompts"
|
|
309
|
+
elif mix.shared_se is None:
|
|
310
|
+
mix_status = "shared_uncertainty_unavailable"
|
|
311
|
+
else:
|
|
312
|
+
mix_status = "available"
|
|
313
|
+
row = {"onset_step": onset, "alert_step": alert_step}
|
|
314
|
+
row.update(
|
|
315
|
+
replication_status=rep.status,
|
|
316
|
+
replicates=rep.replicates,
|
|
317
|
+
replication_horizon_complete=rep.horizon_complete,
|
|
318
|
+
replication_cutoff_step=rep.cutoff_step,
|
|
319
|
+
)
|
|
320
|
+
row.update(
|
|
321
|
+
corroboration_status=corroboration.status,
|
|
322
|
+
corroborates=corroboration.corroborates,
|
|
323
|
+
corroboration_reason=corroboration.reason,
|
|
324
|
+
corroboration_window_start=corroboration.window_start_step,
|
|
325
|
+
corroboration_window_end=corroboration.window_end_step,
|
|
326
|
+
corroboration_horizon_complete=corroboration.horizon_complete,
|
|
327
|
+
)
|
|
328
|
+
row.update(
|
|
329
|
+
reward_status=association.status,
|
|
330
|
+
reward_mean_difference=association.mean_difference,
|
|
331
|
+
reward_n_groups=association.n_groups,
|
|
332
|
+
reward_n_mixed_groups=association.n_mixed_groups,
|
|
333
|
+
reward_n_rollouts=association.n_rollouts,
|
|
334
|
+
reward_n_included_rollouts=association.n_included_rollouts,
|
|
335
|
+
reward_window_start=after[0],
|
|
336
|
+
reward_window_end=after[1],
|
|
337
|
+
)
|
|
338
|
+
row.update(
|
|
339
|
+
promptmix_status=mix_status,
|
|
340
|
+
prompt_before_start=before[0],
|
|
341
|
+
prompt_before_end=before[1],
|
|
342
|
+
prompt_after_start=after[0],
|
|
343
|
+
prompt_after_end=after[1],
|
|
344
|
+
prompt_positive_prompts=mix.positive_prompts,
|
|
345
|
+
prompt_total_prompts=mix.total_prompts,
|
|
346
|
+
prompt_shared_prompts=mix.shared_prompts,
|
|
347
|
+
prompt_shared_sampling_units=mix.shared_sampling_units,
|
|
348
|
+
prompt_shared_change=mix.shared_change,
|
|
349
|
+
prompt_shared_se=mix.shared_se,
|
|
350
|
+
prompt_length_distance=length.distance,
|
|
351
|
+
prompt_length_delta=length.delta,
|
|
352
|
+
)
|
|
353
|
+
details = {
|
|
354
|
+
"replication_halves": rep.halves.assign(run=run, feature=feature),
|
|
355
|
+
"corroboration_halves": corroboration.halves.assign(run=run, feature=feature),
|
|
356
|
+
"reward_groups": association.groups.assign(
|
|
357
|
+
feature=feature, window_start=after[0], window_end=after[1]
|
|
358
|
+
),
|
|
359
|
+
}
|
|
360
|
+
for name, detail in (
|
|
361
|
+
("prompt_values", mix.per_prompt),
|
|
362
|
+
("prompt_shifts", mix.prompt_shift),
|
|
363
|
+
):
|
|
364
|
+
details[name] = detail.assign(
|
|
365
|
+
run=run,
|
|
366
|
+
feature=feature,
|
|
367
|
+
before_start=before[0],
|
|
368
|
+
before_end=before[1],
|
|
369
|
+
after_start=after[0],
|
|
370
|
+
after_end=after[1],
|
|
371
|
+
)
|
|
372
|
+
return row, details, rep.horizon_complete and corroboration.horizon_complete
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def _copy_fixed_evidence(row, details):
|
|
376
|
+
copied = {name: frame.copy(deep=True) for name, frame in details.items()}
|
|
377
|
+
# pandas deep copies retain lists stored in object cells.
|
|
378
|
+
for frame in copied.values():
|
|
379
|
+
for name in frame.select_dtypes(include="object"):
|
|
380
|
+
frame[name] = pd.Series(
|
|
381
|
+
[deepcopy(value) for value in frame[name]],
|
|
382
|
+
index=frame.index,
|
|
383
|
+
dtype=object,
|
|
384
|
+
)
|
|
385
|
+
return deepcopy(row), copied
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def _alert_evidence(
|
|
389
|
+
diagnostic,
|
|
390
|
+
selected_full,
|
|
391
|
+
half_series,
|
|
392
|
+
half_schedules,
|
|
393
|
+
assignments,
|
|
394
|
+
values,
|
|
395
|
+
units,
|
|
396
|
+
*,
|
|
397
|
+
replication_tolerance,
|
|
398
|
+
replication_coverage_policy,
|
|
399
|
+
corroboration_radius,
|
|
400
|
+
persistence_effect_fraction,
|
|
401
|
+
sequential_parameters,
|
|
402
|
+
fixed_cache=None,
|
|
403
|
+
cache_context=None,
|
|
404
|
+
):
|
|
405
|
+
alert = selected_full.alerts.iloc[0]
|
|
406
|
+
run, feature = diagnostic.run, diagnostic.feature
|
|
407
|
+
key = (run, feature)
|
|
408
|
+
steps = selected_full.scores.step.tolist()
|
|
409
|
+
end = steps.index(alert.alert_step) + max(
|
|
410
|
+
replication_tolerance, corroboration_radius
|
|
411
|
+
)
|
|
412
|
+
signature = (
|
|
413
|
+
cache_context,
|
|
414
|
+
tuple(alert.items()),
|
|
415
|
+
tuple(
|
|
416
|
+
(name, getattr(diagnostic, name))
|
|
417
|
+
for name in diagnostic._fields
|
|
418
|
+
if name.startswith("baseline_")
|
|
419
|
+
or name in {"numerical_floor", "scale", "scale_source", "kind"}
|
|
420
|
+
),
|
|
421
|
+
tuple(steps[: end + 1]),
|
|
422
|
+
replication_tolerance,
|
|
423
|
+
replication_coverage_policy,
|
|
424
|
+
corroboration_radius,
|
|
425
|
+
tuple(sorted(sequential_parameters.items())),
|
|
426
|
+
)
|
|
427
|
+
cached = fixed_cache.get(key) if fixed_cache is not None else None
|
|
428
|
+
if cached is not None and cached[0] == signature:
|
|
429
|
+
row, details = _copy_fixed_evidence(cached[1], cached[2])
|
|
430
|
+
else:
|
|
431
|
+
row, details, complete = _fixed_alert_evidence(
|
|
432
|
+
diagnostic,
|
|
433
|
+
selected_full,
|
|
434
|
+
half_series,
|
|
435
|
+
half_schedules,
|
|
436
|
+
assignments,
|
|
437
|
+
values,
|
|
438
|
+
units,
|
|
439
|
+
replication_tolerance=replication_tolerance,
|
|
440
|
+
replication_coverage_policy=replication_coverage_policy,
|
|
441
|
+
corroboration_radius=corroboration_radius,
|
|
442
|
+
sequential_parameters=sequential_parameters,
|
|
443
|
+
)
|
|
444
|
+
if fixed_cache is not None:
|
|
445
|
+
fixed_cache.pop(key, None)
|
|
446
|
+
if complete:
|
|
447
|
+
fixed_cache[key] = (signature, *_copy_fixed_evidence(row, details))
|
|
448
|
+
retained = _persistence_prepared(
|
|
449
|
+
selected_full.scores,
|
|
450
|
+
run=run,
|
|
451
|
+
feature=feature,
|
|
452
|
+
after_step=int(alert.alert_step),
|
|
453
|
+
reference_mean=float(alert.baseline_mean),
|
|
454
|
+
change=float(alert.delta_at_alert),
|
|
455
|
+
effect_fraction=persistence_effect_fraction,
|
|
456
|
+
)
|
|
457
|
+
row.update(
|
|
458
|
+
persistence_status=retained.status,
|
|
459
|
+
persistence_fraction=retained.fraction_retained,
|
|
460
|
+
persistence_coverage=retained.coverage,
|
|
461
|
+
persistence_n_observations=retained.n_observations,
|
|
462
|
+
persistence_n_evaluable=retained.n_evaluable,
|
|
463
|
+
persistence_n_retained=retained.n_retained,
|
|
464
|
+
persistence_n_unavailable=retained.n_unavailable,
|
|
465
|
+
)
|
|
466
|
+
details["persistence_steps"] = retained.steps.assign(
|
|
467
|
+
after_step=int(alert.alert_step),
|
|
468
|
+
reference_mean=retained.reference_mean,
|
|
469
|
+
reference_change=retained.change,
|
|
470
|
+
effect_fraction=retained.effect_fraction,
|
|
471
|
+
threshold=retained.threshold,
|
|
472
|
+
)
|
|
473
|
+
return row, details
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
def _analysis_options(
|
|
477
|
+
*,
|
|
478
|
+
sequential_options,
|
|
479
|
+
retrospective,
|
|
480
|
+
retrospective_options,
|
|
481
|
+
replication_tolerance,
|
|
482
|
+
replication_coverage_policy,
|
|
483
|
+
corroboration_radius,
|
|
484
|
+
persistence_effect_fraction,
|
|
485
|
+
) -> tuple[dict, dict]:
|
|
486
|
+
if type(retrospective) is not bool:
|
|
487
|
+
raise ValueError("retrospective must be a boolean")
|
|
488
|
+
if not retrospective and retrospective_options is not None:
|
|
489
|
+
raise ValueError("retrospective_options require retrospective=True")
|
|
490
|
+
_validate_coverage_policy(replication_coverage_policy)
|
|
491
|
+
if type(corroboration_radius) is not int or corroboration_radius < 0:
|
|
492
|
+
raise ValueError("corroboration_radius must be a nonnegative integer")
|
|
493
|
+
if type(replication_tolerance) is not int or replication_tolerance < 0:
|
|
494
|
+
raise ValueError("replication_tolerance must be a nonnegative integer")
|
|
495
|
+
if (
|
|
496
|
+
isinstance(persistence_effect_fraction, bool)
|
|
497
|
+
or not isinstance(persistence_effect_fraction, Real)
|
|
498
|
+
or not np.isfinite(persistence_effect_fraction)
|
|
499
|
+
or not 0 < persistence_effect_fraction <= 1
|
|
500
|
+
):
|
|
501
|
+
raise ValueError("persistence_effect_fraction must be in (0, 1]")
|
|
502
|
+
sequential_parameters = _options(sequential_options, "sequential_options")
|
|
503
|
+
retrospective_parameters = _options(retrospective_options, "retrospective_options")
|
|
504
|
+
return sequential_parameters, retrospective_parameters
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
def analyze_rollouts(
|
|
508
|
+
rollouts: pd.DataFrame,
|
|
509
|
+
*,
|
|
510
|
+
max_sparse_per_tier: int = 2000,
|
|
511
|
+
min_prevalence: float = 0.05,
|
|
512
|
+
sequential_options: Mapping | None = None,
|
|
513
|
+
retrospective: bool = False,
|
|
514
|
+
retrospective_options: Mapping | None = None,
|
|
515
|
+
replication_tolerance: int = 10,
|
|
516
|
+
replication_coverage_policy: str = "contiguous_prefix",
|
|
517
|
+
corroboration_radius: int = 10,
|
|
518
|
+
persistence_effect_fraction: float = 0.5,
|
|
519
|
+
sampling_units: pd.DataFrame | None = None,
|
|
520
|
+
sensitivity: str = "default",
|
|
521
|
+
) -> AnalysisResult:
|
|
522
|
+
"""Screen all retained features and reward, then attach sequential evidence.
|
|
523
|
+
|
|
524
|
+
Select vocabulary once for every run and both CRC32 whole-unit halves.
|
|
525
|
+
Anchor reward and prompt mix to onset through alert, with initial-baseline
|
|
526
|
+
prompt comparison and post-alert persistence. Pass retrospective=True for
|
|
527
|
+
experimental full-run localization.
|
|
528
|
+
"""
|
|
529
|
+
sequential_parameters, retrospective_parameters = _analysis_options(
|
|
530
|
+
sequential_options=sequential_options,
|
|
531
|
+
retrospective=retrospective,
|
|
532
|
+
retrospective_options=retrospective_options,
|
|
533
|
+
replication_tolerance=replication_tolerance,
|
|
534
|
+
replication_coverage_policy=replication_coverage_policy,
|
|
535
|
+
corroboration_radius=corroboration_radius,
|
|
536
|
+
persistence_effect_fraction=persistence_effect_fraction,
|
|
537
|
+
)
|
|
538
|
+
validate_sensitivity(
|
|
539
|
+
sensitivity,
|
|
540
|
+
vocabulary={
|
|
541
|
+
"max_sparse_per_tier": max_sparse_per_tier,
|
|
542
|
+
"min_prevalence": min_prevalence,
|
|
543
|
+
},
|
|
544
|
+
sequential=sequential_parameters,
|
|
545
|
+
retrospective=retrospective_parameters,
|
|
546
|
+
evidence={
|
|
547
|
+
"replication_tolerance": replication_tolerance,
|
|
548
|
+
"replication_coverage_policy": replication_coverage_policy,
|
|
549
|
+
"corroboration_radius": corroboration_radius,
|
|
550
|
+
"persistence_effect_fraction": persistence_effect_fraction,
|
|
551
|
+
},
|
|
552
|
+
)
|
|
553
|
+
rollouts = validate_rollouts(rollouts)
|
|
554
|
+
units = resolve_sampling_units(rollouts, sampling_units)
|
|
555
|
+
features = featurize(
|
|
556
|
+
rollouts, max_sparse_per_tier=max_sparse_per_tier, min_prevalence=min_prevalence
|
|
557
|
+
)
|
|
558
|
+
result = _analyze_prepared(
|
|
559
|
+
rollouts,
|
|
560
|
+
features,
|
|
561
|
+
sequential_options={} if sensitivity == "lenient" else sequential_parameters,
|
|
562
|
+
retrospective=retrospective,
|
|
563
|
+
retrospective_options=retrospective_parameters if retrospective else None,
|
|
564
|
+
replication_tolerance=replication_tolerance,
|
|
565
|
+
replication_coverage_policy=replication_coverage_policy,
|
|
566
|
+
corroboration_radius=corroboration_radius,
|
|
567
|
+
persistence_effect_fraction=persistence_effect_fraction,
|
|
568
|
+
sampling_units=units,
|
|
569
|
+
_sensitivity=sensitivity,
|
|
570
|
+
)
|
|
571
|
+
try:
|
|
572
|
+
metadata = _benchmark_metadata(
|
|
573
|
+
{
|
|
574
|
+
"max_sparse_per_tier": max_sparse_per_tier,
|
|
575
|
+
"min_prevalence": min_prevalence,
|
|
576
|
+
},
|
|
577
|
+
LENIENT_SEQUENTIAL if sensitivity == "lenient" else sequential_parameters,
|
|
578
|
+
result.retrospective_metadata["effective_settings"],
|
|
579
|
+
sampling_units,
|
|
580
|
+
)
|
|
581
|
+
if sensitivity == "lenient":
|
|
582
|
+
metadata["sensitivity"] = LENIENT_ID
|
|
583
|
+
metadata["analytical_code_sha256s"]["sensitivity.py"] = hashlib.sha256(
|
|
584
|
+
Path(__file__).with_name("sensitivity.py").read_bytes()
|
|
585
|
+
).hexdigest()
|
|
586
|
+
except (OSError, PackageNotFoundError):
|
|
587
|
+
metadata = {"sensitivity": LENIENT_ID} if sensitivity == "lenient" else None
|
|
588
|
+
sensitivity_metadata = result.sensitivity_metadata
|
|
589
|
+
if sensitivity_metadata is not None:
|
|
590
|
+
sensitivity_metadata = profile_metadata(
|
|
591
|
+
sensitivity_metadata["counts"],
|
|
592
|
+
retrospective_settings=result.retrospective_metadata["effective_settings"],
|
|
593
|
+
explicit_sampling_units=sampling_units is not None,
|
|
594
|
+
)
|
|
595
|
+
return replace(
|
|
596
|
+
result, benchmark_metadata=metadata, sensitivity_metadata=sensitivity_metadata
|
|
597
|
+
)
|
|
598
|
+
|
|
599
|
+
|
|
600
|
+
def _analyze_prepared(
|
|
601
|
+
rollouts: pd.DataFrame,
|
|
602
|
+
features: FeatureTables,
|
|
603
|
+
*,
|
|
604
|
+
sequential_options: Mapping | None = None,
|
|
605
|
+
retrospective: bool = False,
|
|
606
|
+
retrospective_options: Mapping | None = None,
|
|
607
|
+
replication_tolerance: int = 10,
|
|
608
|
+
replication_coverage_policy: str = "contiguous_prefix",
|
|
609
|
+
corroboration_radius: int = 10,
|
|
610
|
+
persistence_effect_fraction: float = 0.5,
|
|
611
|
+
sampling_units: pd.DataFrame | None = None,
|
|
612
|
+
_fixed_evidence_cache: dict | None = None,
|
|
613
|
+
_fixed_evidence_context: str | None = None,
|
|
614
|
+
_sensitivity: str = "default",
|
|
615
|
+
) -> AnalysisResult:
|
|
616
|
+
"""Analyze retained features without extracting responses again.
|
|
617
|
+
|
|
618
|
+
Fixed evidence reuse requires immutable, append-only watch prefixes under
|
|
619
|
+
the supplied configuration context. Ordinary analysis does not use it.
|
|
620
|
+
"""
|
|
621
|
+
sequential_parameters, retrospective_parameters = _analysis_options(
|
|
622
|
+
sequential_options=sequential_options,
|
|
623
|
+
retrospective=retrospective,
|
|
624
|
+
retrospective_options=retrospective_options,
|
|
625
|
+
replication_tolerance=replication_tolerance,
|
|
626
|
+
replication_coverage_policy=replication_coverage_policy,
|
|
627
|
+
corroboration_radius=corroboration_radius,
|
|
628
|
+
persistence_effect_fraction=persistence_effect_fraction,
|
|
629
|
+
)
|
|
630
|
+
if _fixed_evidence_cache is not None and (
|
|
631
|
+
type(_fixed_evidence_cache) is not dict
|
|
632
|
+
or not isinstance(_fixed_evidence_context, str)
|
|
633
|
+
or not _fixed_evidence_context
|
|
634
|
+
):
|
|
635
|
+
raise ValueError("fixed evidence reuse requires a watch configuration context")
|
|
636
|
+
rollouts = validate_rollouts(rollouts)
|
|
637
|
+
units = resolve_sampling_units(rollouts, sampling_units)
|
|
638
|
+
aggregation = aggregate(rollouts, features, sampling_units=units)
|
|
639
|
+
sequential = detect_sequential(aggregation.series, **sequential_parameters)
|
|
640
|
+
if retrospective:
|
|
641
|
+
retrospective_result = detect_retrospective(
|
|
642
|
+
aggregation.series, **retrospective_parameters
|
|
643
|
+
)
|
|
644
|
+
settings = _effective_options(detect_retrospective, retrospective_parameters)
|
|
645
|
+
else:
|
|
646
|
+
retrospective_result = disabled_result()
|
|
647
|
+
settings = None
|
|
648
|
+
sensitivity_metadata = None
|
|
649
|
+
if _sensitivity == "lenient":
|
|
650
|
+
sequential, counts = select_lenient(aggregation.series, sequential)
|
|
651
|
+
sensitivity_metadata = profile_metadata(
|
|
652
|
+
counts, explicit_sampling_units=True, retrospective_settings=settings
|
|
653
|
+
)
|
|
654
|
+
|
|
655
|
+
half_features = features
|
|
656
|
+
if _fixed_evidence_cache is not None:
|
|
657
|
+
# Only alerts use sparse half-series. Keep dense validation unchanged.
|
|
658
|
+
needed = (set(features.dense.columns) - {"run", "sample_id"}) | set(
|
|
659
|
+
sequential.alerts.feature
|
|
660
|
+
)
|
|
661
|
+
half_features = FeatureTables(
|
|
662
|
+
dense=features.dense,
|
|
663
|
+
sparse=features.sparse.loc[features.sparse.feature.isin(needed)],
|
|
664
|
+
catalog=features.catalog.loc[features.catalog.feature.isin(needed)],
|
|
665
|
+
)
|
|
666
|
+
assignments, half_series, half_schedules = _aggregate_halves(
|
|
667
|
+
rollouts, half_features, units
|
|
668
|
+
)
|
|
669
|
+
|
|
670
|
+
full_scores = _groups(sequential.scores)
|
|
671
|
+
full_alerts = _groups(sequential.alerts)
|
|
672
|
+
if _fixed_evidence_cache is not None:
|
|
673
|
+
for key in set(_fixed_evidence_cache) - set(full_alerts):
|
|
674
|
+
del _fixed_evidence_cache[key]
|
|
675
|
+
retro_diagnostics = retrospective_result.diagnostics.set_index(["run", "feature"])
|
|
676
|
+
changes = retrospective_result.changepoints.set_index(["run", "feature"])
|
|
677
|
+
assignments_by_run = dict(tuple(assignments.groupby("run", sort=False)))
|
|
678
|
+
units_by_run = dict(tuple(units.groupby("run", sort=False)))
|
|
679
|
+
values = _FeatureValueIndex(rollouts, features)
|
|
680
|
+
rows = []
|
|
681
|
+
detail_parts = {
|
|
682
|
+
"replication_halves": [],
|
|
683
|
+
"corroboration_halves": [],
|
|
684
|
+
"persistence_steps": [],
|
|
685
|
+
"reward_groups": [],
|
|
686
|
+
"prompt_values": [],
|
|
687
|
+
"prompt_shifts": [],
|
|
688
|
+
}
|
|
689
|
+
empty_series = aggregation.series.iloc[:0]
|
|
690
|
+
empty_schedule = aggregation.prompt_schedule.iloc[:0]
|
|
691
|
+
for diagnostic in sequential.diagnostics.itertuples(index=False):
|
|
692
|
+
run, feature = diagnostic.run, diagnostic.feature
|
|
693
|
+
key = (run, feature)
|
|
694
|
+
row = dict.fromkeys(_EVIDENCE_COLUMNS)
|
|
695
|
+
row.update(
|
|
696
|
+
run=run,
|
|
697
|
+
feature=feature,
|
|
698
|
+
tier=diagnostic.tier,
|
|
699
|
+
kind=diagnostic.kind,
|
|
700
|
+
sequential_status=diagnostic.status,
|
|
701
|
+
scale_source=diagnostic.scale_source,
|
|
702
|
+
numerical_floor_dominant=bool(diagnostic.numerical_floor_dominant),
|
|
703
|
+
retrospective_status=retro_diagnostics.loc[key, "status"]
|
|
704
|
+
if retrospective
|
|
705
|
+
else "disabled",
|
|
706
|
+
replication_coverage_policy=replication_coverage_policy,
|
|
707
|
+
)
|
|
708
|
+
if key in changes.index:
|
|
709
|
+
row["changepoint_step"] = int(changes.loc[key, "changepoint_step"])
|
|
710
|
+
if key not in full_alerts:
|
|
711
|
+
if diagnostic.status == "profile_excluded":
|
|
712
|
+
row.update(
|
|
713
|
+
replication_status="not_applicable_profile_excluded",
|
|
714
|
+
corroboration_status="unavailable",
|
|
715
|
+
corroboration_reason="profile_excluded",
|
|
716
|
+
persistence_status="not_applicable_profile_excluded",
|
|
717
|
+
reward_status="not_applicable_profile_excluded",
|
|
718
|
+
promptmix_status="not_applicable_profile_excluded",
|
|
719
|
+
)
|
|
720
|
+
rows.append(row)
|
|
721
|
+
continue
|
|
722
|
+
unscorable = diagnostic.n_scored == 0
|
|
723
|
+
unavailable = (
|
|
724
|
+
"not_applicable_full_unscorable"
|
|
725
|
+
if unscorable
|
|
726
|
+
else "not_applicable_no_sequential_alert"
|
|
727
|
+
)
|
|
728
|
+
row.update(
|
|
729
|
+
replication_status="full_unscorable" if unscorable else unavailable,
|
|
730
|
+
corroboration_status="unavailable",
|
|
731
|
+
corroboration_reason="full_unscorable"
|
|
732
|
+
if unscorable
|
|
733
|
+
else "no_full_alert",
|
|
734
|
+
persistence_status=unavailable,
|
|
735
|
+
reward_status=unavailable,
|
|
736
|
+
promptmix_status=unavailable,
|
|
737
|
+
)
|
|
738
|
+
rows.append(row)
|
|
739
|
+
continue
|
|
740
|
+
|
|
741
|
+
selected_full = DetectionResult(
|
|
742
|
+
full_alerts[key],
|
|
743
|
+
pd.DataFrame([diagnostic._asdict()]),
|
|
744
|
+
full_scores[key],
|
|
745
|
+
sequential.screening.iloc[:0],
|
|
746
|
+
)
|
|
747
|
+
alert_row, details = _alert_evidence(
|
|
748
|
+
diagnostic,
|
|
749
|
+
selected_full,
|
|
750
|
+
{half: half_series[half].get(key, empty_series) for half in (0, 1)},
|
|
751
|
+
{half: half_schedules[half].get(run, empty_schedule) for half in (0, 1)},
|
|
752
|
+
assignments_by_run[run],
|
|
753
|
+
values,
|
|
754
|
+
units_by_run[run],
|
|
755
|
+
replication_tolerance=replication_tolerance,
|
|
756
|
+
replication_coverage_policy=replication_coverage_policy,
|
|
757
|
+
corroboration_radius=corroboration_radius,
|
|
758
|
+
persistence_effect_fraction=persistence_effect_fraction,
|
|
759
|
+
sequential_parameters=LENIENT_SEQUENTIAL
|
|
760
|
+
if _sensitivity == "lenient"
|
|
761
|
+
and diagnostic.tier == "structural"
|
|
762
|
+
and diagnostic.kind == "binary"
|
|
763
|
+
else sequential_parameters,
|
|
764
|
+
fixed_cache=_fixed_evidence_cache,
|
|
765
|
+
cache_context=_fixed_evidence_context,
|
|
766
|
+
)
|
|
767
|
+
row.update(alert_row)
|
|
768
|
+
rows.append(row)
|
|
769
|
+
for name, detail in details.items():
|
|
770
|
+
detail_parts[name].append(detail)
|
|
771
|
+
|
|
772
|
+
evidence = pd.DataFrame(rows, columns=_EVIDENCE_COLUMNS)
|
|
773
|
+
# Keep integer step IDs exact when identities are missing.
|
|
774
|
+
for name in [
|
|
775
|
+
"onset_step",
|
|
776
|
+
"alert_step",
|
|
777
|
+
"changepoint_step",
|
|
778
|
+
"replication_cutoff_step",
|
|
779
|
+
"corroboration_window_start",
|
|
780
|
+
"corroboration_window_end",
|
|
781
|
+
"reward_window_start",
|
|
782
|
+
"reward_window_end",
|
|
783
|
+
"prompt_before_start",
|
|
784
|
+
"prompt_before_end",
|
|
785
|
+
"prompt_after_start",
|
|
786
|
+
"prompt_after_end",
|
|
787
|
+
]:
|
|
788
|
+
evidence[name] = pd.Series([row[name] for row in rows], dtype=object)
|
|
789
|
+
evidence["replicates"] = pd.array(evidence.replicates, dtype="boolean")
|
|
790
|
+
evidence["corroborates"] = pd.array(evidence.corroborates, dtype="boolean")
|
|
791
|
+
evidence["numerical_floor_dominant"] = evidence.numerical_floor_dominant.astype(
|
|
792
|
+
bool
|
|
793
|
+
)
|
|
794
|
+
grouped = _group_events_prepared(values, sequential.alerts)
|
|
795
|
+
return AnalysisResult(
|
|
796
|
+
features=features,
|
|
797
|
+
aggregation=aggregation,
|
|
798
|
+
sequential=sequential,
|
|
799
|
+
retrospective=retrospective_result,
|
|
800
|
+
retrospective_metadata=retrospective_metadata(settings),
|
|
801
|
+
evidence=evidence,
|
|
802
|
+
half_assignments=assignments,
|
|
803
|
+
exemplars=_select_exemplars_prepared(values, sequential.alerts),
|
|
804
|
+
events=grouped.events,
|
|
805
|
+
event_members=grouped.members,
|
|
806
|
+
event_ranking=rank_events(grouped.events, grouped.members, evidence),
|
|
807
|
+
sensitivity_metadata=sensitivity_metadata,
|
|
808
|
+
**{name: _details(parts) for name, parts in detail_parts.items()},
|
|
809
|
+
)
|