rolloutdiff 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. rolloutdiff/__init__.py +130 -0
  2. rolloutdiff/_aria_replay.py +346 -0
  3. rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
  4. rolloutdiff/_dependence.py +195 -0
  5. rolloutdiff/_feature_values.py +79 -0
  6. rolloutdiff/_replay_container.py +568 -0
  7. rolloutdiff/_report_validation.py +758 -0
  8. rolloutdiff/_retrospective_policy.py +33 -0
  9. rolloutdiff/_sampling.py +127 -0
  10. rolloutdiff/_series_validation.py +67 -0
  11. rolloutdiff/_watch_report.py +436 -0
  12. rolloutdiff/_watch_storage.py +800 -0
  13. rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
  14. rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
  15. rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
  16. rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
  17. rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
  18. rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
  19. rolloutdiff/_web/assets/third-party-notices.txt +112 -0
  20. rolloutdiff/_web/index.html +14 -0
  21. rolloutdiff/adapters.py +264 -0
  22. rolloutdiff/app.py +956 -0
  23. rolloutdiff/app_evidence.py +678 -0
  24. rolloutdiff/benchmark_reference.py +458 -0
  25. rolloutdiff/cli.py +1292 -0
  26. rolloutdiff/comparison.py +572 -0
  27. rolloutdiff/comparison_report.py +982 -0
  28. rolloutdiff/corroboration.py +287 -0
  29. rolloutdiff/detect.py +307 -0
  30. rolloutdiff/events.py +173 -0
  31. rolloutdiff/evidence.py +285 -0
  32. rolloutdiff/exemplars.py +240 -0
  33. rolloutdiff/features.py +823 -0
  34. rolloutdiff/html_report.py +726 -0
  35. rolloutdiff/inspect_logs.py +383 -0
  36. rolloutdiff/inspection.py +207 -0
  37. rolloutdiff/integrations.py +552 -0
  38. rolloutdiff/investigation.py +1146 -0
  39. rolloutdiff/notifications.py +406 -0
  40. rolloutdiff/pipeline.py +809 -0
  41. rolloutdiff/producer.py +663 -0
  42. rolloutdiff/prompt_mix.py +258 -0
  43. rolloutdiff/ranking.py +388 -0
  44. rolloutdiff/replay.py +589 -0
  45. rolloutdiff/replication.py +572 -0
  46. rolloutdiff/report.py +838 -0
  47. rolloutdiff/retrospective.py +386 -0
  48. rolloutdiff/schema.py +109 -0
  49. rolloutdiff/sensitivity.py +202 -0
  50. rolloutdiff/series.py +430 -0
  51. rolloutdiff/tool_features.py +422 -0
  52. rolloutdiff/trajectories.py +236 -0
  53. rolloutdiff/watch-compatibility.json +13 -0
  54. rolloutdiff/watch.py +742 -0
  55. rolloutdiff-0.1.0.dist-info/METADATA +158 -0
  56. rolloutdiff-0.1.0.dist-info/RECORD +62 -0
  57. rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
  58. rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
  59. rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
  60. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
  61. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
  62. rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
rolloutdiff/series.py ADDED
@@ -0,0 +1,430 @@
1
+ """Exact-prompt-balanced series with sampling-unit uncertainty."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+
7
+ import numpy as np
8
+ import pandas as pd
9
+ from pandas.api.types import is_complex_dtype, is_numeric_dtype
10
+
11
+ from ._sampling import cluster_standard_error, resolve_sampling_units
12
+ from .features import FeatureTables
13
+ from .schema import validate_rollouts
14
+
15
+ _KEYS = ["run", "sample_id"]
16
+ _GROUPS = ["run", "step", "prompt_id"]
17
+ _STEPS = ["run", "step"]
18
+ _SERIES_COLUMNS = [
19
+ "run",
20
+ "step",
21
+ "feature",
22
+ "tier",
23
+ "kind",
24
+ "mean",
25
+ "se",
26
+ "n_rollouts",
27
+ "n_prompts",
28
+ "n_sampling_units",
29
+ ]
30
+ _SCHEDULE_COLUMNS = [
31
+ "run",
32
+ "step",
33
+ "prompt_id",
34
+ "n_rollouts",
35
+ "prompt_chars",
36
+ "sampling_unit_id",
37
+ ]
38
+
39
+
40
+ @dataclass(frozen=True)
41
+ class Aggregation:
42
+ series: pd.DataFrame
43
+ prompt_schedule: pd.DataFrame
44
+
45
+
46
+ def _require_columns(table: pd.DataFrame, columns: list[str], name: str) -> None:
47
+ if not isinstance(table, pd.DataFrame):
48
+ raise ValueError(f"{name} must be a DataFrame")
49
+ if not table.columns.is_unique:
50
+ raise ValueError(f"{name} contains duplicate columns")
51
+ missing = set(columns) - set(table.columns)
52
+ if missing:
53
+ raise ValueError(f"{name} is missing columns: {sorted(missing)}")
54
+
55
+
56
+ def _validate_features(rollouts: pd.DataFrame, features: FeatureTables) -> list[str]:
57
+ dense, sparse, catalog = features.dense, features.sparse, features.catalog
58
+ _require_columns(dense, _KEYS, "dense features")
59
+ _require_columns(sparse, [*_KEYS, "feature"], "sparse features")
60
+ _require_columns(catalog, ["feature", "tier", "kind"], "feature catalog")
61
+ if (
62
+ not catalog["feature"]
63
+ .map(lambda name: isinstance(name, str) and bool(name))
64
+ .all()
65
+ ):
66
+ raise ValueError("catalog feature names must be nonempty strings")
67
+ if catalog["feature"].duplicated().any():
68
+ raise ValueError("catalog contains duplicate feature names")
69
+ if not catalog["tier"].isin(["structural", "prose"]).all():
70
+ raise ValueError("catalog tier must be structural or prose")
71
+ if not catalog["kind"].isin(["binary", "continuous"]).all():
72
+ raise ValueError("catalog kind must be binary or continuous")
73
+ if catalog["feature"].eq("scalar:reward").any():
74
+ raise ValueError("scalar:reward is reserved for the rollout reward")
75
+
76
+ dense_names = [name for name in dense.columns if name not in _KEYS]
77
+ metadata = catalog.set_index("feature")
78
+ if not set(dense_names).issubset(metadata.index):
79
+ raise ValueError("dense feature columns are missing from the catalog")
80
+ sparse_names = catalog.loc[
81
+ ~catalog["feature"].isin(dense_names), "feature"
82
+ ].tolist()
83
+ if not metadata.loc[sparse_names, "kind"].eq("binary").all():
84
+ raise ValueError("sparse features must be binary")
85
+ if not sparse["feature"].isin(sparse_names).all():
86
+ raise ValueError("sparse feature names must belong to the sparse catalog")
87
+
88
+ expected_keys = pd.MultiIndex.from_frame(rollouts[_KEYS])
89
+ dense_keys = pd.MultiIndex.from_frame(dense[_KEYS])
90
+ if not dense_keys.is_unique:
91
+ raise ValueError("dense features contain duplicate rollout keys")
92
+ if (
93
+ len(dense_keys) != len(expected_keys)
94
+ or not dense_keys.isin(expected_keys).all()
95
+ ):
96
+ raise ValueError("dense feature keys must exactly match rollout keys")
97
+ if sparse.duplicated([*_KEYS, "feature"]).any():
98
+ raise ValueError("sparse features contain duplicate rollout/feature rows")
99
+ if not pd.MultiIndex.from_frame(sparse[_KEYS]).isin(expected_keys).all():
100
+ raise ValueError("sparse features contain unknown rollout keys")
101
+ for name in dense_names:
102
+ values = dense[name]
103
+ if (
104
+ not is_numeric_dtype(values.dtype)
105
+ or is_complex_dtype(values.dtype)
106
+ or not np.isfinite(values.to_numpy(dtype=float)).all()
107
+ ):
108
+ raise ValueError(
109
+ f"dense feature {name!r} must contain finite real numeric values"
110
+ )
111
+ if metadata.loc[name, "kind"] == "binary" and not values.isin([0, 1]).all():
112
+ raise ValueError(
113
+ f"binary dense feature {name!r} must contain only zero or one"
114
+ )
115
+ return dense_names
116
+
117
+
118
+ def _aggregate_dense(
119
+ rollouts: pd.DataFrame,
120
+ features: FeatureTables,
121
+ dense_names: list[str],
122
+ units: pd.DataFrame,
123
+ counts: pd.DataFrame,
124
+ singleton_units: bool,
125
+ ) -> list[pd.DataFrame]:
126
+ # Internal names keep caller feature names from colliding with group keys.
127
+ internal_names = {name: f"__value_{i}" for i, name in enumerate(dense_names)}
128
+ dense = (
129
+ rollouts[[*_KEYS, "step", "prompt_id", "reward"]]
130
+ .rename(columns={"reward": "__reward"})
131
+ .merge(
132
+ features.dense.rename(columns=internal_names),
133
+ on=_KEYS,
134
+ validate="one_to_one",
135
+ )
136
+ )
137
+ value_names = {**internal_names, "scalar:reward": "__reward"}
138
+ prompt_means = dense.groupby(_GROUPS, sort=True, observed=True)[
139
+ list(value_names.values())
140
+ ].mean()
141
+ step_means = prompt_means.groupby(_STEPS, sort=True, observed=True)
142
+ means, deviations = step_means.mean(), step_means.std(ddof=1)
143
+ clustered_errors = None
144
+ if not singleton_units:
145
+ grouped = prompt_means.reset_index().merge(
146
+ units, on=["run", "prompt_id"], validate="many_to_one"
147
+ )
148
+ errors = []
149
+ for (run, step), group in grouped.groupby(_STEPS, sort=True, observed=True):
150
+ errors.append(
151
+ {
152
+ "run": run,
153
+ "step": step,
154
+ **{
155
+ internal: cluster_standard_error(
156
+ group[internal].to_numpy(),
157
+ group.sampling_unit_id.to_numpy(),
158
+ )
159
+ for internal in value_names.values()
160
+ },
161
+ }
162
+ )
163
+ clustered_errors = pd.DataFrame(errors).set_index(_STEPS)
164
+ parts = []
165
+ for name, internal in value_names.items():
166
+ part = pd.DataFrame(
167
+ {
168
+ "mean": means[internal],
169
+ "se": deviations[internal],
170
+ }
171
+ ).reset_index()
172
+ part = part.merge(counts, on=_STEPS, validate="one_to_one")
173
+ if singleton_units:
174
+ part["se"] /= np.sqrt(part["n_prompts"])
175
+ else:
176
+ part["se"] = clustered_errors[internal].to_numpy(dtype=float)
177
+ part["feature"] = name
178
+ parts.append(part)
179
+
180
+ return parts
181
+
182
+
183
+ def _aggregate_sparse(
184
+ rollouts: pd.DataFrame,
185
+ sparse: pd.DataFrame,
186
+ sparse_catalog: pd.DataFrame,
187
+ schedule: pd.DataFrame,
188
+ counts: pd.DataFrame,
189
+ singleton_units: bool,
190
+ ) -> pd.DataFrame:
191
+ sparse_result = counts.merge(sparse_catalog[["feature"]], how="cross")
192
+ prompt_counts = pd.DataFrame()
193
+ if not sparse.empty:
194
+ present = sparse[[*_KEYS, "feature"]].merge(
195
+ rollouts[[*_KEYS, "step", "prompt_id"]], on=_KEYS, validate="many_to_one"
196
+ )
197
+ prompt_counts = (
198
+ present.groupby([*_GROUPS, "feature"], observed=True, sort=False)
199
+ .size()
200
+ .rename("present")
201
+ .reset_index()
202
+ ).merge(
203
+ schedule[[*_GROUPS, "n_rollouts", "sampling_unit_id"]],
204
+ on=_GROUPS,
205
+ validate="many_to_one",
206
+ )
207
+ prompt_counts["p"] = prompt_counts["present"] / prompt_counts["n_rollouts"]
208
+ if singleton_units:
209
+ return _sparse_prompt_errors(sparse_result, prompt_counts)
210
+ return _sparse_cluster_errors(sparse_result, prompt_counts, schedule)
211
+
212
+
213
+ def _sparse_prompt_errors(
214
+ sparse_result: pd.DataFrame, prompt_counts: pd.DataFrame
215
+ ) -> pd.DataFrame:
216
+ # Two passes: deviations of present prompts from the step mean, plus the
217
+ # absent prompts at zero. A one-pass sum of squares leaves roundoff where
218
+ # every prompt shares the same proportion.
219
+ keys = [*_STEPS, "feature"]
220
+ groups = sparse_result["n_prompts"].to_numpy(dtype=float)
221
+ if prompt_counts.empty:
222
+ sparse_result["mean"] = 0.0
223
+ variance = np.zeros(len(sparse_result))
224
+ else:
225
+ moments = (
226
+ prompt_counts.groupby(keys, observed=True, sort=False)
227
+ .agg(sum_p=("p", "sum"), n_present=("p", "size"))
228
+ .reset_index()
229
+ )
230
+ sparse_result = sparse_result.merge(
231
+ moments, on=keys, how="left", validate="one_to_one"
232
+ )
233
+ sparse_result[["sum_p", "n_present"]] = sparse_result[
234
+ ["sum_p", "n_present"]
235
+ ].fillna(0.0)
236
+ sparse_result["mean"] = sparse_result["sum_p"].to_numpy() / groups
237
+ deviations = prompt_counts.merge(
238
+ sparse_result[[*keys, "mean"]], on=keys, validate="many_to_one"
239
+ )
240
+ deviations["d2"] = (deviations["p"] - deviations["mean"]) ** 2
241
+ residuals = (
242
+ deviations.groupby(keys, observed=True, sort=False)
243
+ .d2.sum()
244
+ .rename("present_d2")
245
+ .reset_index()
246
+ )
247
+ sparse_result = sparse_result.merge(
248
+ residuals, on=keys, how="left", validate="one_to_one"
249
+ )
250
+ absent = groups - sparse_result["n_present"].to_numpy()
251
+ variance = (
252
+ sparse_result["present_d2"].fillna(0.0).to_numpy()
253
+ + absent * sparse_result["mean"].to_numpy() ** 2
254
+ )
255
+ sparse_result["se"] = np.sqrt(
256
+ np.divide(
257
+ variance,
258
+ groups * (groups - 1),
259
+ out=np.full(len(groups), np.nan),
260
+ where=groups > 1,
261
+ )
262
+ )
263
+ return sparse_result
264
+
265
+
266
+ def _sparse_cluster_errors(
267
+ sparse_result: pd.DataFrame, prompt_counts: pd.DataFrame, schedule: pd.DataFrame
268
+ ) -> pd.DataFrame:
269
+ # Sum exact-prompt proportions within units. Only units where a
270
+ # feature occurs are materialized; absent-unit residuals use their
271
+ # squared prompt counts, including entirely absent features.
272
+ unit_keys = [*_STEPS, "sampling_unit_id"]
273
+ sizes = (
274
+ schedule.groupby(unit_keys, sort=False, observed=True)
275
+ .size()
276
+ .rename("unit_prompts")
277
+ .reset_index()
278
+ )
279
+ sizes["unit_prompts2"] = sizes.unit_prompts.astype(float) ** 2
280
+ totals = (
281
+ sizes.groupby(_STEPS, sort=False)
282
+ .unit_prompts2.sum()
283
+ .rename("total_prompts2")
284
+ .reset_index()
285
+ )
286
+ sparse_result = sparse_result.merge(totals, on=_STEPS, validate="many_to_one")
287
+ if prompt_counts.empty:
288
+ sparse_result["mean"] = 0.0
289
+ sparse_result["residual2"] = 0.0
290
+ else:
291
+ sums = (
292
+ prompt_counts.groupby([*unit_keys, "feature"], sort=False, observed=True)
293
+ .p.sum()
294
+ .rename("sum_p")
295
+ .reset_index()
296
+ )
297
+ # Keep exact-prompt summation order for stable point estimates.
298
+ first = (
299
+ prompt_counts.groupby([*_STEPS, "feature"], sort=False, observed=True)
300
+ .p.sum()
301
+ .rename("sum_p")
302
+ .reset_index()
303
+ )
304
+ sparse_result = sparse_result.merge(
305
+ first, on=[*_STEPS, "feature"], how="left", validate="one_to_one"
306
+ )
307
+ sparse_result["mean"] = (
308
+ sparse_result.sum_p.fillna(0.0) / sparse_result.n_prompts
309
+ )
310
+ sums = sums.merge(sizes, on=unit_keys, validate="many_to_one").merge(
311
+ sparse_result[[*_STEPS, "feature", "mean"]],
312
+ on=[*_STEPS, "feature"],
313
+ validate="many_to_one",
314
+ )
315
+ sums["active_residual2"] = (sums.sum_p - sums.unit_prompts * sums["mean"]) ** 2
316
+ residuals = (
317
+ sums.groupby([*_STEPS, "feature"], sort=False)
318
+ .agg(
319
+ active_residual2=("active_residual2", "sum"),
320
+ active_prompts2=("unit_prompts2", "sum"),
321
+ )
322
+ .reset_index()
323
+ )
324
+ sparse_result = sparse_result.merge(
325
+ residuals, on=[*_STEPS, "feature"], how="left", validate="one_to_one"
326
+ )
327
+ sparse_result[["active_residual2", "active_prompts2"]] = sparse_result[
328
+ ["active_residual2", "active_prompts2"]
329
+ ].fillna(0.0)
330
+ sparse_result["residual2"] = (
331
+ sparse_result.active_residual2
332
+ + (sparse_result.total_prompts2 - sparse_result.active_prompts2)
333
+ * sparse_result["mean"] ** 2
334
+ )
335
+ groups = sparse_result.n_sampling_units.to_numpy(dtype=float)
336
+ variance = (
337
+ sparse_result.residual2.to_numpy()
338
+ / sparse_result.n_prompts.to_numpy(dtype=float) ** 2
339
+ )
340
+ sparse_result["se"] = np.sqrt(
341
+ np.divide(
342
+ groups * variance,
343
+ groups - 1,
344
+ out=np.full(len(groups), np.nan),
345
+ where=groups > 1,
346
+ )
347
+ )
348
+ return sparse_result
349
+
350
+
351
+ def aggregate(
352
+ rollouts: pd.DataFrame,
353
+ features: FeatureTables,
354
+ *,
355
+ sampling_units: pd.DataFrame | None = None,
356
+ ) -> Aggregation:
357
+ """Aggregate dense, sparse, and reward values by run and step.
358
+
359
+ Weight exact-prompt means equally. Default SE is SD / sqrt(prompt count);
360
+ explicit units use CR1 SE of the same mean. Fewer than two units yields NaN;
361
+ measured zero variance stays zero. SE excludes detector floors.
362
+ Align features by (run, sample_id).
363
+ """
364
+ rollouts = validate_rollouts(rollouts)
365
+ dense_names = _validate_features(rollouts, features)
366
+ units = resolve_sampling_units(rollouts, sampling_units)
367
+ singleton_units = not units.duplicated(["run", "sampling_unit_id"]).any()
368
+ if rollouts.empty:
369
+ return Aggregation(
370
+ pd.DataFrame(columns=_SERIES_COLUMNS),
371
+ pd.DataFrame(columns=_SCHEDULE_COLUMNS),
372
+ )
373
+
374
+ schedule = (
375
+ rollouts.assign(prompt_chars=rollouts["prompt"].str.len())
376
+ .groupby(_GROUPS, sort=True, observed=True)
377
+ .agg(n_rollouts=("sample_id", "size"), prompt_chars=("prompt_chars", "first"))
378
+ .reset_index()
379
+ .merge(units, on=["run", "prompt_id"], validate="many_to_one")
380
+ )
381
+ counts = (
382
+ schedule.groupby(_STEPS, sort=True, observed=True)
383
+ .agg(
384
+ n_rollouts=("n_rollouts", "sum"),
385
+ n_prompts=("prompt_id", "size"),
386
+ n_sampling_units=("sampling_unit_id", "nunique"),
387
+ )
388
+ .reset_index()
389
+ )
390
+ parts = _aggregate_dense(
391
+ rollouts, features, dense_names, units, counts, singleton_units
392
+ )
393
+
394
+ sparse_catalog = features.catalog.loc[
395
+ ~features.catalog["feature"].isin(dense_names)
396
+ ]
397
+ if not sparse_catalog.empty:
398
+ parts.append(
399
+ _aggregate_sparse(
400
+ rollouts,
401
+ features.sparse,
402
+ sparse_catalog,
403
+ schedule,
404
+ counts,
405
+ singleton_units,
406
+ )
407
+ )
408
+
409
+ catalog = pd.concat(
410
+ [
411
+ features.catalog[["feature", "tier", "kind"]],
412
+ pd.DataFrame(
413
+ [
414
+ {
415
+ "feature": "scalar:reward",
416
+ "tier": "structural",
417
+ "kind": "continuous",
418
+ }
419
+ ]
420
+ ),
421
+ ],
422
+ ignore_index=True,
423
+ )
424
+ series = pd.concat(parts, ignore_index=True).merge(
425
+ catalog, on="feature", validate="many_to_one"
426
+ )
427
+ series = (
428
+ series[_SERIES_COLUMNS].sort_values([*_STEPS, "feature"]).reset_index(drop=True)
429
+ )
430
+ return Aggregation(series, schedule[_SCHEDULE_COLUMNS])