rolloutdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rolloutdiff/__init__.py +130 -0
- rolloutdiff/_aria_replay.py +346 -0
- rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
- rolloutdiff/_dependence.py +195 -0
- rolloutdiff/_feature_values.py +79 -0
- rolloutdiff/_replay_container.py +568 -0
- rolloutdiff/_report_validation.py +758 -0
- rolloutdiff/_retrospective_policy.py +33 -0
- rolloutdiff/_sampling.py +127 -0
- rolloutdiff/_series_validation.py +67 -0
- rolloutdiff/_watch_report.py +436 -0
- rolloutdiff/_watch_storage.py +800 -0
- rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
- rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
- rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
- rolloutdiff/_web/assets/third-party-notices.txt +112 -0
- rolloutdiff/_web/index.html +14 -0
- rolloutdiff/adapters.py +264 -0
- rolloutdiff/app.py +956 -0
- rolloutdiff/app_evidence.py +678 -0
- rolloutdiff/benchmark_reference.py +458 -0
- rolloutdiff/cli.py +1292 -0
- rolloutdiff/comparison.py +572 -0
- rolloutdiff/comparison_report.py +982 -0
- rolloutdiff/corroboration.py +287 -0
- rolloutdiff/detect.py +307 -0
- rolloutdiff/events.py +173 -0
- rolloutdiff/evidence.py +285 -0
- rolloutdiff/exemplars.py +240 -0
- rolloutdiff/features.py +823 -0
- rolloutdiff/html_report.py +726 -0
- rolloutdiff/inspect_logs.py +383 -0
- rolloutdiff/inspection.py +207 -0
- rolloutdiff/integrations.py +552 -0
- rolloutdiff/investigation.py +1146 -0
- rolloutdiff/notifications.py +406 -0
- rolloutdiff/pipeline.py +809 -0
- rolloutdiff/producer.py +663 -0
- rolloutdiff/prompt_mix.py +258 -0
- rolloutdiff/ranking.py +388 -0
- rolloutdiff/replay.py +589 -0
- rolloutdiff/replication.py +572 -0
- rolloutdiff/report.py +838 -0
- rolloutdiff/retrospective.py +386 -0
- rolloutdiff/schema.py +109 -0
- rolloutdiff/sensitivity.py +202 -0
- rolloutdiff/series.py +430 -0
- rolloutdiff/tool_features.py +422 -0
- rolloutdiff/trajectories.py +236 -0
- rolloutdiff/watch-compatibility.json +13 -0
- rolloutdiff/watch.py +742 -0
- rolloutdiff-0.1.0.dist-info/METADATA +158 -0
- rolloutdiff-0.1.0.dist-info/RECORD +62 -0
- rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
- rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
- rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
- rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
rolloutdiff/series.py
ADDED
|
@@ -0,0 +1,430 @@
|
|
|
1
|
+
"""Exact-prompt-balanced series with sampling-unit uncertainty."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
import pandas as pd
|
|
9
|
+
from pandas.api.types import is_complex_dtype, is_numeric_dtype
|
|
10
|
+
|
|
11
|
+
from ._sampling import cluster_standard_error, resolve_sampling_units
|
|
12
|
+
from .features import FeatureTables
|
|
13
|
+
from .schema import validate_rollouts
|
|
14
|
+
|
|
15
|
+
_KEYS = ["run", "sample_id"]
|
|
16
|
+
_GROUPS = ["run", "step", "prompt_id"]
|
|
17
|
+
_STEPS = ["run", "step"]
|
|
18
|
+
_SERIES_COLUMNS = [
|
|
19
|
+
"run",
|
|
20
|
+
"step",
|
|
21
|
+
"feature",
|
|
22
|
+
"tier",
|
|
23
|
+
"kind",
|
|
24
|
+
"mean",
|
|
25
|
+
"se",
|
|
26
|
+
"n_rollouts",
|
|
27
|
+
"n_prompts",
|
|
28
|
+
"n_sampling_units",
|
|
29
|
+
]
|
|
30
|
+
_SCHEDULE_COLUMNS = [
|
|
31
|
+
"run",
|
|
32
|
+
"step",
|
|
33
|
+
"prompt_id",
|
|
34
|
+
"n_rollouts",
|
|
35
|
+
"prompt_chars",
|
|
36
|
+
"sampling_unit_id",
|
|
37
|
+
]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass(frozen=True)
|
|
41
|
+
class Aggregation:
|
|
42
|
+
series: pd.DataFrame
|
|
43
|
+
prompt_schedule: pd.DataFrame
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _require_columns(table: pd.DataFrame, columns: list[str], name: str) -> None:
|
|
47
|
+
if not isinstance(table, pd.DataFrame):
|
|
48
|
+
raise ValueError(f"{name} must be a DataFrame")
|
|
49
|
+
if not table.columns.is_unique:
|
|
50
|
+
raise ValueError(f"{name} contains duplicate columns")
|
|
51
|
+
missing = set(columns) - set(table.columns)
|
|
52
|
+
if missing:
|
|
53
|
+
raise ValueError(f"{name} is missing columns: {sorted(missing)}")
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _validate_features(rollouts: pd.DataFrame, features: FeatureTables) -> list[str]:
|
|
57
|
+
dense, sparse, catalog = features.dense, features.sparse, features.catalog
|
|
58
|
+
_require_columns(dense, _KEYS, "dense features")
|
|
59
|
+
_require_columns(sparse, [*_KEYS, "feature"], "sparse features")
|
|
60
|
+
_require_columns(catalog, ["feature", "tier", "kind"], "feature catalog")
|
|
61
|
+
if (
|
|
62
|
+
not catalog["feature"]
|
|
63
|
+
.map(lambda name: isinstance(name, str) and bool(name))
|
|
64
|
+
.all()
|
|
65
|
+
):
|
|
66
|
+
raise ValueError("catalog feature names must be nonempty strings")
|
|
67
|
+
if catalog["feature"].duplicated().any():
|
|
68
|
+
raise ValueError("catalog contains duplicate feature names")
|
|
69
|
+
if not catalog["tier"].isin(["structural", "prose"]).all():
|
|
70
|
+
raise ValueError("catalog tier must be structural or prose")
|
|
71
|
+
if not catalog["kind"].isin(["binary", "continuous"]).all():
|
|
72
|
+
raise ValueError("catalog kind must be binary or continuous")
|
|
73
|
+
if catalog["feature"].eq("scalar:reward").any():
|
|
74
|
+
raise ValueError("scalar:reward is reserved for the rollout reward")
|
|
75
|
+
|
|
76
|
+
dense_names = [name for name in dense.columns if name not in _KEYS]
|
|
77
|
+
metadata = catalog.set_index("feature")
|
|
78
|
+
if not set(dense_names).issubset(metadata.index):
|
|
79
|
+
raise ValueError("dense feature columns are missing from the catalog")
|
|
80
|
+
sparse_names = catalog.loc[
|
|
81
|
+
~catalog["feature"].isin(dense_names), "feature"
|
|
82
|
+
].tolist()
|
|
83
|
+
if not metadata.loc[sparse_names, "kind"].eq("binary").all():
|
|
84
|
+
raise ValueError("sparse features must be binary")
|
|
85
|
+
if not sparse["feature"].isin(sparse_names).all():
|
|
86
|
+
raise ValueError("sparse feature names must belong to the sparse catalog")
|
|
87
|
+
|
|
88
|
+
expected_keys = pd.MultiIndex.from_frame(rollouts[_KEYS])
|
|
89
|
+
dense_keys = pd.MultiIndex.from_frame(dense[_KEYS])
|
|
90
|
+
if not dense_keys.is_unique:
|
|
91
|
+
raise ValueError("dense features contain duplicate rollout keys")
|
|
92
|
+
if (
|
|
93
|
+
len(dense_keys) != len(expected_keys)
|
|
94
|
+
or not dense_keys.isin(expected_keys).all()
|
|
95
|
+
):
|
|
96
|
+
raise ValueError("dense feature keys must exactly match rollout keys")
|
|
97
|
+
if sparse.duplicated([*_KEYS, "feature"]).any():
|
|
98
|
+
raise ValueError("sparse features contain duplicate rollout/feature rows")
|
|
99
|
+
if not pd.MultiIndex.from_frame(sparse[_KEYS]).isin(expected_keys).all():
|
|
100
|
+
raise ValueError("sparse features contain unknown rollout keys")
|
|
101
|
+
for name in dense_names:
|
|
102
|
+
values = dense[name]
|
|
103
|
+
if (
|
|
104
|
+
not is_numeric_dtype(values.dtype)
|
|
105
|
+
or is_complex_dtype(values.dtype)
|
|
106
|
+
or not np.isfinite(values.to_numpy(dtype=float)).all()
|
|
107
|
+
):
|
|
108
|
+
raise ValueError(
|
|
109
|
+
f"dense feature {name!r} must contain finite real numeric values"
|
|
110
|
+
)
|
|
111
|
+
if metadata.loc[name, "kind"] == "binary" and not values.isin([0, 1]).all():
|
|
112
|
+
raise ValueError(
|
|
113
|
+
f"binary dense feature {name!r} must contain only zero or one"
|
|
114
|
+
)
|
|
115
|
+
return dense_names
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _aggregate_dense(
|
|
119
|
+
rollouts: pd.DataFrame,
|
|
120
|
+
features: FeatureTables,
|
|
121
|
+
dense_names: list[str],
|
|
122
|
+
units: pd.DataFrame,
|
|
123
|
+
counts: pd.DataFrame,
|
|
124
|
+
singleton_units: bool,
|
|
125
|
+
) -> list[pd.DataFrame]:
|
|
126
|
+
# Internal names keep caller feature names from colliding with group keys.
|
|
127
|
+
internal_names = {name: f"__value_{i}" for i, name in enumerate(dense_names)}
|
|
128
|
+
dense = (
|
|
129
|
+
rollouts[[*_KEYS, "step", "prompt_id", "reward"]]
|
|
130
|
+
.rename(columns={"reward": "__reward"})
|
|
131
|
+
.merge(
|
|
132
|
+
features.dense.rename(columns=internal_names),
|
|
133
|
+
on=_KEYS,
|
|
134
|
+
validate="one_to_one",
|
|
135
|
+
)
|
|
136
|
+
)
|
|
137
|
+
value_names = {**internal_names, "scalar:reward": "__reward"}
|
|
138
|
+
prompt_means = dense.groupby(_GROUPS, sort=True, observed=True)[
|
|
139
|
+
list(value_names.values())
|
|
140
|
+
].mean()
|
|
141
|
+
step_means = prompt_means.groupby(_STEPS, sort=True, observed=True)
|
|
142
|
+
means, deviations = step_means.mean(), step_means.std(ddof=1)
|
|
143
|
+
clustered_errors = None
|
|
144
|
+
if not singleton_units:
|
|
145
|
+
grouped = prompt_means.reset_index().merge(
|
|
146
|
+
units, on=["run", "prompt_id"], validate="many_to_one"
|
|
147
|
+
)
|
|
148
|
+
errors = []
|
|
149
|
+
for (run, step), group in grouped.groupby(_STEPS, sort=True, observed=True):
|
|
150
|
+
errors.append(
|
|
151
|
+
{
|
|
152
|
+
"run": run,
|
|
153
|
+
"step": step,
|
|
154
|
+
**{
|
|
155
|
+
internal: cluster_standard_error(
|
|
156
|
+
group[internal].to_numpy(),
|
|
157
|
+
group.sampling_unit_id.to_numpy(),
|
|
158
|
+
)
|
|
159
|
+
for internal in value_names.values()
|
|
160
|
+
},
|
|
161
|
+
}
|
|
162
|
+
)
|
|
163
|
+
clustered_errors = pd.DataFrame(errors).set_index(_STEPS)
|
|
164
|
+
parts = []
|
|
165
|
+
for name, internal in value_names.items():
|
|
166
|
+
part = pd.DataFrame(
|
|
167
|
+
{
|
|
168
|
+
"mean": means[internal],
|
|
169
|
+
"se": deviations[internal],
|
|
170
|
+
}
|
|
171
|
+
).reset_index()
|
|
172
|
+
part = part.merge(counts, on=_STEPS, validate="one_to_one")
|
|
173
|
+
if singleton_units:
|
|
174
|
+
part["se"] /= np.sqrt(part["n_prompts"])
|
|
175
|
+
else:
|
|
176
|
+
part["se"] = clustered_errors[internal].to_numpy(dtype=float)
|
|
177
|
+
part["feature"] = name
|
|
178
|
+
parts.append(part)
|
|
179
|
+
|
|
180
|
+
return parts
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _aggregate_sparse(
|
|
184
|
+
rollouts: pd.DataFrame,
|
|
185
|
+
sparse: pd.DataFrame,
|
|
186
|
+
sparse_catalog: pd.DataFrame,
|
|
187
|
+
schedule: pd.DataFrame,
|
|
188
|
+
counts: pd.DataFrame,
|
|
189
|
+
singleton_units: bool,
|
|
190
|
+
) -> pd.DataFrame:
|
|
191
|
+
sparse_result = counts.merge(sparse_catalog[["feature"]], how="cross")
|
|
192
|
+
prompt_counts = pd.DataFrame()
|
|
193
|
+
if not sparse.empty:
|
|
194
|
+
present = sparse[[*_KEYS, "feature"]].merge(
|
|
195
|
+
rollouts[[*_KEYS, "step", "prompt_id"]], on=_KEYS, validate="many_to_one"
|
|
196
|
+
)
|
|
197
|
+
prompt_counts = (
|
|
198
|
+
present.groupby([*_GROUPS, "feature"], observed=True, sort=False)
|
|
199
|
+
.size()
|
|
200
|
+
.rename("present")
|
|
201
|
+
.reset_index()
|
|
202
|
+
).merge(
|
|
203
|
+
schedule[[*_GROUPS, "n_rollouts", "sampling_unit_id"]],
|
|
204
|
+
on=_GROUPS,
|
|
205
|
+
validate="many_to_one",
|
|
206
|
+
)
|
|
207
|
+
prompt_counts["p"] = prompt_counts["present"] / prompt_counts["n_rollouts"]
|
|
208
|
+
if singleton_units:
|
|
209
|
+
return _sparse_prompt_errors(sparse_result, prompt_counts)
|
|
210
|
+
return _sparse_cluster_errors(sparse_result, prompt_counts, schedule)
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _sparse_prompt_errors(
|
|
214
|
+
sparse_result: pd.DataFrame, prompt_counts: pd.DataFrame
|
|
215
|
+
) -> pd.DataFrame:
|
|
216
|
+
# Two passes: deviations of present prompts from the step mean, plus the
|
|
217
|
+
# absent prompts at zero. A one-pass sum of squares leaves roundoff where
|
|
218
|
+
# every prompt shares the same proportion.
|
|
219
|
+
keys = [*_STEPS, "feature"]
|
|
220
|
+
groups = sparse_result["n_prompts"].to_numpy(dtype=float)
|
|
221
|
+
if prompt_counts.empty:
|
|
222
|
+
sparse_result["mean"] = 0.0
|
|
223
|
+
variance = np.zeros(len(sparse_result))
|
|
224
|
+
else:
|
|
225
|
+
moments = (
|
|
226
|
+
prompt_counts.groupby(keys, observed=True, sort=False)
|
|
227
|
+
.agg(sum_p=("p", "sum"), n_present=("p", "size"))
|
|
228
|
+
.reset_index()
|
|
229
|
+
)
|
|
230
|
+
sparse_result = sparse_result.merge(
|
|
231
|
+
moments, on=keys, how="left", validate="one_to_one"
|
|
232
|
+
)
|
|
233
|
+
sparse_result[["sum_p", "n_present"]] = sparse_result[
|
|
234
|
+
["sum_p", "n_present"]
|
|
235
|
+
].fillna(0.0)
|
|
236
|
+
sparse_result["mean"] = sparse_result["sum_p"].to_numpy() / groups
|
|
237
|
+
deviations = prompt_counts.merge(
|
|
238
|
+
sparse_result[[*keys, "mean"]], on=keys, validate="many_to_one"
|
|
239
|
+
)
|
|
240
|
+
deviations["d2"] = (deviations["p"] - deviations["mean"]) ** 2
|
|
241
|
+
residuals = (
|
|
242
|
+
deviations.groupby(keys, observed=True, sort=False)
|
|
243
|
+
.d2.sum()
|
|
244
|
+
.rename("present_d2")
|
|
245
|
+
.reset_index()
|
|
246
|
+
)
|
|
247
|
+
sparse_result = sparse_result.merge(
|
|
248
|
+
residuals, on=keys, how="left", validate="one_to_one"
|
|
249
|
+
)
|
|
250
|
+
absent = groups - sparse_result["n_present"].to_numpy()
|
|
251
|
+
variance = (
|
|
252
|
+
sparse_result["present_d2"].fillna(0.0).to_numpy()
|
|
253
|
+
+ absent * sparse_result["mean"].to_numpy() ** 2
|
|
254
|
+
)
|
|
255
|
+
sparse_result["se"] = np.sqrt(
|
|
256
|
+
np.divide(
|
|
257
|
+
variance,
|
|
258
|
+
groups * (groups - 1),
|
|
259
|
+
out=np.full(len(groups), np.nan),
|
|
260
|
+
where=groups > 1,
|
|
261
|
+
)
|
|
262
|
+
)
|
|
263
|
+
return sparse_result
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _sparse_cluster_errors(
|
|
267
|
+
sparse_result: pd.DataFrame, prompt_counts: pd.DataFrame, schedule: pd.DataFrame
|
|
268
|
+
) -> pd.DataFrame:
|
|
269
|
+
# Sum exact-prompt proportions within units. Only units where a
|
|
270
|
+
# feature occurs are materialized; absent-unit residuals use their
|
|
271
|
+
# squared prompt counts, including entirely absent features.
|
|
272
|
+
unit_keys = [*_STEPS, "sampling_unit_id"]
|
|
273
|
+
sizes = (
|
|
274
|
+
schedule.groupby(unit_keys, sort=False, observed=True)
|
|
275
|
+
.size()
|
|
276
|
+
.rename("unit_prompts")
|
|
277
|
+
.reset_index()
|
|
278
|
+
)
|
|
279
|
+
sizes["unit_prompts2"] = sizes.unit_prompts.astype(float) ** 2
|
|
280
|
+
totals = (
|
|
281
|
+
sizes.groupby(_STEPS, sort=False)
|
|
282
|
+
.unit_prompts2.sum()
|
|
283
|
+
.rename("total_prompts2")
|
|
284
|
+
.reset_index()
|
|
285
|
+
)
|
|
286
|
+
sparse_result = sparse_result.merge(totals, on=_STEPS, validate="many_to_one")
|
|
287
|
+
if prompt_counts.empty:
|
|
288
|
+
sparse_result["mean"] = 0.0
|
|
289
|
+
sparse_result["residual2"] = 0.0
|
|
290
|
+
else:
|
|
291
|
+
sums = (
|
|
292
|
+
prompt_counts.groupby([*unit_keys, "feature"], sort=False, observed=True)
|
|
293
|
+
.p.sum()
|
|
294
|
+
.rename("sum_p")
|
|
295
|
+
.reset_index()
|
|
296
|
+
)
|
|
297
|
+
# Keep exact-prompt summation order for stable point estimates.
|
|
298
|
+
first = (
|
|
299
|
+
prompt_counts.groupby([*_STEPS, "feature"], sort=False, observed=True)
|
|
300
|
+
.p.sum()
|
|
301
|
+
.rename("sum_p")
|
|
302
|
+
.reset_index()
|
|
303
|
+
)
|
|
304
|
+
sparse_result = sparse_result.merge(
|
|
305
|
+
first, on=[*_STEPS, "feature"], how="left", validate="one_to_one"
|
|
306
|
+
)
|
|
307
|
+
sparse_result["mean"] = (
|
|
308
|
+
sparse_result.sum_p.fillna(0.0) / sparse_result.n_prompts
|
|
309
|
+
)
|
|
310
|
+
sums = sums.merge(sizes, on=unit_keys, validate="many_to_one").merge(
|
|
311
|
+
sparse_result[[*_STEPS, "feature", "mean"]],
|
|
312
|
+
on=[*_STEPS, "feature"],
|
|
313
|
+
validate="many_to_one",
|
|
314
|
+
)
|
|
315
|
+
sums["active_residual2"] = (sums.sum_p - sums.unit_prompts * sums["mean"]) ** 2
|
|
316
|
+
residuals = (
|
|
317
|
+
sums.groupby([*_STEPS, "feature"], sort=False)
|
|
318
|
+
.agg(
|
|
319
|
+
active_residual2=("active_residual2", "sum"),
|
|
320
|
+
active_prompts2=("unit_prompts2", "sum"),
|
|
321
|
+
)
|
|
322
|
+
.reset_index()
|
|
323
|
+
)
|
|
324
|
+
sparse_result = sparse_result.merge(
|
|
325
|
+
residuals, on=[*_STEPS, "feature"], how="left", validate="one_to_one"
|
|
326
|
+
)
|
|
327
|
+
sparse_result[["active_residual2", "active_prompts2"]] = sparse_result[
|
|
328
|
+
["active_residual2", "active_prompts2"]
|
|
329
|
+
].fillna(0.0)
|
|
330
|
+
sparse_result["residual2"] = (
|
|
331
|
+
sparse_result.active_residual2
|
|
332
|
+
+ (sparse_result.total_prompts2 - sparse_result.active_prompts2)
|
|
333
|
+
* sparse_result["mean"] ** 2
|
|
334
|
+
)
|
|
335
|
+
groups = sparse_result.n_sampling_units.to_numpy(dtype=float)
|
|
336
|
+
variance = (
|
|
337
|
+
sparse_result.residual2.to_numpy()
|
|
338
|
+
/ sparse_result.n_prompts.to_numpy(dtype=float) ** 2
|
|
339
|
+
)
|
|
340
|
+
sparse_result["se"] = np.sqrt(
|
|
341
|
+
np.divide(
|
|
342
|
+
groups * variance,
|
|
343
|
+
groups - 1,
|
|
344
|
+
out=np.full(len(groups), np.nan),
|
|
345
|
+
where=groups > 1,
|
|
346
|
+
)
|
|
347
|
+
)
|
|
348
|
+
return sparse_result
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def aggregate(
|
|
352
|
+
rollouts: pd.DataFrame,
|
|
353
|
+
features: FeatureTables,
|
|
354
|
+
*,
|
|
355
|
+
sampling_units: pd.DataFrame | None = None,
|
|
356
|
+
) -> Aggregation:
|
|
357
|
+
"""Aggregate dense, sparse, and reward values by run and step.
|
|
358
|
+
|
|
359
|
+
Weight exact-prompt means equally. Default SE is SD / sqrt(prompt count);
|
|
360
|
+
explicit units use CR1 SE of the same mean. Fewer than two units yields NaN;
|
|
361
|
+
measured zero variance stays zero. SE excludes detector floors.
|
|
362
|
+
Align features by (run, sample_id).
|
|
363
|
+
"""
|
|
364
|
+
rollouts = validate_rollouts(rollouts)
|
|
365
|
+
dense_names = _validate_features(rollouts, features)
|
|
366
|
+
units = resolve_sampling_units(rollouts, sampling_units)
|
|
367
|
+
singleton_units = not units.duplicated(["run", "sampling_unit_id"]).any()
|
|
368
|
+
if rollouts.empty:
|
|
369
|
+
return Aggregation(
|
|
370
|
+
pd.DataFrame(columns=_SERIES_COLUMNS),
|
|
371
|
+
pd.DataFrame(columns=_SCHEDULE_COLUMNS),
|
|
372
|
+
)
|
|
373
|
+
|
|
374
|
+
schedule = (
|
|
375
|
+
rollouts.assign(prompt_chars=rollouts["prompt"].str.len())
|
|
376
|
+
.groupby(_GROUPS, sort=True, observed=True)
|
|
377
|
+
.agg(n_rollouts=("sample_id", "size"), prompt_chars=("prompt_chars", "first"))
|
|
378
|
+
.reset_index()
|
|
379
|
+
.merge(units, on=["run", "prompt_id"], validate="many_to_one")
|
|
380
|
+
)
|
|
381
|
+
counts = (
|
|
382
|
+
schedule.groupby(_STEPS, sort=True, observed=True)
|
|
383
|
+
.agg(
|
|
384
|
+
n_rollouts=("n_rollouts", "sum"),
|
|
385
|
+
n_prompts=("prompt_id", "size"),
|
|
386
|
+
n_sampling_units=("sampling_unit_id", "nunique"),
|
|
387
|
+
)
|
|
388
|
+
.reset_index()
|
|
389
|
+
)
|
|
390
|
+
parts = _aggregate_dense(
|
|
391
|
+
rollouts, features, dense_names, units, counts, singleton_units
|
|
392
|
+
)
|
|
393
|
+
|
|
394
|
+
sparse_catalog = features.catalog.loc[
|
|
395
|
+
~features.catalog["feature"].isin(dense_names)
|
|
396
|
+
]
|
|
397
|
+
if not sparse_catalog.empty:
|
|
398
|
+
parts.append(
|
|
399
|
+
_aggregate_sparse(
|
|
400
|
+
rollouts,
|
|
401
|
+
features.sparse,
|
|
402
|
+
sparse_catalog,
|
|
403
|
+
schedule,
|
|
404
|
+
counts,
|
|
405
|
+
singleton_units,
|
|
406
|
+
)
|
|
407
|
+
)
|
|
408
|
+
|
|
409
|
+
catalog = pd.concat(
|
|
410
|
+
[
|
|
411
|
+
features.catalog[["feature", "tier", "kind"]],
|
|
412
|
+
pd.DataFrame(
|
|
413
|
+
[
|
|
414
|
+
{
|
|
415
|
+
"feature": "scalar:reward",
|
|
416
|
+
"tier": "structural",
|
|
417
|
+
"kind": "continuous",
|
|
418
|
+
}
|
|
419
|
+
]
|
|
420
|
+
),
|
|
421
|
+
],
|
|
422
|
+
ignore_index=True,
|
|
423
|
+
)
|
|
424
|
+
series = pd.concat(parts, ignore_index=True).merge(
|
|
425
|
+
catalog, on="feature", validate="many_to_one"
|
|
426
|
+
)
|
|
427
|
+
series = (
|
|
428
|
+
series[_SERIES_COLUMNS].sort_values([*_STEPS, "feature"]).reset_index(drop=True)
|
|
429
|
+
)
|
|
430
|
+
return Aggregation(series, schedule[_SCHEDULE_COLUMNS])
|