rolloutdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rolloutdiff/__init__.py +130 -0
- rolloutdiff/_aria_replay.py +346 -0
- rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
- rolloutdiff/_dependence.py +195 -0
- rolloutdiff/_feature_values.py +79 -0
- rolloutdiff/_replay_container.py +568 -0
- rolloutdiff/_report_validation.py +758 -0
- rolloutdiff/_retrospective_policy.py +33 -0
- rolloutdiff/_sampling.py +127 -0
- rolloutdiff/_series_validation.py +67 -0
- rolloutdiff/_watch_report.py +436 -0
- rolloutdiff/_watch_storage.py +800 -0
- rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
- rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
- rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
- rolloutdiff/_web/assets/third-party-notices.txt +112 -0
- rolloutdiff/_web/index.html +14 -0
- rolloutdiff/adapters.py +264 -0
- rolloutdiff/app.py +956 -0
- rolloutdiff/app_evidence.py +678 -0
- rolloutdiff/benchmark_reference.py +458 -0
- rolloutdiff/cli.py +1292 -0
- rolloutdiff/comparison.py +572 -0
- rolloutdiff/comparison_report.py +982 -0
- rolloutdiff/corroboration.py +287 -0
- rolloutdiff/detect.py +307 -0
- rolloutdiff/events.py +173 -0
- rolloutdiff/evidence.py +285 -0
- rolloutdiff/exemplars.py +240 -0
- rolloutdiff/features.py +823 -0
- rolloutdiff/html_report.py +726 -0
- rolloutdiff/inspect_logs.py +383 -0
- rolloutdiff/inspection.py +207 -0
- rolloutdiff/integrations.py +552 -0
- rolloutdiff/investigation.py +1146 -0
- rolloutdiff/notifications.py +406 -0
- rolloutdiff/pipeline.py +809 -0
- rolloutdiff/producer.py +663 -0
- rolloutdiff/prompt_mix.py +258 -0
- rolloutdiff/ranking.py +388 -0
- rolloutdiff/replay.py +589 -0
- rolloutdiff/replication.py +572 -0
- rolloutdiff/report.py +838 -0
- rolloutdiff/retrospective.py +386 -0
- rolloutdiff/schema.py +109 -0
- rolloutdiff/sensitivity.py +202 -0
- rolloutdiff/series.py +430 -0
- rolloutdiff/tool_features.py +422 -0
- rolloutdiff/trajectories.py +236 -0
- rolloutdiff/watch-compatibility.json +13 -0
- rolloutdiff/watch.py +742 -0
- rolloutdiff-0.1.0.dist-info/METADATA +158 -0
- rolloutdiff-0.1.0.dist-info/RECORD +62 -0
- rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
- rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
- rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
- rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""Execution policy for experimental full-run localization."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import pandas as pd
|
|
6
|
+
|
|
7
|
+
from .retrospective import _CHANGES, _DIAGNOSTICS, _SCREENING, RetrospectiveResult
|
|
8
|
+
|
|
9
|
+
ALGORITHM = "penalized_l2_mean_shift"
|
|
10
|
+
WARNING = (
|
|
11
|
+
"Experimental retrospective localization. The historical AR(1) procedure selected "
|
|
12
|
+
"changes in 16/768 fitted null trials (2.08%), above the original 0.5% ceiling. "
|
|
13
|
+
"These trials do not establish an error rate for this run."
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def retrospective_metadata(settings: dict | None) -> dict:
|
|
18
|
+
enabled = settings is not None
|
|
19
|
+
return {
|
|
20
|
+
"status": "ran" if enabled else "disabled",
|
|
21
|
+
"experimental": enabled,
|
|
22
|
+
"algorithm": ALGORITHM if enabled else None,
|
|
23
|
+
"effective_settings": settings,
|
|
24
|
+
"warning": WARNING if enabled else None,
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def disabled_result() -> RetrospectiveResult:
|
|
29
|
+
return RetrospectiveResult(
|
|
30
|
+
pd.DataFrame(columns=_CHANGES),
|
|
31
|
+
pd.DataFrame(columns=_DIAGNOSTICS),
|
|
32
|
+
pd.DataFrame(columns=_SCREENING),
|
|
33
|
+
)
|
rolloutdiff/_sampling.py
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""Explicit dependence units, separate from canonical exact prompt identities."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
import pandas as pd
|
|
7
|
+
|
|
8
|
+
_KEYS = ["run", "prompt_id"]
|
|
9
|
+
_COLUMNS = [*_KEYS, "sampling_unit_id"]
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _prompt_keys(rollouts: pd.DataFrame) -> pd.DataFrame:
|
|
13
|
+
if (
|
|
14
|
+
not isinstance(rollouts, pd.DataFrame)
|
|
15
|
+
or not rollouts.columns.is_unique
|
|
16
|
+
or not set(_KEYS).issubset(rollouts.columns)
|
|
17
|
+
):
|
|
18
|
+
raise ValueError("rollouts must contain unique run and prompt_id columns")
|
|
19
|
+
keys = rollouts[_KEYS].drop_duplicates()
|
|
20
|
+
for name in _KEYS:
|
|
21
|
+
if (
|
|
22
|
+
not keys[name]
|
|
23
|
+
.map(lambda value: isinstance(value, str) and bool(value.strip()))
|
|
24
|
+
.all()
|
|
25
|
+
):
|
|
26
|
+
raise ValueError(f"{name} must contain nonblank string identifiers")
|
|
27
|
+
return keys.sort_values(_KEYS).reset_index(drop=True)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def resolve_sampling_units(
|
|
31
|
+
rollouts: pd.DataFrame,
|
|
32
|
+
mapping: pd.DataFrame | None = None,
|
|
33
|
+
) -> pd.DataFrame:
|
|
34
|
+
"""Return a stable unit per (run, exact prompt ID).
|
|
35
|
+
|
|
36
|
+
Supplied maps must exactly cover observed keys. Units are scoped by run and
|
|
37
|
+
may group prompts.
|
|
38
|
+
"""
|
|
39
|
+
keys = _prompt_keys(rollouts)
|
|
40
|
+
if mapping is None:
|
|
41
|
+
return keys.assign(sampling_unit_id=keys.prompt_id)
|
|
42
|
+
if (
|
|
43
|
+
not isinstance(mapping, pd.DataFrame)
|
|
44
|
+
or not mapping.columns.is_unique
|
|
45
|
+
or set(mapping.columns) != set(_COLUMNS)
|
|
46
|
+
):
|
|
47
|
+
raise ValueError(f"sampling_units must contain exactly {_COLUMNS}")
|
|
48
|
+
units = mapping[_COLUMNS].copy()
|
|
49
|
+
for name in _COLUMNS:
|
|
50
|
+
if (
|
|
51
|
+
not units[name]
|
|
52
|
+
.map(lambda value: isinstance(value, str) and bool(value.strip()))
|
|
53
|
+
.all()
|
|
54
|
+
):
|
|
55
|
+
raise ValueError(
|
|
56
|
+
f"sampling_units {name} must contain nonblank string identifiers"
|
|
57
|
+
)
|
|
58
|
+
if units.duplicated(_KEYS).any():
|
|
59
|
+
raise ValueError("sampling_units contains duplicate (run, prompt_id) keys")
|
|
60
|
+
expected = pd.MultiIndex.from_frame(keys)
|
|
61
|
+
actual = pd.MultiIndex.from_frame(units[_KEYS])
|
|
62
|
+
if len(actual) != len(expected) or not actual.isin(expected).all():
|
|
63
|
+
raise ValueError("sampling_units keys must exactly match rollout prompt keys")
|
|
64
|
+
return units.sort_values(_KEYS).reset_index(drop=True)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def subset_sampling_units(
|
|
68
|
+
mapping: pd.DataFrame, rollouts: pd.DataFrame
|
|
69
|
+
) -> pd.DataFrame:
|
|
70
|
+
"""Restrict a validated map to a rollout subset, checking full subset coverage."""
|
|
71
|
+
keys = _prompt_keys(rollouts)
|
|
72
|
+
if (
|
|
73
|
+
not isinstance(mapping, pd.DataFrame)
|
|
74
|
+
or not mapping.columns.is_unique
|
|
75
|
+
or set(mapping.columns) != set(_COLUMNS)
|
|
76
|
+
):
|
|
77
|
+
raise ValueError(f"sampling_units must contain exactly {_COLUMNS}")
|
|
78
|
+
selected = pd.MultiIndex.from_frame(mapping[_KEYS]).isin(
|
|
79
|
+
pd.MultiIndex.from_frame(keys)
|
|
80
|
+
)
|
|
81
|
+
return resolve_sampling_units(keys, mapping.loc[selected])
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def cluster_standard_error(values, unit_ids) -> float | None:
|
|
85
|
+
"""CR1 SE of an equal-prompt mean over explicit sampling units.
|
|
86
|
+
|
|
87
|
+
For P prompts and U units: sqrt(U/(U-1) * sum(unit residual sums**2)) / P.
|
|
88
|
+
Unequal unit sizes retain prompt weights. Fewer than two units returns None.
|
|
89
|
+
"""
|
|
90
|
+
data = np.asarray(values)
|
|
91
|
+
units = np.asarray(unit_ids, dtype=object)
|
|
92
|
+
if (
|
|
93
|
+
data.ndim != 1
|
|
94
|
+
or units.ndim != 1
|
|
95
|
+
or len(data) != len(units)
|
|
96
|
+
or data.dtype.kind not in "biuf"
|
|
97
|
+
or not np.isfinite(data).all()
|
|
98
|
+
):
|
|
99
|
+
raise ValueError(
|
|
100
|
+
"cluster SE requires aligned finite real values and unit identifiers"
|
|
101
|
+
)
|
|
102
|
+
if not all(isinstance(value, str) and bool(value.strip()) for value in units):
|
|
103
|
+
raise ValueError("cluster SE unit identifiers must be nonblank strings")
|
|
104
|
+
if not len(data):
|
|
105
|
+
return None
|
|
106
|
+
codes, names = pd.factorize(units, sort=True)
|
|
107
|
+
count = len(names)
|
|
108
|
+
if count < 2:
|
|
109
|
+
return None
|
|
110
|
+
# Preserve small differences by shifting before scaling. If subtraction
|
|
111
|
+
# overflows, scale raw values first. Neither path squares raw data.
|
|
112
|
+
data = data.astype(float)
|
|
113
|
+
with np.errstate(over="ignore", invalid="ignore"):
|
|
114
|
+
shifted = data - data[0]
|
|
115
|
+
centered_data = shifted if np.isfinite(shifted).all() else data
|
|
116
|
+
scale = float(np.max(np.abs(centered_data)))
|
|
117
|
+
if scale == 0:
|
|
118
|
+
return 0.0
|
|
119
|
+
normalized = centered_data / scale
|
|
120
|
+
residuals = normalized - normalized.mean()
|
|
121
|
+
sums = np.bincount(codes, weights=residuals, minlength=count) / len(data)
|
|
122
|
+
normalized_se = np.sqrt(count / (count - 1) * np.dot(sums, sums))
|
|
123
|
+
with np.errstate(over="ignore", invalid="ignore"):
|
|
124
|
+
result = float(normalized_se * scale)
|
|
125
|
+
if not np.isfinite(result):
|
|
126
|
+
raise ValueError("numerical overflow computing cluster standard error")
|
|
127
|
+
return result
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Shared validation of step-series identities, values, and measured uncertainty."""
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
import pandas as pd
|
|
5
|
+
from pandas.api.types import is_complex_dtype, is_integer_dtype, is_numeric_dtype
|
|
6
|
+
|
|
7
|
+
_REQUIRED = [
|
|
8
|
+
"run",
|
|
9
|
+
"step",
|
|
10
|
+
"feature",
|
|
11
|
+
"tier",
|
|
12
|
+
"kind",
|
|
13
|
+
"mean",
|
|
14
|
+
"se",
|
|
15
|
+
"n_rollouts",
|
|
16
|
+
"n_prompts",
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def validate_series(series: pd.DataFrame) -> pd.DataFrame:
|
|
21
|
+
if not isinstance(series, pd.DataFrame) or not series.columns.is_unique:
|
|
22
|
+
raise ValueError("series must be a DataFrame with unique columns")
|
|
23
|
+
if set(_REQUIRED) - set(series.columns):
|
|
24
|
+
raise ValueError("series is missing required aggregate columns")
|
|
25
|
+
frame = series[_REQUIRED].copy()
|
|
26
|
+
frame["n_sampling_units"] = series.get("n_sampling_units", frame.n_prompts)
|
|
27
|
+
if frame.empty:
|
|
28
|
+
return frame.reset_index(drop=True)
|
|
29
|
+
for name in ("run", "feature"):
|
|
30
|
+
if not frame[name].map(lambda v: isinstance(v, str) and bool(v.strip())).all():
|
|
31
|
+
raise ValueError(f"{name} must contain nonempty strings")
|
|
32
|
+
for name in ("step", "n_rollouts", "n_prompts", "n_sampling_units"):
|
|
33
|
+
if not is_integer_dtype(frame[name].dtype) or frame[name].isna().any():
|
|
34
|
+
raise ValueError(f"{name} must contain integers")
|
|
35
|
+
if (frame[name] < (0 if name == "step" else 1)).any():
|
|
36
|
+
raise ValueError(f"invalid {name}")
|
|
37
|
+
if (frame.n_sampling_units > frame.n_prompts).any():
|
|
38
|
+
raise ValueError("n_sampling_units cannot exceed n_prompts")
|
|
39
|
+
if (frame.n_prompts > frame.n_rollouts).any():
|
|
40
|
+
raise ValueError("n_prompts cannot exceed n_rollouts")
|
|
41
|
+
for name in ("mean", "se"):
|
|
42
|
+
if not is_numeric_dtype(frame[name].dtype) or is_complex_dtype(
|
|
43
|
+
frame[name].dtype
|
|
44
|
+
):
|
|
45
|
+
raise ValueError(f"{name} must contain real numbers")
|
|
46
|
+
frame[name] = frame[name].to_numpy(dtype=float, na_value=np.nan)
|
|
47
|
+
if not np.isfinite(frame["mean"]).all():
|
|
48
|
+
raise ValueError("mean must be finite")
|
|
49
|
+
if np.isinf(frame.se).any() or frame.se.lt(0).any():
|
|
50
|
+
raise ValueError("se must be nonnegative or unavailable (NaN)")
|
|
51
|
+
if (frame.n_prompts.eq(1) & frame.se.notna()).any():
|
|
52
|
+
raise ValueError("across-prompt SE must be unavailable with one prompt")
|
|
53
|
+
if (frame.n_sampling_units.eq(1) & frame.se.notna()).any():
|
|
54
|
+
raise ValueError("across-unit SE must be unavailable with one sampling unit")
|
|
55
|
+
if (
|
|
56
|
+
not frame.tier.isin(["structural", "prose"]).all()
|
|
57
|
+
or not frame.kind.isin(["binary", "continuous"]).all()
|
|
58
|
+
):
|
|
59
|
+
raise ValueError("invalid feature metadata")
|
|
60
|
+
if frame.duplicated(["run", "feature", "step"]).any():
|
|
61
|
+
raise ValueError("duplicate (run, feature, step) series rows")
|
|
62
|
+
if frame.groupby(["run", "feature"])[["tier", "kind"]].nunique().gt(1).any().any():
|
|
63
|
+
raise ValueError("feature metadata changes within a series")
|
|
64
|
+
binary = frame.loc[frame.kind.eq("binary"), "mean"]
|
|
65
|
+
if not binary.between(0, 1).all():
|
|
66
|
+
raise ValueError("binary prevalence must be between zero and one")
|
|
67
|
+
return frame.sort_values(["run", "feature", "step"]).reset_index(drop=True)
|
|
@@ -0,0 +1,436 @@
|
|
|
1
|
+
"""Validate and inspect immutable passive-watch report snapshots."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from html import escape
|
|
7
|
+
from itertools import pairwise
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Annotated, Literal
|
|
10
|
+
|
|
11
|
+
from pydantic import AfterValidator, BaseModel, ConfigDict, Field, ValidationError
|
|
12
|
+
|
|
13
|
+
from ._report_validation import _constant, _object, validate_report
|
|
14
|
+
from .adapters import _TRL_FILE, _VERL_FILE
|
|
15
|
+
from .comparison import canonical_fingerprint
|
|
16
|
+
from .html_report import render_report
|
|
17
|
+
from .inspection import resolve_exemplars
|
|
18
|
+
from .report import _IDENTIFIERS, _STEPS, _identifier
|
|
19
|
+
from .sensitivity import _DEFAULT_RETROSPECTIVE
|
|
20
|
+
|
|
21
|
+
Identifier = Annotated[str, AfterValidator(_identifier)]
|
|
22
|
+
Count = Annotated[int, Field(ge=0)]
|
|
23
|
+
PositiveCount = Annotated[int, Field(ge=1)]
|
|
24
|
+
Digest = Annotated[str, Field(pattern=r"^[0-9a-f]{64}$")]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class _Model(BaseModel):
|
|
28
|
+
model_config = ConfigDict(strict=True, extra="forbid", allow_inf_nan=False)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class _Policy(_Model):
|
|
32
|
+
id: Literal["prefix-peak-v1", "prefix-peak-v2"]
|
|
33
|
+
configuration_sha256: Digest
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class _File(_Model):
|
|
37
|
+
path: str
|
|
38
|
+
sha256: Digest
|
|
39
|
+
size_bytes: Count
|
|
40
|
+
n_rollouts: PositiveCount
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class _Source(_Model):
|
|
44
|
+
run: Identifier
|
|
45
|
+
format: Literal["trl", "verl"]
|
|
46
|
+
reward_column: Identifier | None
|
|
47
|
+
canonical_sha256: Digest
|
|
48
|
+
files: list[_File]
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class _Catalog(_Model):
|
|
52
|
+
active: Count
|
|
53
|
+
added: Count
|
|
54
|
+
removed: Count
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class _Emission(_Model):
|
|
58
|
+
feature: Identifier
|
|
59
|
+
first_admitted_step: Count
|
|
60
|
+
first_emitted_step: Count
|
|
61
|
+
first_emission_sequence: PositiveCount
|
|
62
|
+
onset_step: Count
|
|
63
|
+
alert_step: Count
|
|
64
|
+
direction: Literal["up", "down"]
|
|
65
|
+
active: bool
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class _Screening(_Model):
|
|
69
|
+
run: Identifier
|
|
70
|
+
n_series: Count
|
|
71
|
+
n_screened: Count
|
|
72
|
+
n_changes: Count
|
|
73
|
+
n_unscorable: Count
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class _Change(_Model):
|
|
77
|
+
feature: Identifier
|
|
78
|
+
kind: Literal["binary", "continuous"]
|
|
79
|
+
tier: Literal["structural", "prose"]
|
|
80
|
+
changepoint_step: Count
|
|
81
|
+
previous_step: Count
|
|
82
|
+
delta: int | float | None
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class _Retrospective(_Model):
|
|
86
|
+
as_of_step: Count
|
|
87
|
+
screening: list[_Screening]
|
|
88
|
+
changes: list[_Change]
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class _Report(_Model):
|
|
92
|
+
report_type: Literal["watch"]
|
|
93
|
+
schema_version: int
|
|
94
|
+
sequence: PositiveCount
|
|
95
|
+
as_of_step: Count
|
|
96
|
+
policy: _Policy
|
|
97
|
+
source: _Source
|
|
98
|
+
catalog: _Catalog
|
|
99
|
+
emissions: list[_Emission]
|
|
100
|
+
analysis: dict
|
|
101
|
+
retrospective: _Retrospective | None
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _source_steps(report: _Report) -> list[int]:
|
|
105
|
+
source = report.source
|
|
106
|
+
pattern = _TRL_FILE if source.format == "trl" else _VERL_FILE
|
|
107
|
+
steps = []
|
|
108
|
+
for entry in source.files:
|
|
109
|
+
match = pattern.fullmatch(entry.path)
|
|
110
|
+
if match is None:
|
|
111
|
+
raise ValueError("watch report source filename is invalid")
|
|
112
|
+
steps.append(int(match.group(1)))
|
|
113
|
+
if (
|
|
114
|
+
len(steps) != report.sequence
|
|
115
|
+
or len(set(steps)) != len(steps)
|
|
116
|
+
or steps != sorted(steps)
|
|
117
|
+
or steps[-1] != report.as_of_step
|
|
118
|
+
or len({right - left for left, right in pairwise(steps)}) > 1
|
|
119
|
+
or (source.format == "verl" and source.reward_column is not None)
|
|
120
|
+
):
|
|
121
|
+
raise ValueError(
|
|
122
|
+
"watch report source schedule or loader settings are inconsistent"
|
|
123
|
+
)
|
|
124
|
+
return steps
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _check_analysis_fields(value, as_of_step: int) -> None:
|
|
128
|
+
step_fields = _STEPS | {
|
|
129
|
+
"cutoff_step",
|
|
130
|
+
"window_start",
|
|
131
|
+
"window_end",
|
|
132
|
+
"before_start",
|
|
133
|
+
"before_end",
|
|
134
|
+
"after_start",
|
|
135
|
+
"after_end",
|
|
136
|
+
}
|
|
137
|
+
if isinstance(value, list):
|
|
138
|
+
for item in value:
|
|
139
|
+
_check_analysis_fields(item, as_of_step)
|
|
140
|
+
elif isinstance(value, dict):
|
|
141
|
+
for name, item in value.items():
|
|
142
|
+
if item is not None:
|
|
143
|
+
if name in _IDENTIFIERS:
|
|
144
|
+
_identifier(item)
|
|
145
|
+
if name in step_fields and item > as_of_step:
|
|
146
|
+
raise ValueError("watch report analysis includes a future step")
|
|
147
|
+
_check_analysis_fields(item, as_of_step)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _check_emissions(report: _Report, steps: list[int], members: dict) -> None:
|
|
151
|
+
seen = set()
|
|
152
|
+
active = set()
|
|
153
|
+
for item in report.emissions:
|
|
154
|
+
anchors = (
|
|
155
|
+
item.first_admitted_step,
|
|
156
|
+
item.first_emitted_step,
|
|
157
|
+
item.onset_step,
|
|
158
|
+
item.alert_step,
|
|
159
|
+
)
|
|
160
|
+
if (
|
|
161
|
+
item.feature in seen
|
|
162
|
+
or not set(anchors).issubset(steps)
|
|
163
|
+
or not item.first_admitted_step
|
|
164
|
+
<= item.first_emitted_step
|
|
165
|
+
<= report.as_of_step
|
|
166
|
+
or not item.onset_step <= item.alert_step <= item.first_emitted_step
|
|
167
|
+
or item.first_emission_sequence > report.sequence
|
|
168
|
+
or steps[item.first_emission_sequence - 1] != item.first_emitted_step
|
|
169
|
+
):
|
|
170
|
+
raise ValueError(
|
|
171
|
+
"watch report emission dates or identities are inconsistent"
|
|
172
|
+
)
|
|
173
|
+
seen.add(item.feature)
|
|
174
|
+
if item.active:
|
|
175
|
+
member = members.get(item.feature)
|
|
176
|
+
if member is None or any(
|
|
177
|
+
getattr(item, name) != member[name]
|
|
178
|
+
for name in ("onset_step", "alert_step", "direction")
|
|
179
|
+
):
|
|
180
|
+
raise ValueError(
|
|
181
|
+
"watch report emission does not match current analysis"
|
|
182
|
+
)
|
|
183
|
+
active.add(item.feature)
|
|
184
|
+
if active != set(members):
|
|
185
|
+
raise ValueError(
|
|
186
|
+
"watch report emissions do not cover current sequential alerts"
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _check_retrospective(report: _Report, steps: list[int], run: dict) -> None:
|
|
191
|
+
retrospective = report.retrospective
|
|
192
|
+
if report.schema_version == 2:
|
|
193
|
+
inner = run["retrospective"]
|
|
194
|
+
if inner["status"] == "disabled":
|
|
195
|
+
if retrospective is not None:
|
|
196
|
+
raise ValueError(
|
|
197
|
+
"watch report contains disabled retrospective evidence"
|
|
198
|
+
)
|
|
199
|
+
return
|
|
200
|
+
if (
|
|
201
|
+
retrospective is None
|
|
202
|
+
or report.analysis["retrospective"]["effective_settings"]
|
|
203
|
+
!= _DEFAULT_RETROSPECTIVE
|
|
204
|
+
):
|
|
205
|
+
raise ValueError("watch report retrospective policy is inconsistent")
|
|
206
|
+
if retrospective is None:
|
|
207
|
+
raise ValueError("watch report retrospective evidence is missing")
|
|
208
|
+
n_series = run["counts"]["series"]
|
|
209
|
+
if (
|
|
210
|
+
retrospective.as_of_step != report.as_of_step
|
|
211
|
+
or len(retrospective.screening) != 1
|
|
212
|
+
):
|
|
213
|
+
raise ValueError("watch report retrospective scope is inconsistent")
|
|
214
|
+
screening = retrospective.screening[0]
|
|
215
|
+
if (
|
|
216
|
+
screening.run != report.source.run
|
|
217
|
+
or screening.n_series != n_series
|
|
218
|
+
or screening.n_screened + screening.n_unscorable != n_series
|
|
219
|
+
or screening.n_changes != len(retrospective.changes)
|
|
220
|
+
or screening.n_changes > screening.n_screened
|
|
221
|
+
):
|
|
222
|
+
raise ValueError("watch report retrospective counts are inconsistent")
|
|
223
|
+
if report.schema_version == 2:
|
|
224
|
+
inner = run["retrospective"]
|
|
225
|
+
if screening.model_dump(exclude={"run"}) != inner["screening"] or {
|
|
226
|
+
(c.feature, c.changepoint_step) for c in retrospective.changes
|
|
227
|
+
} != {(c["feature"], c["changepoint_step"]) for c in inner["changes"]}:
|
|
228
|
+
raise ValueError(
|
|
229
|
+
"watch report retrospective evidence differs from analysis"
|
|
230
|
+
)
|
|
231
|
+
predecessors = {right: left for left, right in pairwise(steps)}
|
|
232
|
+
seen = set()
|
|
233
|
+
for change in retrospective.changes:
|
|
234
|
+
if (
|
|
235
|
+
change.feature in seen
|
|
236
|
+
or predecessors.get(change.changepoint_step) != change.previous_step
|
|
237
|
+
):
|
|
238
|
+
raise ValueError("watch report retrospective boundaries are inconsistent")
|
|
239
|
+
seen.add(change.feature)
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def validate_watch_report(value: object) -> dict:
|
|
243
|
+
"""Validate the watch envelope and return a detached, sanitized snapshot."""
|
|
244
|
+
if not isinstance(value, dict):
|
|
245
|
+
raise ValueError("watch report must be a JSON object")
|
|
246
|
+
if type(value.get("schema_version")) is not int or value["schema_version"] not in (
|
|
247
|
+
1,
|
|
248
|
+
2,
|
|
249
|
+
):
|
|
250
|
+
raise ValueError("unsupported watch report version")
|
|
251
|
+
try:
|
|
252
|
+
report = _Report.model_validate(value)
|
|
253
|
+
analysis = validate_report(report.analysis)
|
|
254
|
+
legacy = report.schema_version == 1
|
|
255
|
+
if (
|
|
256
|
+
report.policy.id != ("prefix-peak-v1" if legacy else "prefix-peak-v2")
|
|
257
|
+
or analysis["schema_version"] != (1 if legacy else 3)
|
|
258
|
+
or "sensitivity" in analysis
|
|
259
|
+
):
|
|
260
|
+
raise ValueError("watch report policy differs from the analysis contract")
|
|
261
|
+
_check_analysis_fields(analysis, report.as_of_step)
|
|
262
|
+
except (ValidationError, ValueError, RecursionError):
|
|
263
|
+
raise ValueError("invalid watch report fields or types") from None
|
|
264
|
+
steps = _source_steps(report)
|
|
265
|
+
if len(analysis["runs"]) != 1 or analysis["runs"][0]["run"] != report.source.run:
|
|
266
|
+
raise ValueError("watch report requires one matching analysis run")
|
|
267
|
+
run = analysis["runs"][0]
|
|
268
|
+
counts = run["counts"]
|
|
269
|
+
if (
|
|
270
|
+
run["status"] != "analyzed"
|
|
271
|
+
or counts["rollouts"] != sum(entry.n_rollouts for entry in report.source.files)
|
|
272
|
+
or counts["observed_steps"] != report.sequence
|
|
273
|
+
or not 0 < counts["prompts"] <= counts["rollouts"]
|
|
274
|
+
or counts["series"] != report.catalog.active + 1
|
|
275
|
+
or counts["screened_series"] + counts["unscorable_series"] != counts["series"]
|
|
276
|
+
or counts["sequential_alerts"] > counts["screened_series"]
|
|
277
|
+
or report.catalog.added > report.catalog.active
|
|
278
|
+
):
|
|
279
|
+
raise ValueError("watch report analysis or catalog counts are inconsistent")
|
|
280
|
+
members = {
|
|
281
|
+
member["feature"]: member
|
|
282
|
+
for event in run["events"]
|
|
283
|
+
for member in event["members"]
|
|
284
|
+
}
|
|
285
|
+
_check_emissions(report, steps, members)
|
|
286
|
+
_check_retrospective(report, steps, run)
|
|
287
|
+
result = report.model_dump()
|
|
288
|
+
result["analysis"] = analysis
|
|
289
|
+
return result
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def load_watch_report(path: str | Path) -> dict:
|
|
293
|
+
"""Read strict UTF-8 watch JSON without echoing malformed payloads."""
|
|
294
|
+
try:
|
|
295
|
+
with Path(path).open(encoding="utf-8") as stream:
|
|
296
|
+
value = json.load(
|
|
297
|
+
stream, object_pairs_hook=_object, parse_constant=_constant
|
|
298
|
+
)
|
|
299
|
+
except (ValueError, RecursionError):
|
|
300
|
+
raise ValueError("invalid watch report JSON") from None
|
|
301
|
+
return validate_watch_report(value)
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def _table(headers: tuple[str, ...], rows) -> str:
|
|
305
|
+
head = "".join(f"<th>{escape(header)}</th>" for header in headers)
|
|
306
|
+
body = "".join(
|
|
307
|
+
"<tr>"
|
|
308
|
+
+ "".join(
|
|
309
|
+
'<td><span class="unavailable">Unavailable</span></td>'
|
|
310
|
+
if item is None
|
|
311
|
+
else f"<td>{escape(str(item))}</td>"
|
|
312
|
+
for item in row
|
|
313
|
+
)
|
|
314
|
+
+ "</tr>"
|
|
315
|
+
for row in rows
|
|
316
|
+
)
|
|
317
|
+
return (
|
|
318
|
+
'<div class="table-scroll" tabindex="0">'
|
|
319
|
+
f"<table><thead><tr>{head}</tr></thead><tbody>{body}</tbody></table></div>"
|
|
320
|
+
)
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def render_watch_report(value: object) -> str:
|
|
324
|
+
"""Add escaped watch timing and retrospective context to the analysis HTML."""
|
|
325
|
+
report = validate_watch_report(value)
|
|
326
|
+
step = report["as_of_step"]
|
|
327
|
+
context = [
|
|
328
|
+
'<section class="watch-context" aria-label="Watch snapshot">',
|
|
329
|
+
(
|
|
330
|
+
f"<h2>As of step {step}</h2><p>Commit sequence {report['sequence']}; "
|
|
331
|
+
f"policy <code>{escape(report['policy']['id'])}</code>.</p>"
|
|
332
|
+
),
|
|
333
|
+
(
|
|
334
|
+
"<p>First emissions identify the processed prefix where this watch exposed a sequential alert. "
|
|
335
|
+
"Inferred onset and detector alert steps can precede emission. Prefix steps are not wall-clock training or delivery timestamps.</p>"
|
|
336
|
+
),
|
|
337
|
+
"<h3>Sequential emissions</h3>",
|
|
338
|
+
_table(
|
|
339
|
+
(
|
|
340
|
+
"Feature",
|
|
341
|
+
"First admitted prefix step",
|
|
342
|
+
"First emitted prefix step",
|
|
343
|
+
"Emission sequence",
|
|
344
|
+
"Onset step",
|
|
345
|
+
"Alert step",
|
|
346
|
+
"Direction",
|
|
347
|
+
"Current selection",
|
|
348
|
+
),
|
|
349
|
+
(
|
|
350
|
+
(
|
|
351
|
+
item["feature"],
|
|
352
|
+
item["first_admitted_step"],
|
|
353
|
+
item["first_emitted_step"],
|
|
354
|
+
item["first_emission_sequence"],
|
|
355
|
+
item["onset_step"],
|
|
356
|
+
item["alert_step"],
|
|
357
|
+
item["direction"],
|
|
358
|
+
"Active" if item["active"] else "Selected out",
|
|
359
|
+
)
|
|
360
|
+
for item in sorted(
|
|
361
|
+
report["emissions"], key=lambda item: item["feature"]
|
|
362
|
+
)
|
|
363
|
+
),
|
|
364
|
+
),
|
|
365
|
+
]
|
|
366
|
+
retrospective = report["retrospective"]
|
|
367
|
+
if retrospective is None:
|
|
368
|
+
context.append("<p>Retrospective localization was disabled for this watch.</p>")
|
|
369
|
+
else:
|
|
370
|
+
context.extend(
|
|
371
|
+
[
|
|
372
|
+
f"<h3>Retrospective as of step {step}</h3>",
|
|
373
|
+
(
|
|
374
|
+
"<p>These boundaries use the current prefix and may change as steps arrive. "
|
|
375
|
+
"They are separate from sequential emissions.</p>"
|
|
376
|
+
),
|
|
377
|
+
_table(
|
|
378
|
+
("Series", "Screened", "Changes", "Unscorable"),
|
|
379
|
+
(
|
|
380
|
+
(
|
|
381
|
+
item["n_series"],
|
|
382
|
+
item["n_screened"],
|
|
383
|
+
item["n_changes"],
|
|
384
|
+
item["n_unscorable"],
|
|
385
|
+
)
|
|
386
|
+
for item in retrospective["screening"]
|
|
387
|
+
),
|
|
388
|
+
),
|
|
389
|
+
_table(
|
|
390
|
+
(
|
|
391
|
+
"Feature",
|
|
392
|
+
"Kind",
|
|
393
|
+
"Tier",
|
|
394
|
+
"Previous step",
|
|
395
|
+
"Changepoint step",
|
|
396
|
+
"Delta",
|
|
397
|
+
),
|
|
398
|
+
(
|
|
399
|
+
(
|
|
400
|
+
item["feature"],
|
|
401
|
+
item["kind"],
|
|
402
|
+
item["tier"],
|
|
403
|
+
item["previous_step"],
|
|
404
|
+
item["changepoint_step"],
|
|
405
|
+
item["delta"],
|
|
406
|
+
)
|
|
407
|
+
for item in sorted(
|
|
408
|
+
retrospective["changes"], key=lambda item: item["feature"]
|
|
409
|
+
)
|
|
410
|
+
),
|
|
411
|
+
),
|
|
412
|
+
]
|
|
413
|
+
)
|
|
414
|
+
context.append("</section>")
|
|
415
|
+
return (
|
|
416
|
+
render_report(report["analysis"])
|
|
417
|
+
.replace("<h1>Investigation report</h1>", "<h1>Passive watch report</h1>", 1)
|
|
418
|
+
.replace("</header>", "</header>" + "".join(context), 1)
|
|
419
|
+
)
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
def resolve_watch_exemplars(
|
|
423
|
+
report: object, rollouts, *, event_rank: int, with_absent: bool = False
|
|
424
|
+
) -> dict:
|
|
425
|
+
"""Resolve only the canonical prefix bound to this watch snapshot."""
|
|
426
|
+
report = validate_watch_report(report)
|
|
427
|
+
run = report["source"]["run"]
|
|
428
|
+
if canonical_fingerprint(rollouts, run=run) != report["source"]["canonical_sha256"]:
|
|
429
|
+
raise ValueError("watch report canonical prefix fingerprint does not match")
|
|
430
|
+
return resolve_exemplars(
|
|
431
|
+
report["analysis"],
|
|
432
|
+
rollouts,
|
|
433
|
+
run=run,
|
|
434
|
+
event_rank=event_rank,
|
|
435
|
+
with_absent=with_absent,
|
|
436
|
+
)
|