millforge 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- millforge/__init__.py +1174 -0
- millforge/_forge/LICENSE +21 -0
- millforge/_forge/PROVENANCE.json +295 -0
- millforge/_forge/UPDATE_POLICY.md +24 -0
- millforge/_forge/__init__.py +14 -0
- millforge/_forge/adapter.py +2232 -0
- millforge/_forge/base_runner.py +121 -0
- millforge/_forge/clients/__init__.py +10 -0
- millforge/_forge/clients/base.py +200 -0
- millforge/_forge/context/__init__.py +23 -0
- millforge/_forge/context/manager.py +178 -0
- millforge/_forge/context/strategies.py +335 -0
- millforge/_forge/core/__init__.py +16 -0
- millforge/_forge/core/inference.py +433 -0
- millforge/_forge/core/messages.py +119 -0
- millforge/_forge/core/runner.py +479 -0
- millforge/_forge/core/steps.py +108 -0
- millforge/_forge/core/workflow.py +400 -0
- millforge/_forge/errors.py +222 -0
- millforge/_forge/guardrails/__init__.py +21 -0
- millforge/_forge/guardrails/error_tracker.py +71 -0
- millforge/_forge/guardrails/guardrails.py +194 -0
- millforge/_forge/guardrails/nudge.py +47 -0
- millforge/_forge/guardrails/response_validator.py +119 -0
- millforge/_forge/guardrails/step_enforcer.py +183 -0
- millforge/_forge/prompts/__init__.py +16 -0
- millforge/_forge/prompts/nudges.py +95 -0
- millforge/_forge/prompts/templates.py +285 -0
- millforge/_version.py +3 -0
- millforge/artifacts.py +570 -0
- millforge/base/__init__.py +97 -0
- millforge/base/composition.py +402 -0
- millforge/base/context.py +285 -0
- millforge/base/harness.py +138 -0
- millforge/base/identity.py +465 -0
- millforge/base/options.py +34 -0
- millforge/base/platform.py +17 -0
- millforge/base/prompt.py +317 -0
- millforge/base/runner.py +546 -0
- millforge/compiled_plan.py +970 -0
- millforge/compiler/__init__.py +231 -0
- millforge/compiler/artifact_validation.py +257 -0
- millforge/compiler/canonicalization.py +169 -0
- millforge/compiler/capabilities.py +66 -0
- millforge/compiler/catalogs.py +500 -0
- millforge/compiler/diagnostics.py +491 -0
- millforge/compiler/graph.py +678 -0
- millforge/compiler/lowering.py +198 -0
- millforge/compiler/output.py +692 -0
- millforge/compiler/parsing.py +1424 -0
- millforge/compiler/requests.py +1180 -0
- millforge/compiler/schema_validation.py +272 -0
- millforge/compiler/semantic.py +490 -0
- millforge/compiler/service.py +448 -0
- millforge/compiler/source.py +375 -0
- millforge/compiler/validators.py +184 -0
- millforge/connectors/__init__.py +95 -0
- millforge/connectors/admission.py +801 -0
- millforge/connectors/broker.py +202 -0
- millforge/connectors/contracts.py +1159 -0
- millforge/connectors/diagnostics.py +189 -0
- millforge/connectors/fake.py +66 -0
- millforge/connectors/runtime.py +236 -0
- millforge/contracts.py +2860 -0
- millforge/custom_tools/__init__.py +67 -0
- millforge/custom_tools/compiler.py +724 -0
- millforge/custom_tools/contracts.py +1093 -0
- millforge/custom_tools/diagnostics.py +205 -0
- millforge/eval_artifacts.py +952 -0
- millforge/eval_boundary.py +2435 -0
- millforge/eval_fixtures/__init__.py +1 -0
- millforge/eval_fixtures/default_pack/__init__.py +1 -0
- millforge/eval_fixtures/default_pack/fixtures/fixture.08a.bug_diagnosis.traceback.v1.json +52 -0
- millforge/eval_fixtures/default_pack/fixtures/fixture.08a.direct_edit.import_sort.v1.json +52 -0
- millforge/eval_fixtures/default_pack/fixtures/fixture.08a.evidence_discipline.no_source_change.v1.json +51 -0
- millforge/eval_fixtures/default_pack/fixtures/fixture.08a.false_closure.visible_green.v1.json +52 -0
- millforge/eval_fixtures/default_pack/fixtures/fixture.08a.multi_file.api_contract.v1.json +54 -0
- millforge/eval_fixtures/default_pack/fixtures/fixture.08a.recovery.malformed_artifact.v1.json +54 -0
- millforge/eval_fixtures/default_pack/manifest.json +12 -0
- millforge/eval_modes.py +1282 -0
- millforge/eval_presets.py +1398 -0
- millforge/eval_reports.py +2517 -0
- millforge/eval_suite.py +2429 -0
- millforge/eval_trials.py +2632 -0
- millforge/eval_workflow.py +794 -0
- millforge/exceptions.py +122 -0
- millforge/model_backend.py +2098 -0
- millforge/protocols.py +340 -0
- millforge/py.typed +0 -0
- millforge/runtime.py +1791 -0
- millforge/testing/__init__.py +1089 -0
- millforge/tools/__init__.py +83 -0
- millforge/tools/builtin_runtime.py +1339 -0
- millforge/tools/builtins.py +773 -0
- millforge/tools/execution.py +1545 -0
- millforge/tools/path_policy.py +155 -0
- millforge/tools/pi_compat/PI_LICENSE +21 -0
- millforge/tools/pi_compat/PROVENANCE.json +55 -0
- millforge/tools/pi_compat/UPDATE_POLICY.md +36 -0
- millforge/tools/pi_compat/__init__.py +34 -0
- millforge/tools/pi_compat/contracts.py +49 -0
- millforge/tools/pi_compat/editing.py +390 -0
- millforge/tools/pi_compat/mutations.py +57 -0
- millforge/tools/pi_compat/operations.py +401 -0
- millforge/tools/pi_compat/paths.py +155 -0
- millforge/tools/pi_compat/process.py +1375 -0
- millforge/tools/pi_compat/search.py +738 -0
- millforge/tools/pi_compat/truncation.py +267 -0
- millforge/tools/pi_compat_catalog.py +396 -0
- millforge/tools/pi_compat_runtime.py +460 -0
- millforge/tools/registry.py +553 -0
- millforge/tools/results.py +533 -0
- millforge-0.1.0.dist-info/METADATA +844 -0
- millforge-0.1.0.dist-info/RECORD +116 -0
- millforge-0.1.0.dist-info/WHEEL +4 -0
- millforge-0.1.0.dist-info/licenses/LICENSE +201 -0
millforge/eval_trials.py
ADDED
|
@@ -0,0 +1,2632 @@
|
|
|
1
|
+
"""Public 08B offline eval-trial contract boundary."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import random
|
|
8
|
+
import re
|
|
9
|
+
from collections.abc import Mapping
|
|
10
|
+
from enum import Enum
|
|
11
|
+
from pathlib import Path, PurePosixPath
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictInt, StrictStr
|
|
15
|
+
from pydantic import field_serializer
|
|
16
|
+
from pydantic import model_validator
|
|
17
|
+
|
|
18
|
+
from millforge.eval_artifacts import (
|
|
19
|
+
EvalArtifactId,
|
|
20
|
+
EvalArtifactManifestArtifact,
|
|
21
|
+
EvalArtifactManifestEntry,
|
|
22
|
+
calculate_eval_artifact_manifest_sha256,
|
|
23
|
+
eval_artifact_layout_entry,
|
|
24
|
+
)
|
|
25
|
+
from millforge.eval_modes import (
|
|
26
|
+
EVAL_SMALL_MILLFORGE_MODE_ID,
|
|
27
|
+
EVAL_SMALL_PI_MODE_ID,
|
|
28
|
+
EvalModeDescriptor,
|
|
29
|
+
default_eval_small_millforge_mode,
|
|
30
|
+
default_eval_small_pi_mode,
|
|
31
|
+
)
|
|
32
|
+
from millforge.eval_presets import (
|
|
33
|
+
EvalPresetReadinessReport,
|
|
34
|
+
eval_preset_readiness_report,
|
|
35
|
+
)
|
|
36
|
+
from millforge.eval_suite import (
|
|
37
|
+
EvalCampaignManifest,
|
|
38
|
+
EvalFixturePackSummary,
|
|
39
|
+
EvalHashRecord,
|
|
40
|
+
EvalPublicArtifactProjection,
|
|
41
|
+
EvalScorerInput,
|
|
42
|
+
EvalScorerResult,
|
|
43
|
+
EvalSuiteExecutionMode,
|
|
44
|
+
EvalCapabilityAuditSummary,
|
|
45
|
+
EvalCheckResult,
|
|
46
|
+
EvalTaskFixture,
|
|
47
|
+
EvalTrialOutcome,
|
|
48
|
+
calculate_eval_scorer_result_hash,
|
|
49
|
+
calculate_eval_scorer_input_hash,
|
|
50
|
+
default_eval_suite_campaign_manifest,
|
|
51
|
+
eval_public_artifact_projection,
|
|
52
|
+
load_eval_fixture_pack_summary,
|
|
53
|
+
load_eval_task_fixture,
|
|
54
|
+
load_eval_task_fixtures,
|
|
55
|
+
score_eval_trial,
|
|
56
|
+
)
|
|
57
|
+
from millforge.eval_workflow import EvalStageId, EvalTerminalResult
|
|
58
|
+
|
|
59
|
+
EVAL_TRIAL_SCHEMA_VERSION = 1
|
|
60
|
+
EVAL_TRIAL_PLAN_HASH_KIND = "eval_trial_plan_sha256_v1"
|
|
61
|
+
EVAL_TRIAL_ARTIFACT_BUNDLE_HASH_KIND = "eval_trial_artifact_bundle_sha256_v1"
|
|
62
|
+
EVAL_TRIAL_RECORD_HASH_KIND = "eval_trial_record_sha256_v1"
|
|
63
|
+
EVAL_TRIAL_STORE_MANIFEST_HASH_KIND = "eval_trial_store_manifest_sha256_v1"
|
|
64
|
+
EVAL_TRIAL_RESUME_INDEX_HASH_KIND = "eval_trial_resume_index_sha256_v1"
|
|
65
|
+
EVAL_TRIAL_DEFAULT_CREATED_AT = "1970-01-01T00:00:00Z"
|
|
66
|
+
EVAL_TRIAL_ADMITTED_ARM_IDS: tuple[str, str] = (
|
|
67
|
+
EVAL_SMALL_PI_MODE_ID,
|
|
68
|
+
EVAL_SMALL_MILLFORGE_MODE_ID,
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
_SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
|
|
72
|
+
_UTC_TIMESTAMP_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$")
|
|
73
|
+
_DENIED_TEXT_TOKENS = (
|
|
74
|
+
"api_key",
|
|
75
|
+
"authorization:",
|
|
76
|
+
"bearer ",
|
|
77
|
+
"credential",
|
|
78
|
+
"password",
|
|
79
|
+
"access token",
|
|
80
|
+
"auth token",
|
|
81
|
+
"millrace-agents",
|
|
82
|
+
".millrace",
|
|
83
|
+
"daemon state",
|
|
84
|
+
"endpoint_url",
|
|
85
|
+
"external service",
|
|
86
|
+
"hidden_checks",
|
|
87
|
+
"hidden scorer",
|
|
88
|
+
"hidden answer",
|
|
89
|
+
"private runtime",
|
|
90
|
+
"network access",
|
|
91
|
+
"package install",
|
|
92
|
+
"scorer_rubric",
|
|
93
|
+
"ref-forge",
|
|
94
|
+
".claude",
|
|
95
|
+
".codex",
|
|
96
|
+
)
|
|
97
|
+
_WINDOWS_ABSOLUTE_PATH = re.compile(
|
|
98
|
+
r"(?:^|[\s\"'`([{<])(?:[A-Za-z]:[\\/]|\\\\)[^\s\"'`)\]}>,]*"
|
|
99
|
+
)
|
|
100
|
+
_POSIX_ABSOLUTE_PATH = re.compile(r"(?:^|[\s\"'`([{<])/(?!/)[^\s\"'`)\]}>,]*")
|
|
101
|
+
_USER_HOME_PATH = re.compile(r"(?:^|[\s\"'`([{<])~(?:/|\\)[^\s\"'`)\]}>,]*")
|
|
102
|
+
_ENDPOINT_URL = re.compile(r"https?://|localhost(?::|/|$)|127\.0\.0\.1|0\.0\.0\.0")
|
|
103
|
+
_CREDENTIAL_VALUE_PATTERNS = (
|
|
104
|
+
re.compile(r"\bsk-(?:live|proj|test)-[A-Za-z0-9_-]{10,}\b"),
|
|
105
|
+
re.compile(r"\b[rs]k_(?:live|test)_[A-Za-z0-9]{16,}\b"),
|
|
106
|
+
re.compile(r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b"),
|
|
107
|
+
re.compile(r"\bAIza[0-9A-Za-z_-]{20,}\b"),
|
|
108
|
+
re.compile(r"\bgh[opsu]_[A-Za-z0-9_]{20,}\b"),
|
|
109
|
+
)
|
|
110
|
+
_SECRET_FIELD_MARKERS = (
|
|
111
|
+
"api_key",
|
|
112
|
+
"apikey",
|
|
113
|
+
"auth_header",
|
|
114
|
+
"authorization",
|
|
115
|
+
"bearer",
|
|
116
|
+
"client_secret",
|
|
117
|
+
"credential",
|
|
118
|
+
"password",
|
|
119
|
+
"private_key",
|
|
120
|
+
"secret",
|
|
121
|
+
"access_token",
|
|
122
|
+
"auth_token",
|
|
123
|
+
"refresh_token",
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
class EvalTrialContractModel(BaseModel):
|
|
128
|
+
"""Closed, frozen base for public eval-trial contracts."""
|
|
129
|
+
|
|
130
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
131
|
+
|
|
132
|
+
@model_validator(mode="before")
|
|
133
|
+
@classmethod
|
|
134
|
+
def _reject_private_material(cls, data: Any) -> Any:
|
|
135
|
+
_reject_forbidden_material(data)
|
|
136
|
+
return data
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
class EvalTrialArmId(str, Enum):
|
|
140
|
+
"""Exactly admitted offline comparison arms for 08B trials."""
|
|
141
|
+
|
|
142
|
+
EVAL_SMALL_PI = EVAL_SMALL_PI_MODE_ID
|
|
143
|
+
EVAL_SMALL_MILLFORGE = EVAL_SMALL_MILLFORGE_MODE_ID
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
class EvalFakeOutcomeScriptKind(str, Enum):
|
|
147
|
+
"""Closed offline fake outcome script kinds."""
|
|
148
|
+
|
|
149
|
+
VALID_COMPLETION = "valid_completion"
|
|
150
|
+
CORRECT_BLOCK = "correct_block"
|
|
151
|
+
FALSE_CLOSURE = "false_closure"
|
|
152
|
+
FALSE_SUCCESS_WITHOUT_CLOSURE = "false_success_without_closure"
|
|
153
|
+
RUNTIME_FAILURE = "runtime_failure"
|
|
154
|
+
PROVIDER_FAILURE = "provider_failure"
|
|
155
|
+
INVALID_TRIAL = "invalid_trial"
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
class EvalTrialInvalidDiagnosticCode(str, Enum):
|
|
159
|
+
"""Closed invalid-trial diagnostic codes emitted by offline contracts."""
|
|
160
|
+
|
|
161
|
+
LIVE_EXECUTION_UNAVAILABLE = "live_execution_unavailable"
|
|
162
|
+
ARM_NOT_ADMITTED = "arm_not_admitted"
|
|
163
|
+
ARM_PARITY_DRIFT = "arm_parity_drift"
|
|
164
|
+
FIXTURE_COPY_UNAVAILABLE = "fixture_copy_unavailable"
|
|
165
|
+
INVALID_PLANNING_REQUEST = "invalid_planning_request"
|
|
166
|
+
PLAN_HASH_MISMATCH = "plan_hash_mismatch"
|
|
167
|
+
ARTIFACT_BUNDLE_HASH_MISMATCH = "artifact_bundle_hash_mismatch"
|
|
168
|
+
RECORD_HASH_MISMATCH = "record_hash_mismatch"
|
|
169
|
+
RESUME_INDEX_HASH_MISMATCH = "resume_index_hash_mismatch"
|
|
170
|
+
STORE_MANIFEST_HASH_MISMATCH = "store_manifest_hash_mismatch"
|
|
171
|
+
TREATMENT_HARNESS_HASH_MISMATCH = "treatment_harness_hash_mismatch"
|
|
172
|
+
BASELINE_RUNTIME_HASH_UNAVAILABLE = "baseline_runtime_hash_unavailable"
|
|
173
|
+
TRIAL_LOG_DUPLICATE_TRIAL_ID = "trial_log_duplicate_trial_id"
|
|
174
|
+
TRIAL_LOG_INVALID_RECORD = "trial_log_invalid_record"
|
|
175
|
+
TRIAL_LOG_MALFORMED_TRAILING_LINE = "trial_log_malformed_trailing_line"
|
|
176
|
+
TRIAL_LOG_MISSING_FINAL_NEWLINE = "trial_log_missing_final_newline"
|
|
177
|
+
TRIAL_LOG_PARTIAL_TRAILING_RECORD = "trial_log_partial_trailing_record"
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
class EvalTrialInvalidDiagnostic(EvalTrialContractModel):
|
|
181
|
+
"""Typed fail-closed diagnostic for invalid or unavailable trials."""
|
|
182
|
+
|
|
183
|
+
diagnostic_code: EvalTrialInvalidDiagnosticCode
|
|
184
|
+
rule_id: StrictStr
|
|
185
|
+
summary: StrictStr
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
class EvalTrialArmDefinition(EvalTrialContractModel):
|
|
189
|
+
"""One admitted comparison arm bound to an existing eval mode descriptor."""
|
|
190
|
+
|
|
191
|
+
arm_id: EvalTrialArmId
|
|
192
|
+
mode_id: StrictStr
|
|
193
|
+
descriptor_fingerprint: StrictStr
|
|
194
|
+
fairness_fingerprint: StrictStr
|
|
195
|
+
runner_kind: StrictStr
|
|
196
|
+
descriptor: EvalModeDescriptor
|
|
197
|
+
|
|
198
|
+
@model_validator(mode="after")
|
|
199
|
+
def _arm_valid(self) -> EvalTrialArmDefinition:
|
|
200
|
+
if self.mode_id != self.arm_id.value:
|
|
201
|
+
raise ValueError("arm mode_id must match arm_id")
|
|
202
|
+
if self.descriptor.mode_id != self.arm_id.value:
|
|
203
|
+
raise ValueError("arm descriptor mode_id must match arm_id")
|
|
204
|
+
if self.descriptor.descriptor_fingerprint != self.descriptor_fingerprint:
|
|
205
|
+
raise ValueError("descriptor_fingerprint must match descriptor")
|
|
206
|
+
if self.descriptor.fairness_fingerprint != self.fairness_fingerprint:
|
|
207
|
+
raise ValueError("fairness_fingerprint must match descriptor")
|
|
208
|
+
runner_kinds = {
|
|
209
|
+
binding.runner_kind.value for binding in self.descriptor.runner_bindings
|
|
210
|
+
}
|
|
211
|
+
if runner_kinds != {self.runner_kind}:
|
|
212
|
+
raise ValueError("runner_kind must match descriptor bindings")
|
|
213
|
+
return self
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
class EvalTrialArmParityEvidence(EvalTrialContractModel):
|
|
217
|
+
"""Hash evidence proving the two offline arms share fairness-critical inputs."""
|
|
218
|
+
|
|
219
|
+
left_arm_id: EvalTrialArmId
|
|
220
|
+
right_arm_id: EvalTrialArmId
|
|
221
|
+
shared_fairness_fingerprint: StrictStr
|
|
222
|
+
left_descriptor_fingerprint: StrictStr
|
|
223
|
+
right_descriptor_fingerprint: StrictStr
|
|
224
|
+
workflow_graph_hash: StrictStr
|
|
225
|
+
model_profile_hash: StrictStr
|
|
226
|
+
fixture_pack_hash: StrictStr
|
|
227
|
+
comparable_offline: StrictBool
|
|
228
|
+
diagnostics: tuple[EvalTrialInvalidDiagnostic, ...] = Field(default_factory=tuple)
|
|
229
|
+
|
|
230
|
+
@model_validator(mode="after")
|
|
231
|
+
def _parity_valid(self) -> EvalTrialArmParityEvidence:
|
|
232
|
+
if self.left_arm_id == self.right_arm_id:
|
|
233
|
+
raise ValueError("parity evidence requires distinct arms")
|
|
234
|
+
for digest in (
|
|
235
|
+
self.shared_fairness_fingerprint,
|
|
236
|
+
self.left_descriptor_fingerprint,
|
|
237
|
+
self.right_descriptor_fingerprint,
|
|
238
|
+
self.workflow_graph_hash,
|
|
239
|
+
self.model_profile_hash,
|
|
240
|
+
self.fixture_pack_hash,
|
|
241
|
+
):
|
|
242
|
+
_validate_sha256(digest)
|
|
243
|
+
if self.comparable_offline and self.diagnostics:
|
|
244
|
+
raise ValueError("comparable offline arms must not carry diagnostics")
|
|
245
|
+
if not self.comparable_offline and not self.diagnostics:
|
|
246
|
+
raise ValueError("non-comparable parity evidence requires diagnostics")
|
|
247
|
+
return self
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
class EvalTrialFixtureInstance(EvalTrialContractModel):
|
|
251
|
+
"""One deterministic fixture instance selected for an offline trial."""
|
|
252
|
+
|
|
253
|
+
fixture_instance_id: StrictStr
|
|
254
|
+
fixture_id: StrictStr
|
|
255
|
+
fixture_hash: StrictStr
|
|
256
|
+
fixture_snapshot_hash: StrictStr
|
|
257
|
+
fixture_pack_id: StrictStr
|
|
258
|
+
fixture_pack_hash: StrictStr
|
|
259
|
+
hidden_check_set_ids: tuple[StrictStr, ...]
|
|
260
|
+
hidden_check_set_hash: StrictStr
|
|
261
|
+
public_projection: EvalPublicArtifactProjection
|
|
262
|
+
fixture_pack_summary: EvalFixturePackSummary
|
|
263
|
+
physical_copy_available: StrictBool = False
|
|
264
|
+
copy_unavailable_diagnostic: EvalTrialInvalidDiagnostic | None = None
|
|
265
|
+
|
|
266
|
+
@model_validator(mode="after")
|
|
267
|
+
def _fixture_instance_valid(self) -> EvalTrialFixtureInstance:
|
|
268
|
+
if self.public_projection.fixture_id != self.fixture_id:
|
|
269
|
+
raise ValueError("fixture_id must match public fixture projection")
|
|
270
|
+
if self.fixture_pack_summary.fixture_pack_id != self.fixture_pack_id:
|
|
271
|
+
raise ValueError("fixture_pack_id must match fixture pack summary")
|
|
272
|
+
if self.fixture_pack_summary.fixture_pack_hash != self.fixture_pack_hash:
|
|
273
|
+
raise ValueError("fixture_pack_hash must match fixture pack summary")
|
|
274
|
+
pack_hashes = dict(
|
|
275
|
+
zip(
|
|
276
|
+
self.fixture_pack_summary.fixture_ids,
|
|
277
|
+
self.fixture_pack_summary.fixture_hashes,
|
|
278
|
+
)
|
|
279
|
+
)
|
|
280
|
+
fixture_hash_record = pack_hashes.get(self.fixture_id)
|
|
281
|
+
if fixture_hash_record is None:
|
|
282
|
+
raise ValueError("fixture_id must be present in fixture pack summary")
|
|
283
|
+
if fixture_hash_record.sha256 != self.fixture_hash:
|
|
284
|
+
raise ValueError("fixture_hash must match fixture pack summary")
|
|
285
|
+
_validate_sha256(self.fixture_hash)
|
|
286
|
+
_validate_sha256(self.fixture_snapshot_hash)
|
|
287
|
+
_validate_sha256(self.fixture_pack_hash)
|
|
288
|
+
_validate_sha256(self.hidden_check_set_hash)
|
|
289
|
+
if not self.hidden_check_set_ids:
|
|
290
|
+
raise ValueError("fixture instances must include hidden-check-set IDs")
|
|
291
|
+
if self.hidden_check_set_hash != _hidden_check_set_hash(
|
|
292
|
+
self.hidden_check_set_ids
|
|
293
|
+
):
|
|
294
|
+
raise ValueError("hidden-check-set hash must match hidden-check-set IDs")
|
|
295
|
+
if (
|
|
296
|
+
not self.physical_copy_available
|
|
297
|
+
and self.copy_unavailable_diagnostic is None
|
|
298
|
+
):
|
|
299
|
+
raise ValueError("deferred fixture copying requires a typed diagnostic")
|
|
300
|
+
if (
|
|
301
|
+
self.physical_copy_available
|
|
302
|
+
and self.copy_unavailable_diagnostic is not None
|
|
303
|
+
):
|
|
304
|
+
raise ValueError(
|
|
305
|
+
"available fixture copies must not carry unavailable diagnostics"
|
|
306
|
+
)
|
|
307
|
+
return self
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
class EvalTrialArmPlan(EvalTrialContractModel):
|
|
311
|
+
"""Per-arm deterministic planning evidence for one paired trial."""
|
|
312
|
+
|
|
313
|
+
trial_id: StrictStr
|
|
314
|
+
trial_index: StrictInt
|
|
315
|
+
arm_id: EvalTrialArmId
|
|
316
|
+
arm_order_index: StrictInt
|
|
317
|
+
paired_seed: StrictInt
|
|
318
|
+
artifact_root: StrictStr
|
|
319
|
+
runner_kind: StrictStr
|
|
320
|
+
runner_descriptor_id: StrictStr
|
|
321
|
+
campaign_manifest_hash: StrictStr
|
|
322
|
+
model_manifest_hash: StrictStr
|
|
323
|
+
workflow_graph_hash: StrictStr
|
|
324
|
+
fixture_pack_hash: StrictStr
|
|
325
|
+
fixture_id: StrictStr
|
|
326
|
+
fixture_hash: StrictStr
|
|
327
|
+
visible_acceptance_criteria_hash: StrictStr
|
|
328
|
+
hidden_check_set_ids: tuple[StrictStr, ...]
|
|
329
|
+
hidden_check_set_hash: StrictStr
|
|
330
|
+
scorer_version: StrictStr
|
|
331
|
+
treatment_compiled_harness_hashes: Mapping[StrictStr, StrictStr] = Field(
|
|
332
|
+
default_factory=dict
|
|
333
|
+
)
|
|
334
|
+
baseline_pi_runtime_hash: StrictStr | None = None
|
|
335
|
+
baseline_runtime_diagnostic: EvalTrialInvalidDiagnostic | None = None
|
|
336
|
+
|
|
337
|
+
@model_validator(mode="after")
|
|
338
|
+
def _arm_plan_valid(self) -> EvalTrialArmPlan:
|
|
339
|
+
if self.trial_index < 0:
|
|
340
|
+
raise ValueError("trial_index must be non-negative")
|
|
341
|
+
if self.arm_order_index not in {0, 1}:
|
|
342
|
+
raise ValueError("arm_order_index must be 0 or 1")
|
|
343
|
+
if self.paired_seed < 0:
|
|
344
|
+
raise ValueError("paired_seed must be non-negative")
|
|
345
|
+
_validate_relative_artifact_root(self.artifact_root)
|
|
346
|
+
for digest in (
|
|
347
|
+
self.campaign_manifest_hash,
|
|
348
|
+
self.model_manifest_hash,
|
|
349
|
+
self.workflow_graph_hash,
|
|
350
|
+
self.fixture_pack_hash,
|
|
351
|
+
self.fixture_hash,
|
|
352
|
+
self.visible_acceptance_criteria_hash,
|
|
353
|
+
self.hidden_check_set_hash,
|
|
354
|
+
):
|
|
355
|
+
_validate_sha256(digest)
|
|
356
|
+
if self.hidden_check_set_hash != _hidden_check_set_hash(
|
|
357
|
+
self.hidden_check_set_ids
|
|
358
|
+
):
|
|
359
|
+
raise ValueError("hidden-check-set hash must match hidden-check-set IDs")
|
|
360
|
+
object.__setattr__(
|
|
361
|
+
self,
|
|
362
|
+
"treatment_compiled_harness_hashes",
|
|
363
|
+
_freeze_trial_mapping(self.treatment_compiled_harness_hashes),
|
|
364
|
+
)
|
|
365
|
+
for digest in self.treatment_compiled_harness_hashes.values():
|
|
366
|
+
_validate_sha256(digest)
|
|
367
|
+
if self.arm_id is EvalTrialArmId.EVAL_SMALL_MILLFORGE:
|
|
368
|
+
if self.baseline_pi_runtime_hash is not None:
|
|
369
|
+
raise ValueError("treatment arms must not carry Pi runtime hashes")
|
|
370
|
+
if self.baseline_runtime_diagnostic is not None:
|
|
371
|
+
raise ValueError("treatment arms must not carry Pi runtime diagnostics")
|
|
372
|
+
if self.arm_id is EvalTrialArmId.EVAL_SMALL_PI:
|
|
373
|
+
if self.treatment_compiled_harness_hashes:
|
|
374
|
+
raise ValueError(
|
|
375
|
+
"baseline arms must not carry treatment harness hashes"
|
|
376
|
+
)
|
|
377
|
+
if self.baseline_pi_runtime_hash is not None:
|
|
378
|
+
_validate_sha256(self.baseline_pi_runtime_hash)
|
|
379
|
+
_reject_fake_pi_runtime_hash(self.baseline_pi_runtime_hash)
|
|
380
|
+
if (
|
|
381
|
+
self.baseline_pi_runtime_hash is None
|
|
382
|
+
and self.baseline_runtime_diagnostic is None
|
|
383
|
+
):
|
|
384
|
+
raise ValueError(
|
|
385
|
+
"baseline arms require a Pi runtime hash or deferred diagnostic"
|
|
386
|
+
)
|
|
387
|
+
return self
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
class EvalTrialFakeRunnerScript(EvalTrialContractModel):
|
|
391
|
+
"""Offline fake runner script descriptor without live execution claims."""
|
|
392
|
+
|
|
393
|
+
script_id: StrictStr
|
|
394
|
+
script_kind: EvalFakeOutcomeScriptKind
|
|
395
|
+
terminal_results: tuple[EvalTerminalResult, ...]
|
|
396
|
+
expected_outcome: EvalTrialOutcome
|
|
397
|
+
stage_result_summaries: Mapping[EvalStageId, StrictStr] = Field(
|
|
398
|
+
default_factory=dict
|
|
399
|
+
)
|
|
400
|
+
zero_usage: StrictBool = True
|
|
401
|
+
|
|
402
|
+
@model_validator(mode="after")
|
|
403
|
+
def _script_valid(self) -> EvalTrialFakeRunnerScript:
|
|
404
|
+
if not self.terminal_results:
|
|
405
|
+
raise ValueError("fake scripts must declare terminal results")
|
|
406
|
+
if not self.zero_usage:
|
|
407
|
+
raise ValueError("offline fake scripts must preserve zero-usage semantics")
|
|
408
|
+
object.__setattr__(
|
|
409
|
+
self,
|
|
410
|
+
"stage_result_summaries",
|
|
411
|
+
_freeze_trial_mapping(self.stage_result_summaries),
|
|
412
|
+
)
|
|
413
|
+
return self
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
class EvalTrialPlan(EvalTrialContractModel):
|
|
417
|
+
"""Hashable paired offline eval-trial plan."""
|
|
418
|
+
|
|
419
|
+
schema_version: StrictInt = EVAL_TRIAL_SCHEMA_VERSION
|
|
420
|
+
trial_id: StrictStr
|
|
421
|
+
trial_index: StrictInt = 0
|
|
422
|
+
trial_plan_id: StrictStr
|
|
423
|
+
paired_seed: StrictInt = 0
|
|
424
|
+
campaign_store_root: StrictStr
|
|
425
|
+
campaign_manifest: EvalCampaignManifest
|
|
426
|
+
arms: tuple[EvalTrialArmDefinition, EvalTrialArmDefinition]
|
|
427
|
+
arm_order: tuple[EvalTrialArmId, EvalTrialArmId]
|
|
428
|
+
arm_plans: tuple[EvalTrialArmPlan, EvalTrialArmPlan]
|
|
429
|
+
parity_evidence: EvalTrialArmParityEvidence
|
|
430
|
+
fixture_instance: EvalTrialFixtureInstance
|
|
431
|
+
fake_runner_script: EvalTrialFakeRunnerScript
|
|
432
|
+
spec_07_readiness: EvalPresetReadinessReport
|
|
433
|
+
created_at: StrictStr = EVAL_TRIAL_DEFAULT_CREATED_AT
|
|
434
|
+
plan_hash_kind: StrictStr = EVAL_TRIAL_PLAN_HASH_KIND
|
|
435
|
+
plan_hash: StrictStr
|
|
436
|
+
|
|
437
|
+
@model_validator(mode="after")
|
|
438
|
+
def _plan_valid(self) -> EvalTrialPlan:
|
|
439
|
+
if self.schema_version != EVAL_TRIAL_SCHEMA_VERSION:
|
|
440
|
+
raise ValueError("unsupported eval-trial schema_version")
|
|
441
|
+
if not _UTC_TIMESTAMP_RE.fullmatch(self.created_at):
|
|
442
|
+
raise ValueError("trial plan created_at must be a UTC timestamp")
|
|
443
|
+
if self.trial_index < 0:
|
|
444
|
+
raise ValueError("trial_index must be non-negative")
|
|
445
|
+
if self.paired_seed < 0:
|
|
446
|
+
raise ValueError("paired_seed must be non-negative")
|
|
447
|
+
_validate_relative_artifact_root(self.campaign_store_root)
|
|
448
|
+
arm_ids = tuple(arm.arm_id for arm in self.arms)
|
|
449
|
+
if arm_ids != (
|
|
450
|
+
EvalTrialArmId.EVAL_SMALL_PI,
|
|
451
|
+
EvalTrialArmId.EVAL_SMALL_MILLFORGE,
|
|
452
|
+
):
|
|
453
|
+
raise ValueError("trial plans must contain exactly the admitted arms")
|
|
454
|
+
if set(self.arm_order) != set(arm_ids) or len(set(self.arm_order)) != 2:
|
|
455
|
+
raise ValueError("arm_order must contain each admitted arm exactly once")
|
|
456
|
+
if {plan.arm_id for plan in self.arm_plans} != set(arm_ids):
|
|
457
|
+
raise ValueError("arm_plans must contain each admitted arm exactly once")
|
|
458
|
+
if self.campaign_manifest.execution_mode != EvalSuiteExecutionMode.OFFLINE_FAKE:
|
|
459
|
+
raise ValueError("eval-trial plans are offline fake contracts only")
|
|
460
|
+
if self.campaign_manifest.live_execution_admitted:
|
|
461
|
+
raise ValueError("eval-trial planning does not admit live execution")
|
|
462
|
+
if self.campaign_manifest.pi_eval_mode_id != EvalTrialArmId.EVAL_SMALL_PI.value:
|
|
463
|
+
raise ValueError("campaign pi arm must match admitted pi mode")
|
|
464
|
+
if (
|
|
465
|
+
self.campaign_manifest.millforge_eval_mode_id
|
|
466
|
+
!= EvalTrialArmId.EVAL_SMALL_MILLFORGE.value
|
|
467
|
+
):
|
|
468
|
+
raise ValueError(
|
|
469
|
+
"campaign millforge arm must match admitted millforge mode"
|
|
470
|
+
)
|
|
471
|
+
left_arm, right_arm = self.arms
|
|
472
|
+
if self.parity_evidence.left_arm_id != left_arm.arm_id:
|
|
473
|
+
raise ValueError("parity left arm must match trial arms")
|
|
474
|
+
if self.parity_evidence.right_arm_id != right_arm.arm_id:
|
|
475
|
+
raise ValueError("parity right arm must match trial arms")
|
|
476
|
+
if (
|
|
477
|
+
self.parity_evidence.left_descriptor_fingerprint
|
|
478
|
+
!= left_arm.descriptor_fingerprint
|
|
479
|
+
):
|
|
480
|
+
raise ValueError("parity left descriptor fingerprint must match arm")
|
|
481
|
+
if (
|
|
482
|
+
self.parity_evidence.right_descriptor_fingerprint
|
|
483
|
+
!= right_arm.descriptor_fingerprint
|
|
484
|
+
):
|
|
485
|
+
raise ValueError("parity right descriptor fingerprint must match arm")
|
|
486
|
+
if (
|
|
487
|
+
self.parity_evidence.shared_fairness_fingerprint
|
|
488
|
+
not in {
|
|
489
|
+
left_arm.fairness_fingerprint,
|
|
490
|
+
right_arm.fairness_fingerprint,
|
|
491
|
+
}
|
|
492
|
+
or left_arm.fairness_fingerprint != right_arm.fairness_fingerprint
|
|
493
|
+
):
|
|
494
|
+
raise ValueError("parity fairness fingerprint must match both arms")
|
|
495
|
+
if self.parity_evidence.workflow_graph_hash not in {
|
|
496
|
+
left_arm.descriptor.graph_sha256,
|
|
497
|
+
right_arm.descriptor.graph_sha256,
|
|
498
|
+
self.campaign_manifest.workflow_graph_hash,
|
|
499
|
+
} or not (
|
|
500
|
+
left_arm.descriptor.graph_sha256
|
|
501
|
+
== right_arm.descriptor.graph_sha256
|
|
502
|
+
== self.campaign_manifest.workflow_graph_hash
|
|
503
|
+
):
|
|
504
|
+
raise ValueError("parity workflow graph hash must match arms and campaign")
|
|
505
|
+
if self.parity_evidence.model_profile_hash not in {
|
|
506
|
+
left_arm.descriptor.model_profile.model_profile_hash,
|
|
507
|
+
right_arm.descriptor.model_profile.model_profile_hash,
|
|
508
|
+
} or (
|
|
509
|
+
left_arm.descriptor.model_profile.model_profile_hash
|
|
510
|
+
!= right_arm.descriptor.model_profile.model_profile_hash
|
|
511
|
+
):
|
|
512
|
+
raise ValueError("parity model profile hash must match both arms")
|
|
513
|
+
if (
|
|
514
|
+
self.parity_evidence.fixture_pack_hash
|
|
515
|
+
!= self.campaign_manifest.fixture_pack_hash
|
|
516
|
+
or self.parity_evidence.fixture_pack_hash
|
|
517
|
+
!= self.fixture_instance.fixture_pack_hash
|
|
518
|
+
):
|
|
519
|
+
raise ValueError(
|
|
520
|
+
"parity fixture pack hash must match campaign and fixture instance"
|
|
521
|
+
)
|
|
522
|
+
expected_treatment_hashes = _compiled_harness_hashes_by_stage(
|
|
523
|
+
self.spec_07_readiness
|
|
524
|
+
)
|
|
525
|
+
shared_arm_values = {
|
|
526
|
+
"trial_id": self.trial_id,
|
|
527
|
+
"trial_index": self.trial_index,
|
|
528
|
+
"paired_seed": self.paired_seed,
|
|
529
|
+
"campaign_manifest_hash": self.campaign_manifest.campaign_manifest_hash,
|
|
530
|
+
"model_manifest_hash": self.campaign_manifest.model_manifest_hash,
|
|
531
|
+
"workflow_graph_hash": self.campaign_manifest.workflow_graph_hash,
|
|
532
|
+
"fixture_pack_hash": self.fixture_instance.fixture_pack_hash,
|
|
533
|
+
"fixture_id": self.fixture_instance.fixture_id,
|
|
534
|
+
"fixture_hash": self.fixture_instance.fixture_hash,
|
|
535
|
+
"visible_acceptance_criteria_hash": _visible_acceptance_criteria_hash(
|
|
536
|
+
self.fixture_instance.public_projection.visible_acceptance_criteria
|
|
537
|
+
),
|
|
538
|
+
"hidden_check_set_hash": self.fixture_instance.hidden_check_set_hash,
|
|
539
|
+
"scorer_version": self.campaign_manifest.scorer_version,
|
|
540
|
+
}
|
|
541
|
+
arms_by_id = {arm.arm_id: arm for arm in self.arms}
|
|
542
|
+
for arm_plan in self.arm_plans:
|
|
543
|
+
_validate_artifact_root_under_campaign(
|
|
544
|
+
arm_plan.artifact_root,
|
|
545
|
+
self.campaign_store_root,
|
|
546
|
+
)
|
|
547
|
+
for field_name, expected_value in shared_arm_values.items():
|
|
548
|
+
if getattr(arm_plan, field_name) != expected_value:
|
|
549
|
+
raise ValueError(
|
|
550
|
+
f"arm plan {field_name} must match paired trial inputs"
|
|
551
|
+
)
|
|
552
|
+
if (
|
|
553
|
+
arm_plan.hidden_check_set_ids
|
|
554
|
+
!= self.fixture_instance.hidden_check_set_ids
|
|
555
|
+
):
|
|
556
|
+
raise ValueError("arm plan hidden-check-set IDs must match fixture")
|
|
557
|
+
descriptor = arms_by_id[arm_plan.arm_id]
|
|
558
|
+
if arm_plan.runner_kind != descriptor.runner_kind:
|
|
559
|
+
raise ValueError("arm plan runner_kind must match descriptor")
|
|
560
|
+
if arm_plan.runner_descriptor_id != descriptor.descriptor_fingerprint:
|
|
561
|
+
raise ValueError("arm plan runner descriptor must match descriptor")
|
|
562
|
+
if arm_plan.arm_order_index != self.arm_order.index(arm_plan.arm_id):
|
|
563
|
+
raise ValueError("arm plan order index must match arm_order")
|
|
564
|
+
if arm_plan.arm_id is EvalTrialArmId.EVAL_SMALL_MILLFORGE and dict(
|
|
565
|
+
arm_plan.treatment_compiled_harness_hashes
|
|
566
|
+
) != dict(expected_treatment_hashes):
|
|
567
|
+
raise ValueError(
|
|
568
|
+
"treatment compiled harness hashes must match Spec 07E readiness"
|
|
569
|
+
)
|
|
570
|
+
pairings = {
|
|
571
|
+
(plan.fixture_id, plan.arm_id.value, plan.trial_index)
|
|
572
|
+
for plan in self.arm_plans
|
|
573
|
+
}
|
|
574
|
+
if len(pairings) != len(self.arm_plans):
|
|
575
|
+
raise ValueError("duplicate fixture/arm/trial-index pairings are invalid")
|
|
576
|
+
if not self.parity_evidence.comparable_offline:
|
|
577
|
+
raise ValueError("trial plans require comparable offline parity evidence")
|
|
578
|
+
if self.plan_hash_kind != EVAL_TRIAL_PLAN_HASH_KIND:
|
|
579
|
+
raise ValueError("unsupported trial plan hash kind")
|
|
580
|
+
_validate_sha256(self.plan_hash)
|
|
581
|
+
expected = calculate_eval_trial_plan_hash(self)
|
|
582
|
+
if self.plan_hash != expected:
|
|
583
|
+
raise ValueError("plan_hash does not match trial plan payload")
|
|
584
|
+
return self
|
|
585
|
+
|
|
586
|
+
|
|
587
|
+
class EvalFakeRunnerArtifactBundle(EvalTrialContractModel):
|
|
588
|
+
"""Offline fake-runner artifact bundle manifest for one arm attempt."""
|
|
589
|
+
|
|
590
|
+
schema_version: StrictInt = EVAL_TRIAL_SCHEMA_VERSION
|
|
591
|
+
artifact_bundle_id: StrictStr
|
|
592
|
+
trial_plan_hash: StrictStr
|
|
593
|
+
arm_id: EvalTrialArmId
|
|
594
|
+
artifact_root: StrictStr
|
|
595
|
+
artifact_manifest: EvalArtifactManifestArtifact
|
|
596
|
+
artifact_hashes: tuple[EvalHashRecord, ...] = Field(default_factory=tuple)
|
|
597
|
+
zero_model_usage: StrictBool = True
|
|
598
|
+
zero_external_usage: StrictBool = True
|
|
599
|
+
artifact_bundle_hash_kind: StrictStr = EVAL_TRIAL_ARTIFACT_BUNDLE_HASH_KIND
|
|
600
|
+
artifact_bundle_hash: StrictStr
|
|
601
|
+
|
|
602
|
+
@model_validator(mode="after")
|
|
603
|
+
def _bundle_valid(self) -> EvalFakeRunnerArtifactBundle:
|
|
604
|
+
if self.schema_version != EVAL_TRIAL_SCHEMA_VERSION:
|
|
605
|
+
raise ValueError("unsupported eval-trial artifact schema_version")
|
|
606
|
+
_validate_sha256(self.trial_plan_hash)
|
|
607
|
+
_validate_relative_artifact_root(self.artifact_root)
|
|
608
|
+
if not self.zero_model_usage or not self.zero_external_usage:
|
|
609
|
+
raise ValueError("offline fake bundles must preserve zero-usage semantics")
|
|
610
|
+
if len({record.sha256 for record in self.artifact_hashes}) != len(
|
|
611
|
+
self.artifact_hashes
|
|
612
|
+
):
|
|
613
|
+
raise ValueError("artifact bundle hashes must be unique")
|
|
614
|
+
if self.artifact_bundle_hash_kind != EVAL_TRIAL_ARTIFACT_BUNDLE_HASH_KIND:
|
|
615
|
+
raise ValueError("unsupported artifact bundle hash kind")
|
|
616
|
+
_validate_sha256(self.artifact_bundle_hash)
|
|
617
|
+
expected = calculate_eval_fake_runner_artifact_bundle_hash(self)
|
|
618
|
+
if self.artifact_bundle_hash != expected:
|
|
619
|
+
raise ValueError("artifact_bundle_hash does not match payload")
|
|
620
|
+
return self
|
|
621
|
+
|
|
622
|
+
|
|
623
|
+
class EvalOfflineFakeTrialRun(EvalTrialContractModel):
|
|
624
|
+
"""Deterministic paired fake-runner output for one planned offline trial."""
|
|
625
|
+
|
|
626
|
+
schema_version: StrictInt = EVAL_TRIAL_SCHEMA_VERSION
|
|
627
|
+
trial_id: StrictStr
|
|
628
|
+
trial_plan_hash: StrictStr
|
|
629
|
+
artifact_bundles: tuple[EvalFakeRunnerArtifactBundle, EvalFakeRunnerArtifactBundle]
|
|
630
|
+
execution_results: tuple[EvalTrialExecutionResult, EvalTrialExecutionResult]
|
|
631
|
+
trial_record: EvalTrialRecord
|
|
632
|
+
live_execution_admitted: StrictBool = False
|
|
633
|
+
diagnostics: tuple[EvalTrialInvalidDiagnostic, ...] = Field(default_factory=tuple)
|
|
634
|
+
|
|
635
|
+
@model_validator(mode="after")
|
|
636
|
+
def _run_valid(self) -> EvalOfflineFakeTrialRun:
|
|
637
|
+
if self.schema_version != EVAL_TRIAL_SCHEMA_VERSION:
|
|
638
|
+
raise ValueError("unsupported eval-trial fake-run schema_version")
|
|
639
|
+
_validate_sha256(self.trial_plan_hash)
|
|
640
|
+
if self.live_execution_admitted:
|
|
641
|
+
raise ValueError("offline fake trial runs do not admit live execution")
|
|
642
|
+
bundle_arm_ids = tuple(bundle.arm_id for bundle in self.artifact_bundles)
|
|
643
|
+
result_arm_ids = tuple(result.arm_id for result in self.execution_results)
|
|
644
|
+
expected_arm_ids = (
|
|
645
|
+
EvalTrialArmId.EVAL_SMALL_PI,
|
|
646
|
+
EvalTrialArmId.EVAL_SMALL_MILLFORGE,
|
|
647
|
+
)
|
|
648
|
+
if bundle_arm_ids != expected_arm_ids or result_arm_ids != expected_arm_ids:
|
|
649
|
+
raise ValueError("fake trial runs must contain exactly the admitted arms")
|
|
650
|
+
if self.trial_record.arm_results != self.execution_results:
|
|
651
|
+
raise ValueError("trial_record must mirror execution_results")
|
|
652
|
+
return self
|
|
653
|
+
|
|
654
|
+
|
|
655
|
+
class EvalTrialExecutionResult(EvalTrialContractModel):
|
|
656
|
+
"""Scorer-facing execution result for one offline arm."""
|
|
657
|
+
|
|
658
|
+
schema_version: StrictInt = EVAL_TRIAL_SCHEMA_VERSION
|
|
659
|
+
trial_id: StrictStr
|
|
660
|
+
trial_plan_hash: StrictStr
|
|
661
|
+
arm_id: EvalTrialArmId
|
|
662
|
+
terminal_results: tuple[EvalTerminalResult, ...]
|
|
663
|
+
artifact_bundle_hash: StrictStr
|
|
664
|
+
scorer_input: EvalScorerInput
|
|
665
|
+
scorer_result: EvalScorerResult
|
|
666
|
+
live_execution_admitted: StrictBool = False
|
|
667
|
+
diagnostics: tuple[EvalTrialInvalidDiagnostic, ...] = Field(default_factory=tuple)
|
|
668
|
+
|
|
669
|
+
@model_validator(mode="after")
|
|
670
|
+
def _execution_result_valid(self) -> EvalTrialExecutionResult:
|
|
671
|
+
if self.schema_version != EVAL_TRIAL_SCHEMA_VERSION:
|
|
672
|
+
raise ValueError("unsupported eval-trial result schema_version")
|
|
673
|
+
_validate_sha256(self.trial_plan_hash)
|
|
674
|
+
_validate_sha256(self.artifact_bundle_hash)
|
|
675
|
+
if self.scorer_input.trial_id != self.trial_id:
|
|
676
|
+
raise ValueError("scorer_input trial_id must match result")
|
|
677
|
+
if self.scorer_result.trial_id != self.trial_id:
|
|
678
|
+
raise ValueError("scorer_result trial_id must match result")
|
|
679
|
+
if self.live_execution_admitted:
|
|
680
|
+
raise ValueError("eval-trial contracts do not admit live execution")
|
|
681
|
+
return self
|
|
682
|
+
|
|
683
|
+
|
|
684
|
+
class EvalTrialRunnerRecordSummary(EvalTrialContractModel):
|
|
685
|
+
"""Public runner identity and runtime diagnostic for one arm record."""
|
|
686
|
+
|
|
687
|
+
arm_id: EvalTrialArmId
|
|
688
|
+
runner_kind: StrictStr
|
|
689
|
+
runner_descriptor_id: StrictStr
|
|
690
|
+
runner_descriptor_version: StrictStr
|
|
691
|
+
baseline_pi_runtime_hash: StrictStr | None = None
|
|
692
|
+
baseline_runtime_diagnostic: EvalTrialInvalidDiagnostic | None = None
|
|
693
|
+
|
|
694
|
+
@model_validator(mode="after")
|
|
695
|
+
def _runner_summary_valid(self) -> EvalTrialRunnerRecordSummary:
|
|
696
|
+
if self.arm_id is EvalTrialArmId.EVAL_SMALL_PI:
|
|
697
|
+
if self.baseline_pi_runtime_hash is not None:
|
|
698
|
+
_validate_sha256(self.baseline_pi_runtime_hash)
|
|
699
|
+
_reject_fake_pi_runtime_hash(self.baseline_pi_runtime_hash)
|
|
700
|
+
if (
|
|
701
|
+
self.baseline_pi_runtime_hash is None
|
|
702
|
+
and self.baseline_runtime_diagnostic is None
|
|
703
|
+
):
|
|
704
|
+
raise ValueError(
|
|
705
|
+
"Pi baseline records require a runtime hash or deferred diagnostic"
|
|
706
|
+
)
|
|
707
|
+
else:
|
|
708
|
+
if self.baseline_pi_runtime_hash is not None:
|
|
709
|
+
raise ValueError("treatment runner summaries must not carry Pi hashes")
|
|
710
|
+
if self.baseline_runtime_diagnostic is not None:
|
|
711
|
+
raise ValueError(
|
|
712
|
+
"treatment runner summaries must not carry Pi diagnostics"
|
|
713
|
+
)
|
|
714
|
+
return self
|
|
715
|
+
|
|
716
|
+
|
|
717
|
+
class EvalTrialScorerPublicSummary(EvalTrialContractModel):
|
|
718
|
+
"""Public scorer result digest without hidden scorer material."""
|
|
719
|
+
|
|
720
|
+
arm_id: EvalTrialArmId
|
|
721
|
+
scorer_version: StrictStr
|
|
722
|
+
final_outcome: EvalTrialOutcome
|
|
723
|
+
primary_success: StrictBool
|
|
724
|
+
false_closure: StrictBool
|
|
725
|
+
false_success: StrictBool
|
|
726
|
+
correctly_blocked: StrictBool
|
|
727
|
+
capability_violation: StrictBool
|
|
728
|
+
artifact_complete: StrictBool
|
|
729
|
+
missing_artifact_ids: tuple[StrictStr, ...] = Field(default_factory=tuple)
|
|
730
|
+
malformed_artifact_ids: tuple[StrictStr, ...] = Field(default_factory=tuple)
|
|
731
|
+
failure_labels: tuple[StrictStr, ...] = Field(default_factory=tuple)
|
|
732
|
+
public_diagnostics: tuple[StrictStr, ...] = Field(default_factory=tuple)
|
|
733
|
+
invalid_trial_explanation: StrictStr | None = None
|
|
734
|
+
|
|
735
|
+
|
|
736
|
+
class EvalTrialResourceSummary(EvalTrialContractModel):
|
|
737
|
+
"""Public resource-use summary for deterministic offline records."""
|
|
738
|
+
|
|
739
|
+
artifact_count: StrictInt
|
|
740
|
+
artifact_bytes: StrictInt = 0
|
|
741
|
+
turn_count: StrictInt = 0
|
|
742
|
+
invalid_tool_call_count: StrictInt = 0
|
|
743
|
+
malformed_argument_count: StrictInt = 0
|
|
744
|
+
prerequisite_violation_count: StrictInt = 0
|
|
745
|
+
premature_terminal_count: StrictInt = 0
|
|
746
|
+
tool_recovery_count: StrictInt = 0
|
|
747
|
+
resource_artifact_hashes: tuple[EvalHashRecord, ...] = Field(default_factory=tuple)
|
|
748
|
+
zero_external_usage: StrictBool = True
|
|
749
|
+
|
|
750
|
+
@model_validator(mode="after")
|
|
751
|
+
def _resource_summary_valid(self) -> EvalTrialResourceSummary:
|
|
752
|
+
for field_name in (
|
|
753
|
+
"artifact_count",
|
|
754
|
+
"artifact_bytes",
|
|
755
|
+
"turn_count",
|
|
756
|
+
"invalid_tool_call_count",
|
|
757
|
+
"malformed_argument_count",
|
|
758
|
+
"prerequisite_violation_count",
|
|
759
|
+
"premature_terminal_count",
|
|
760
|
+
"tool_recovery_count",
|
|
761
|
+
):
|
|
762
|
+
if getattr(self, field_name) < 0:
|
|
763
|
+
raise ValueError(f"{field_name} must be non-negative")
|
|
764
|
+
if not self.zero_external_usage:
|
|
765
|
+
raise ValueError("offline trial records must preserve zero external usage")
|
|
766
|
+
return self
|
|
767
|
+
|
|
768
|
+
|
|
769
|
+
class EvalTrialModelUsageSummary(EvalTrialContractModel):
|
|
770
|
+
"""Public model-use summary for deterministic offline records."""
|
|
771
|
+
|
|
772
|
+
zero_model_usage: StrictBool = True
|
|
773
|
+
input_tokens: StrictInt = 0
|
|
774
|
+
output_tokens: StrictInt = 0
|
|
775
|
+
model_call_count: StrictInt = 0
|
|
776
|
+
|
|
777
|
+
@model_validator(mode="after")
|
|
778
|
+
def _model_usage_summary_valid(self) -> EvalTrialModelUsageSummary:
|
|
779
|
+
if not self.zero_model_usage:
|
|
780
|
+
raise ValueError("offline trial records must preserve zero model usage")
|
|
781
|
+
if (
|
|
782
|
+
self.input_tokens != 0
|
|
783
|
+
or self.output_tokens != 0
|
|
784
|
+
or self.model_call_count != 0
|
|
785
|
+
):
|
|
786
|
+
raise ValueError("offline trial records must report zero model usage")
|
|
787
|
+
return self
|
|
788
|
+
|
|
789
|
+
|
|
790
|
+
class EvalTrialRecord(EvalTrialContractModel):
|
|
791
|
+
"""Append-only paired trial record."""
|
|
792
|
+
|
|
793
|
+
schema_version: StrictInt = EVAL_TRIAL_SCHEMA_VERSION
|
|
794
|
+
campaign_id: StrictStr
|
|
795
|
+
task_fixture_id: StrictStr
|
|
796
|
+
task_category: StrictStr
|
|
797
|
+
trial_id: StrictStr
|
|
798
|
+
trial_index: StrictInt
|
|
799
|
+
arm: EvalTrialArmId
|
|
800
|
+
arm_order_index: StrictInt
|
|
801
|
+
arm_order: tuple[EvalTrialArmId, EvalTrialArmId]
|
|
802
|
+
seed_marker: StrictStr
|
|
803
|
+
trial_plan_hash: StrictStr
|
|
804
|
+
campaign_manifest_hash: StrictStr
|
|
805
|
+
model_manifest_hash: StrictStr
|
|
806
|
+
workflow_graph_hash: StrictStr
|
|
807
|
+
fixture_pack_hash: StrictStr
|
|
808
|
+
fixture_instance_id: StrictStr
|
|
809
|
+
fixture_id: StrictStr
|
|
810
|
+
fixture_hash: StrictStr
|
|
811
|
+
fixture_snapshot_hash: StrictStr
|
|
812
|
+
runner_summaries: Mapping[EvalTrialArmId, EvalTrialRunnerRecordSummary]
|
|
813
|
+
compiled_harness_hashes: Mapping[StrictStr, StrictStr]
|
|
814
|
+
arm_results: tuple[EvalTrialExecutionResult, EvalTrialExecutionResult]
|
|
815
|
+
final_outcomes: Mapping[EvalTrialArmId, EvalTrialOutcome]
|
|
816
|
+
scorer_result_hashes: Mapping[EvalTrialArmId, StrictStr]
|
|
817
|
+
scorer_public_summaries: Mapping[EvalTrialArmId, EvalTrialScorerPublicSummary]
|
|
818
|
+
artifact_manifest_hashes: Mapping[EvalTrialArmId, StrictStr]
|
|
819
|
+
artifact_roots: Mapping[EvalTrialArmId, StrictStr]
|
|
820
|
+
resource_summary: EvalTrialResourceSummary
|
|
821
|
+
model_usage_summary: EvalTrialModelUsageSummary
|
|
822
|
+
invalid_trial_explanations: Mapping[EvalTrialArmId, StrictStr] = Field(
|
|
823
|
+
default_factory=dict
|
|
824
|
+
)
|
|
825
|
+
started_at: StrictStr
|
|
826
|
+
ended_at: StrictStr
|
|
827
|
+
created_at: StrictStr = EVAL_TRIAL_DEFAULT_CREATED_AT
|
|
828
|
+
record_hash_kind: StrictStr = EVAL_TRIAL_RECORD_HASH_KIND
|
|
829
|
+
record_hash: StrictStr
|
|
830
|
+
|
|
831
|
+
@field_serializer("arm_results")
|
|
832
|
+
def _serialize_public_arm_results(
|
|
833
|
+
self,
|
|
834
|
+
value: tuple[EvalTrialExecutionResult, EvalTrialExecutionResult],
|
|
835
|
+
) -> tuple[dict[str, Any], ...]:
|
|
836
|
+
rendered: list[dict[str, Any]] = []
|
|
837
|
+
for result in value:
|
|
838
|
+
payload = result.model_dump(mode="json")
|
|
839
|
+
scorer_input = dict(payload["scorer_input"])
|
|
840
|
+
scorer_input.pop("hidden_check_results", None)
|
|
841
|
+
payload["scorer_input"] = scorer_input
|
|
842
|
+
scorer_result = dict(payload["scorer_result"])
|
|
843
|
+
scorer_result.pop("scorer_only_diagnostics", None)
|
|
844
|
+
payload["scorer_result"] = scorer_result
|
|
845
|
+
rendered.append(payload)
|
|
846
|
+
return tuple(rendered)
|
|
847
|
+
|
|
848
|
+
@model_validator(mode="after")
|
|
849
|
+
def _record_valid(self) -> EvalTrialRecord:
|
|
850
|
+
if self.schema_version != EVAL_TRIAL_SCHEMA_VERSION:
|
|
851
|
+
raise ValueError("unsupported eval-trial record schema_version")
|
|
852
|
+
for field_name in ("created_at", "started_at", "ended_at"):
|
|
853
|
+
if not _UTC_TIMESTAMP_RE.fullmatch(getattr(self, field_name)):
|
|
854
|
+
raise ValueError(f"trial record {field_name} must be a UTC timestamp")
|
|
855
|
+
if self.ended_at < self.started_at:
|
|
856
|
+
raise ValueError("trial record ended_at must not precede started_at")
|
|
857
|
+
if self.task_fixture_id != self.fixture_id:
|
|
858
|
+
raise ValueError("task_fixture_id must match fixture_id")
|
|
859
|
+
if not self.fixture_instance_id:
|
|
860
|
+
raise ValueError("fixture_instance_id must not be empty")
|
|
861
|
+
if self.trial_index < 0:
|
|
862
|
+
raise ValueError("trial_index must be non-negative")
|
|
863
|
+
if self.arm_order_index not in {0, 1}:
|
|
864
|
+
raise ValueError("arm_order_index must be 0 or 1")
|
|
865
|
+
if self.arm_order_index != self.arm_order.index(self.arm):
|
|
866
|
+
raise ValueError("arm_order_index must match record arm")
|
|
867
|
+
for digest in (
|
|
868
|
+
self.trial_plan_hash,
|
|
869
|
+
self.campaign_manifest_hash,
|
|
870
|
+
self.model_manifest_hash,
|
|
871
|
+
self.workflow_graph_hash,
|
|
872
|
+
self.fixture_pack_hash,
|
|
873
|
+
self.fixture_hash,
|
|
874
|
+
self.fixture_snapshot_hash,
|
|
875
|
+
):
|
|
876
|
+
_validate_sha256(digest)
|
|
877
|
+
result_arm_ids = tuple(result.arm_id for result in self.arm_results)
|
|
878
|
+
if result_arm_ids != (
|
|
879
|
+
EvalTrialArmId.EVAL_SMALL_PI,
|
|
880
|
+
EvalTrialArmId.EVAL_SMALL_MILLFORGE,
|
|
881
|
+
):
|
|
882
|
+
raise ValueError("trial records must contain exactly the admitted arms")
|
|
883
|
+
expected_outcomes = {
|
|
884
|
+
result.arm_id: result.scorer_result.final_outcome
|
|
885
|
+
for result in self.arm_results
|
|
886
|
+
}
|
|
887
|
+
if dict(self.final_outcomes) != expected_outcomes:
|
|
888
|
+
raise ValueError("final_outcomes must mirror scorer results")
|
|
889
|
+
expected_hashes = {
|
|
890
|
+
result.arm_id: result.scorer_result.result_hash
|
|
891
|
+
for result in self.arm_results
|
|
892
|
+
}
|
|
893
|
+
if dict(self.scorer_result_hashes) != expected_hashes:
|
|
894
|
+
raise ValueError("scorer_result_hashes must mirror scorer results")
|
|
895
|
+
for result in self.arm_results:
|
|
896
|
+
if result.scorer_result.result_hash != calculate_eval_scorer_result_hash(
|
|
897
|
+
result.scorer_result
|
|
898
|
+
):
|
|
899
|
+
raise ValueError("scorer result hash must match scorer payload")
|
|
900
|
+
expected_invalid = {
|
|
901
|
+
result.arm_id: result.scorer_result.invalid_trial_explanation
|
|
902
|
+
for result in self.arm_results
|
|
903
|
+
if result.scorer_result.invalid_trial_explanation
|
|
904
|
+
}
|
|
905
|
+
if dict(self.invalid_trial_explanations) != expected_invalid:
|
|
906
|
+
raise ValueError(
|
|
907
|
+
"invalid_trial_explanations must mirror public scorer explanations"
|
|
908
|
+
)
|
|
909
|
+
for arm_id, summary in self.scorer_public_summaries.items():
|
|
910
|
+
matching_result = next(
|
|
911
|
+
(
|
|
912
|
+
candidate
|
|
913
|
+
for candidate in self.arm_results
|
|
914
|
+
if candidate.arm_id == arm_id
|
|
915
|
+
),
|
|
916
|
+
None,
|
|
917
|
+
)
|
|
918
|
+
if (
|
|
919
|
+
matching_result is None
|
|
920
|
+
or summary.final_outcome != matching_result.scorer_result.final_outcome
|
|
921
|
+
):
|
|
922
|
+
raise ValueError("scorer public summaries must mirror scorer results")
|
|
923
|
+
expected_treatment_hashes = _compiled_harness_hashes_by_stage(
|
|
924
|
+
eval_preset_readiness_report()
|
|
925
|
+
)
|
|
926
|
+
if dict(self.compiled_harness_hashes) != dict(expected_treatment_hashes):
|
|
927
|
+
raise ValueError(
|
|
928
|
+
"treatment compiled harness hashes must match current Spec 07E readiness"
|
|
929
|
+
)
|
|
930
|
+
if set(self.runner_summaries) != set(result_arm_ids):
|
|
931
|
+
raise ValueError("runner summaries must cover all trial arms")
|
|
932
|
+
if set(self.artifact_manifest_hashes) != set(result_arm_ids):
|
|
933
|
+
raise ValueError("artifact manifest hashes must cover all trial arms")
|
|
934
|
+
if set(self.artifact_roots) != set(result_arm_ids):
|
|
935
|
+
raise ValueError("artifact roots must cover all trial arms")
|
|
936
|
+
for digest in self.artifact_manifest_hashes.values():
|
|
937
|
+
_validate_sha256(digest)
|
|
938
|
+
for path in self.artifact_roots.values():
|
|
939
|
+
_validate_relative_artifact_root(path)
|
|
940
|
+
object.__setattr__(
|
|
941
|
+
self, "final_outcomes", _freeze_trial_mapping(self.final_outcomes)
|
|
942
|
+
)
|
|
943
|
+
for field_name in (
|
|
944
|
+
"runner_summaries",
|
|
945
|
+
"compiled_harness_hashes",
|
|
946
|
+
"scorer_result_hashes",
|
|
947
|
+
"scorer_public_summaries",
|
|
948
|
+
"artifact_manifest_hashes",
|
|
949
|
+
"artifact_roots",
|
|
950
|
+
"invalid_trial_explanations",
|
|
951
|
+
):
|
|
952
|
+
object.__setattr__(
|
|
953
|
+
self, field_name, _freeze_trial_mapping(getattr(self, field_name))
|
|
954
|
+
)
|
|
955
|
+
if self.record_hash_kind != EVAL_TRIAL_RECORD_HASH_KIND:
|
|
956
|
+
raise ValueError("unsupported trial record hash kind")
|
|
957
|
+
_validate_sha256(self.record_hash)
|
|
958
|
+
expected = calculate_eval_trial_record_hash(self)
|
|
959
|
+
if self.record_hash != expected:
|
|
960
|
+
raise ValueError("record_hash does not match trial record payload")
|
|
961
|
+
return self
|
|
962
|
+
|
|
963
|
+
|
|
964
|
+
class EvalTrialStoreManifest(EvalTrialContractModel):
|
|
965
|
+
"""Append-only campaign store manifest."""
|
|
966
|
+
|
|
967
|
+
schema_version: StrictInt = EVAL_TRIAL_SCHEMA_VERSION
|
|
968
|
+
store_manifest_id: StrictStr
|
|
969
|
+
campaign_manifest_hash: StrictStr
|
|
970
|
+
record_hashes: tuple[StrictStr, ...] = Field(default_factory=tuple)
|
|
971
|
+
append_only: StrictBool = True
|
|
972
|
+
store_manifest_hash_kind: StrictStr = EVAL_TRIAL_STORE_MANIFEST_HASH_KIND
|
|
973
|
+
store_manifest_hash: StrictStr
|
|
974
|
+
|
|
975
|
+
@model_validator(mode="after")
|
|
976
|
+
def _store_manifest_valid(self) -> EvalTrialStoreManifest:
|
|
977
|
+
if self.schema_version != EVAL_TRIAL_SCHEMA_VERSION:
|
|
978
|
+
raise ValueError("unsupported eval-trial store schema_version")
|
|
979
|
+
_validate_sha256(self.campaign_manifest_hash)
|
|
980
|
+
for digest in self.record_hashes:
|
|
981
|
+
_validate_sha256(digest)
|
|
982
|
+
if len(set(self.record_hashes)) != len(self.record_hashes):
|
|
983
|
+
raise ValueError("record_hashes must be unique")
|
|
984
|
+
if not self.append_only:
|
|
985
|
+
raise ValueError("eval-trial stores are append-only")
|
|
986
|
+
if self.store_manifest_hash_kind != EVAL_TRIAL_STORE_MANIFEST_HASH_KIND:
|
|
987
|
+
raise ValueError("unsupported store manifest hash kind")
|
|
988
|
+
_validate_sha256(self.store_manifest_hash)
|
|
989
|
+
expected = calculate_eval_trial_store_manifest_hash(self)
|
|
990
|
+
if self.store_manifest_hash != expected:
|
|
991
|
+
raise ValueError("store_manifest_hash does not match payload")
|
|
992
|
+
return self
|
|
993
|
+
|
|
994
|
+
|
|
995
|
+
class EvalTrialResumeIndex(EvalTrialContractModel):
|
|
996
|
+
"""Deterministic resume index for append-only offline campaigns."""
|
|
997
|
+
|
|
998
|
+
schema_version: StrictInt = EVAL_TRIAL_SCHEMA_VERSION
|
|
999
|
+
resume_index_id: StrictStr
|
|
1000
|
+
campaign_manifest_hash: StrictStr
|
|
1001
|
+
completed_trial_record_hashes: tuple[StrictStr, ...] = Field(default_factory=tuple)
|
|
1002
|
+
pending_trial_plan_hashes: tuple[StrictStr, ...] = Field(default_factory=tuple)
|
|
1003
|
+
invalid_trial_diagnostics: tuple[EvalTrialInvalidDiagnostic, ...] = Field(
|
|
1004
|
+
default_factory=tuple
|
|
1005
|
+
)
|
|
1006
|
+
resume_index_hash_kind: StrictStr = EVAL_TRIAL_RESUME_INDEX_HASH_KIND
|
|
1007
|
+
resume_index_hash: StrictStr
|
|
1008
|
+
|
|
1009
|
+
@model_validator(mode="after")
|
|
1010
|
+
def _resume_index_valid(self) -> EvalTrialResumeIndex:
|
|
1011
|
+
if self.schema_version != EVAL_TRIAL_SCHEMA_VERSION:
|
|
1012
|
+
raise ValueError("unsupported eval-trial resume schema_version")
|
|
1013
|
+
_validate_sha256(self.campaign_manifest_hash)
|
|
1014
|
+
for digest in (
|
|
1015
|
+
self.completed_trial_record_hashes + self.pending_trial_plan_hashes
|
|
1016
|
+
):
|
|
1017
|
+
_validate_sha256(digest)
|
|
1018
|
+
if len(set(self.completed_trial_record_hashes)) != len(
|
|
1019
|
+
self.completed_trial_record_hashes
|
|
1020
|
+
):
|
|
1021
|
+
raise ValueError("completed trial record hashes must be unique")
|
|
1022
|
+
if len(set(self.pending_trial_plan_hashes)) != len(
|
|
1023
|
+
self.pending_trial_plan_hashes
|
|
1024
|
+
):
|
|
1025
|
+
raise ValueError("pending trial plan hashes must be unique")
|
|
1026
|
+
if self.resume_index_hash_kind != EVAL_TRIAL_RESUME_INDEX_HASH_KIND:
|
|
1027
|
+
raise ValueError("unsupported resume index hash kind")
|
|
1028
|
+
_validate_sha256(self.resume_index_hash)
|
|
1029
|
+
expected = calculate_eval_trial_resume_index_hash(self)
|
|
1030
|
+
if self.resume_index_hash != expected:
|
|
1031
|
+
raise ValueError("resume_index_hash does not match payload")
|
|
1032
|
+
return self
|
|
1033
|
+
|
|
1034
|
+
|
|
1035
|
+
class EvalTrialCampaignStoreAppendResult(EvalTrialContractModel):
|
|
1036
|
+
"""Result from one append-only campaign-store record write."""
|
|
1037
|
+
|
|
1038
|
+
campaign_store_root: StrictStr
|
|
1039
|
+
manifest: EvalTrialStoreManifest
|
|
1040
|
+
plan: EvalTrialPlan
|
|
1041
|
+
appended_trial_id: StrictStr
|
|
1042
|
+
appended_record_hash: StrictStr
|
|
1043
|
+
completed_trial_ids: tuple[StrictStr, ...]
|
|
1044
|
+
resume_index: EvalTrialResumeIndex
|
|
1045
|
+
|
|
1046
|
+
|
|
1047
|
+
class EvalTrialCampaignStoreRecordSummary(EvalTrialContractModel):
|
|
1048
|
+
"""Public record identity reconstructed from an append-only trial log."""
|
|
1049
|
+
|
|
1050
|
+
trial_id: StrictStr
|
|
1051
|
+
trial_plan_hash: StrictStr
|
|
1052
|
+
record_hash: StrictStr
|
|
1053
|
+
|
|
1054
|
+
@model_validator(mode="after")
|
|
1055
|
+
def _summary_valid(self) -> EvalTrialCampaignStoreRecordSummary:
|
|
1056
|
+
_validate_sha256(self.trial_plan_hash)
|
|
1057
|
+
_validate_sha256(self.record_hash)
|
|
1058
|
+
return self
|
|
1059
|
+
|
|
1060
|
+
|
|
1061
|
+
class EvalTrialCampaignStoreResumeResult(EvalTrialContractModel):
|
|
1062
|
+
"""Non-mutating resume view plus diagnostics for one campaign store."""
|
|
1063
|
+
|
|
1064
|
+
campaign_store_root: StrictStr
|
|
1065
|
+
plan: EvalTrialPlan
|
|
1066
|
+
completed_trial_ids: tuple[StrictStr, ...]
|
|
1067
|
+
pending_trial_ids: tuple[StrictStr, ...]
|
|
1068
|
+
records: tuple[EvalTrialCampaignStoreRecordSummary, ...]
|
|
1069
|
+
resume_index: EvalTrialResumeIndex | None = None
|
|
1070
|
+
diagnostics: tuple[EvalTrialInvalidDiagnostic, ...] = Field(default_factory=tuple)
|
|
1071
|
+
|
|
1072
|
+
|
|
1073
|
+
def default_eval_trial_arm_definitions() -> tuple[
|
|
1074
|
+
EvalTrialArmDefinition, EvalTrialArmDefinition
|
|
1075
|
+
]:
|
|
1076
|
+
"""Return the two admitted offline comparison arm definitions."""
|
|
1077
|
+
return (
|
|
1078
|
+
_arm_definition(default_eval_small_pi_mode(), EvalTrialArmId.EVAL_SMALL_PI),
|
|
1079
|
+
_arm_definition(
|
|
1080
|
+
default_eval_small_millforge_mode(spec_07_static_presets_ready=True),
|
|
1081
|
+
EvalTrialArmId.EVAL_SMALL_MILLFORGE,
|
|
1082
|
+
),
|
|
1083
|
+
)
|
|
1084
|
+
|
|
1085
|
+
|
|
1086
|
+
def default_eval_trial_parity_evidence(
|
|
1087
|
+
arms: tuple[EvalTrialArmDefinition, EvalTrialArmDefinition] | None = None,
|
|
1088
|
+
) -> EvalTrialArmParityEvidence:
|
|
1089
|
+
"""Return deterministic offline parity evidence for the admitted arms."""
|
|
1090
|
+
arms = arms or default_eval_trial_arm_definitions()
|
|
1091
|
+
fixture_pack_hash = load_eval_fixture_pack_summary().fixture_pack_hash
|
|
1092
|
+
left, right = arms
|
|
1093
|
+
comparable = left.fairness_fingerprint == right.fairness_fingerprint
|
|
1094
|
+
return EvalTrialArmParityEvidence(
|
|
1095
|
+
left_arm_id=left.arm_id,
|
|
1096
|
+
right_arm_id=right.arm_id,
|
|
1097
|
+
shared_fairness_fingerprint=left.fairness_fingerprint,
|
|
1098
|
+
left_descriptor_fingerprint=left.descriptor_fingerprint,
|
|
1099
|
+
right_descriptor_fingerprint=right.descriptor_fingerprint,
|
|
1100
|
+
workflow_graph_hash=left.descriptor.graph_sha256,
|
|
1101
|
+
model_profile_hash=left.descriptor.model_profile.model_profile_hash,
|
|
1102
|
+
fixture_pack_hash=fixture_pack_hash,
|
|
1103
|
+
comparable_offline=comparable,
|
|
1104
|
+
diagnostics=()
|
|
1105
|
+
if comparable
|
|
1106
|
+
else (
|
|
1107
|
+
EvalTrialInvalidDiagnostic(
|
|
1108
|
+
diagnostic_code=EvalTrialInvalidDiagnosticCode.ARM_PARITY_DRIFT,
|
|
1109
|
+
rule_id="eval_trials.arm_parity.fairness_fingerprint",
|
|
1110
|
+
summary="admitted arms do not share fairness-critical fingerprints",
|
|
1111
|
+
),
|
|
1112
|
+
),
|
|
1113
|
+
)
|
|
1114
|
+
|
|
1115
|
+
|
|
1116
|
+
def default_eval_trial_fixture_instance(
|
|
1117
|
+
fixture: EvalTaskFixture,
|
|
1118
|
+
*,
|
|
1119
|
+
fixture_instance_id: str | None = None,
|
|
1120
|
+
trial_id: str = "trial.08b.default.v1",
|
|
1121
|
+
trial_index: int = 0,
|
|
1122
|
+
paired_seed: int = 0,
|
|
1123
|
+
) -> EvalTrialFixtureInstance:
|
|
1124
|
+
"""Bind one 08A fixture to a deterministic trial fixture instance."""
|
|
1125
|
+
pack = load_eval_fixture_pack_summary()
|
|
1126
|
+
instance_id = fixture_instance_id or _fixture_instance_id(
|
|
1127
|
+
trial_id=trial_id,
|
|
1128
|
+
trial_index=trial_index,
|
|
1129
|
+
fixture_id=fixture.fixture_id,
|
|
1130
|
+
paired_seed=paired_seed,
|
|
1131
|
+
)
|
|
1132
|
+
hidden_check_ids = tuple(check.check_id for check in fixture.hidden_checks)
|
|
1133
|
+
return EvalTrialFixtureInstance(
|
|
1134
|
+
fixture_instance_id=instance_id,
|
|
1135
|
+
fixture_id=fixture.fixture_id,
|
|
1136
|
+
fixture_hash=fixture.fixture_hash,
|
|
1137
|
+
fixture_snapshot_hash=_fixture_snapshot_hash(
|
|
1138
|
+
fixture=fixture,
|
|
1139
|
+
fixture_instance_id=instance_id,
|
|
1140
|
+
paired_seed=paired_seed,
|
|
1141
|
+
),
|
|
1142
|
+
fixture_pack_id=pack.fixture_pack_id,
|
|
1143
|
+
fixture_pack_hash=pack.fixture_pack_hash,
|
|
1144
|
+
hidden_check_set_ids=hidden_check_ids,
|
|
1145
|
+
hidden_check_set_hash=_hidden_check_set_hash(hidden_check_ids),
|
|
1146
|
+
public_projection=eval_public_artifact_projection(fixture),
|
|
1147
|
+
fixture_pack_summary=pack,
|
|
1148
|
+
physical_copy_available=False,
|
|
1149
|
+
copy_unavailable_diagnostic=fixture_copy_unavailable_diagnostic(),
|
|
1150
|
+
)
|
|
1151
|
+
|
|
1152
|
+
|
|
1153
|
+
def default_eval_trial_plan(
|
|
1154
|
+
*,
|
|
1155
|
+
trial_plan_id: str,
|
|
1156
|
+
fixture: EvalTaskFixture,
|
|
1157
|
+
fake_runner_script: EvalTrialFakeRunnerScript,
|
|
1158
|
+
trial_id: str | None = None,
|
|
1159
|
+
trial_index: int = 0,
|
|
1160
|
+
paired_seed: int = 0,
|
|
1161
|
+
arm_order: tuple[EvalTrialArmId, EvalTrialArmId] | None = None,
|
|
1162
|
+
campaign_store_root: str | None = None,
|
|
1163
|
+
artifact_roots_by_arm: Mapping[EvalTrialArmId | str, str] | None = None,
|
|
1164
|
+
campaign_manifest: EvalCampaignManifest | None = None,
|
|
1165
|
+
created_at: str = EVAL_TRIAL_DEFAULT_CREATED_AT,
|
|
1166
|
+
) -> EvalTrialPlan:
|
|
1167
|
+
"""Build a hash-finalized offline paired trial plan."""
|
|
1168
|
+
campaign_manifest = campaign_manifest or default_eval_suite_campaign_manifest()
|
|
1169
|
+
trial_id = trial_id or trial_plan_id.removesuffix(".plan")
|
|
1170
|
+
campaign_store_root = campaign_store_root or _default_campaign_store_root(
|
|
1171
|
+
campaign_manifest
|
|
1172
|
+
)
|
|
1173
|
+
arm_order = arm_order or (
|
|
1174
|
+
EvalTrialArmId.EVAL_SMALL_PI,
|
|
1175
|
+
EvalTrialArmId.EVAL_SMALL_MILLFORGE,
|
|
1176
|
+
)
|
|
1177
|
+
arms = default_eval_trial_arm_definitions()
|
|
1178
|
+
readiness = eval_preset_readiness_report()
|
|
1179
|
+
fixture_instance = default_eval_trial_fixture_instance(
|
|
1180
|
+
fixture,
|
|
1181
|
+
trial_id=trial_id,
|
|
1182
|
+
trial_index=trial_index,
|
|
1183
|
+
paired_seed=paired_seed,
|
|
1184
|
+
)
|
|
1185
|
+
plan = EvalTrialPlan.model_construct(
|
|
1186
|
+
schema_version=EVAL_TRIAL_SCHEMA_VERSION,
|
|
1187
|
+
trial_id=trial_id,
|
|
1188
|
+
trial_index=trial_index,
|
|
1189
|
+
trial_plan_id=trial_plan_id,
|
|
1190
|
+
paired_seed=paired_seed,
|
|
1191
|
+
campaign_store_root=campaign_store_root,
|
|
1192
|
+
campaign_manifest=campaign_manifest,
|
|
1193
|
+
arms=arms,
|
|
1194
|
+
arm_order=arm_order,
|
|
1195
|
+
arm_plans=_default_eval_trial_arm_plans(
|
|
1196
|
+
trial_id=trial_id,
|
|
1197
|
+
trial_index=trial_index,
|
|
1198
|
+
paired_seed=paired_seed,
|
|
1199
|
+
campaign_store_root=campaign_store_root,
|
|
1200
|
+
campaign_manifest=campaign_manifest,
|
|
1201
|
+
arms=arms,
|
|
1202
|
+
arm_order=arm_order,
|
|
1203
|
+
fixture=fixture,
|
|
1204
|
+
readiness=readiness,
|
|
1205
|
+
artifact_roots_by_arm=artifact_roots_by_arm,
|
|
1206
|
+
),
|
|
1207
|
+
parity_evidence=default_eval_trial_parity_evidence(arms),
|
|
1208
|
+
fixture_instance=fixture_instance,
|
|
1209
|
+
fake_runner_script=fake_runner_script,
|
|
1210
|
+
spec_07_readiness=readiness,
|
|
1211
|
+
created_at=created_at,
|
|
1212
|
+
plan_hash_kind=EVAL_TRIAL_PLAN_HASH_KIND,
|
|
1213
|
+
plan_hash="0" * 64,
|
|
1214
|
+
)
|
|
1215
|
+
return EvalTrialPlan.model_validate(
|
|
1216
|
+
plan.model_copy(update={"plan_hash": calculate_eval_trial_plan_hash(plan)})
|
|
1217
|
+
)
|
|
1218
|
+
|
|
1219
|
+
|
|
1220
|
+
def plan_paired_eval_trials(
|
|
1221
|
+
*,
|
|
1222
|
+
fixtures: tuple[EvalTaskFixture, ...] | None = None,
|
|
1223
|
+
fake_runner_script: EvalTrialFakeRunnerScript,
|
|
1224
|
+
seed: int = 0,
|
|
1225
|
+
campaign_manifest: EvalCampaignManifest | None = None,
|
|
1226
|
+
arms: tuple[EvalTrialArmDefinition, EvalTrialArmDefinition] | None = None,
|
|
1227
|
+
trial_indexes: tuple[int, ...] | None = None,
|
|
1228
|
+
trial_ids: tuple[str, ...] | None = None,
|
|
1229
|
+
campaign_store_root: str | None = None,
|
|
1230
|
+
artifact_roots_by_trial_arm: Mapping[tuple[str, EvalTrialArmId | str], str]
|
|
1231
|
+
| None = None,
|
|
1232
|
+
created_at: str = EVAL_TRIAL_DEFAULT_CREATED_AT,
|
|
1233
|
+
) -> tuple[EvalTrialPlan, ...]:
|
|
1234
|
+
"""Generate deterministic paired Pi-vs-Millforge trial plans."""
|
|
1235
|
+
campaign_manifest = campaign_manifest or default_eval_suite_campaign_manifest()
|
|
1236
|
+
if campaign_manifest.execution_mode != EvalSuiteExecutionMode.OFFLINE_FAKE:
|
|
1237
|
+
raise ValueError("eval-trial planning rejects live execution requests")
|
|
1238
|
+
if campaign_manifest.live_execution_admitted:
|
|
1239
|
+
raise ValueError("eval-trial planning rejects live execution requests")
|
|
1240
|
+
arms = arms or default_eval_trial_arm_definitions()
|
|
1241
|
+
_validate_admitted_planning_arms(arms)
|
|
1242
|
+
fixtures = fixtures or load_eval_task_fixtures()
|
|
1243
|
+
if not fixtures:
|
|
1244
|
+
raise ValueError("paired trial planning requires at least one fixture")
|
|
1245
|
+
if any(not fixture.fixture_hash for fixture in fixtures):
|
|
1246
|
+
raise ValueError("paired trial planning rejects missing fixture hashes")
|
|
1247
|
+
if trial_indexes is None:
|
|
1248
|
+
trial_indexes = tuple(range(len(fixtures)))
|
|
1249
|
+
if len(trial_indexes) != len(fixtures):
|
|
1250
|
+
raise ValueError("trial_indexes must match fixtures")
|
|
1251
|
+
if len(set(trial_indexes)) != len(trial_indexes):
|
|
1252
|
+
raise ValueError("duplicate fixture/arm/trial-index pairings are invalid")
|
|
1253
|
+
campaign_store_root = campaign_store_root or _default_campaign_store_root(
|
|
1254
|
+
campaign_manifest
|
|
1255
|
+
)
|
|
1256
|
+
_validate_relative_artifact_root(campaign_store_root)
|
|
1257
|
+
|
|
1258
|
+
planned_trial_ids = trial_ids or tuple(
|
|
1259
|
+
_trial_id(
|
|
1260
|
+
campaign_id=campaign_manifest.campaign_id,
|
|
1261
|
+
fixture_id=fixture.fixture_id,
|
|
1262
|
+
trial_index=trial_index,
|
|
1263
|
+
seed=seed,
|
|
1264
|
+
)
|
|
1265
|
+
for fixture, trial_index in zip(fixtures, trial_indexes)
|
|
1266
|
+
)
|
|
1267
|
+
if len(planned_trial_ids) != len(fixtures):
|
|
1268
|
+
raise ValueError("trial_ids must match fixtures")
|
|
1269
|
+
if len(set(planned_trial_ids)) != len(planned_trial_ids):
|
|
1270
|
+
raise ValueError("duplicate trial IDs are invalid")
|
|
1271
|
+
|
|
1272
|
+
plans: list[EvalTrialPlan] = []
|
|
1273
|
+
seen_pairings: set[tuple[str, str, int]] = set()
|
|
1274
|
+
for fixture, trial_index, planned_trial_id in zip(
|
|
1275
|
+
fixtures, trial_indexes, planned_trial_ids
|
|
1276
|
+
):
|
|
1277
|
+
paired_seed = _paired_seed(
|
|
1278
|
+
seed=seed,
|
|
1279
|
+
fixture_id=fixture.fixture_id,
|
|
1280
|
+
trial_index=trial_index,
|
|
1281
|
+
)
|
|
1282
|
+
arm_order = _randomized_arm_order(paired_seed)
|
|
1283
|
+
per_trial_roots = {
|
|
1284
|
+
arm_id: root
|
|
1285
|
+
for (root_trial_id, arm_id), root in (
|
|
1286
|
+
artifact_roots_by_trial_arm or {}
|
|
1287
|
+
).items()
|
|
1288
|
+
if root_trial_id == planned_trial_id
|
|
1289
|
+
}
|
|
1290
|
+
for arm_id in arm_order:
|
|
1291
|
+
pairing = (fixture.fixture_id, arm_id.value, trial_index)
|
|
1292
|
+
if pairing in seen_pairings:
|
|
1293
|
+
raise ValueError(
|
|
1294
|
+
"duplicate fixture/arm/trial-index pairings are invalid"
|
|
1295
|
+
)
|
|
1296
|
+
seen_pairings.add(pairing)
|
|
1297
|
+
plans.append(
|
|
1298
|
+
default_eval_trial_plan(
|
|
1299
|
+
trial_plan_id=f"{planned_trial_id}.plan",
|
|
1300
|
+
trial_id=planned_trial_id,
|
|
1301
|
+
trial_index=trial_index,
|
|
1302
|
+
paired_seed=paired_seed,
|
|
1303
|
+
arm_order=arm_order,
|
|
1304
|
+
campaign_store_root=campaign_store_root,
|
|
1305
|
+
artifact_roots_by_arm=per_trial_roots,
|
|
1306
|
+
fixture=fixture,
|
|
1307
|
+
fake_runner_script=fake_runner_script,
|
|
1308
|
+
campaign_manifest=campaign_manifest,
|
|
1309
|
+
created_at=created_at,
|
|
1310
|
+
)
|
|
1311
|
+
)
|
|
1312
|
+
return tuple(plans)
|
|
1313
|
+
|
|
1314
|
+
|
|
1315
|
+
def deny_eval_trial_live_execution(
|
|
1316
|
+
summary: str | None = None,
|
|
1317
|
+
) -> EvalTrialInvalidDiagnostic:
|
|
1318
|
+
"""Return the stable fail-closed diagnostic for live trial execution attempts."""
|
|
1319
|
+
return EvalTrialInvalidDiagnostic(
|
|
1320
|
+
diagnostic_code=EvalTrialInvalidDiagnosticCode.LIVE_EXECUTION_UNAVAILABLE,
|
|
1321
|
+
rule_id="eval_trials.live_execution.unavailable",
|
|
1322
|
+
summary=summary
|
|
1323
|
+
or "08B eval-trial contracts are offline-only and do not admit live execution.",
|
|
1324
|
+
)
|
|
1325
|
+
|
|
1326
|
+
|
|
1327
|
+
def fixture_copy_unavailable_diagnostic(
|
|
1328
|
+
summary: str | None = None,
|
|
1329
|
+
) -> EvalTrialInvalidDiagnostic:
|
|
1330
|
+
"""Return the typed diagnostic for deferred physical fixture copying."""
|
|
1331
|
+
return EvalTrialInvalidDiagnostic(
|
|
1332
|
+
diagnostic_code=EvalTrialInvalidDiagnosticCode.FIXTURE_COPY_UNAVAILABLE,
|
|
1333
|
+
rule_id="eval_trials.fixture_copy.unavailable",
|
|
1334
|
+
summary=summary
|
|
1335
|
+
or (
|
|
1336
|
+
"Physical fixture copying is deferred; planning records a fresh "
|
|
1337
|
+
"fixture instance identity and snapshot hash without admitting live execution."
|
|
1338
|
+
),
|
|
1339
|
+
)
|
|
1340
|
+
|
|
1341
|
+
|
|
1342
|
+
def run_offline_fake_eval_trial(
|
|
1343
|
+
plan: EvalTrialPlan,
|
|
1344
|
+
*,
|
|
1345
|
+
fixture: EvalTaskFixture | None = None,
|
|
1346
|
+
execution_mode: EvalSuiteExecutionMode = EvalSuiteExecutionMode.OFFLINE_FAKE,
|
|
1347
|
+
live_execution_admitted: bool = False,
|
|
1348
|
+
allow_live_model_call: bool = False,
|
|
1349
|
+
allow_pi_execution: bool = False,
|
|
1350
|
+
allow_millforge_harness_execution: bool = False,
|
|
1351
|
+
) -> EvalOfflineFakeTrialRun:
|
|
1352
|
+
"""Execute one planned trial through the deterministic offline fake runner."""
|
|
1353
|
+
diagnostics = _offline_fake_runner_live_denials(
|
|
1354
|
+
execution_mode=execution_mode,
|
|
1355
|
+
live_execution_admitted=live_execution_admitted,
|
|
1356
|
+
allow_live_model_call=allow_live_model_call,
|
|
1357
|
+
allow_pi_execution=allow_pi_execution,
|
|
1358
|
+
allow_millforge_harness_execution=allow_millforge_harness_execution,
|
|
1359
|
+
)
|
|
1360
|
+
if diagnostics:
|
|
1361
|
+
raise ValueError(diagnostics[0].summary)
|
|
1362
|
+
if plan.campaign_manifest.execution_mode != EvalSuiteExecutionMode.OFFLINE_FAKE:
|
|
1363
|
+
raise ValueError("offline fake runner accepts only offline fake trial plans")
|
|
1364
|
+
if plan.campaign_manifest.live_execution_admitted:
|
|
1365
|
+
raise ValueError("offline fake runner rejects live execution admission")
|
|
1366
|
+
|
|
1367
|
+
fixture = fixture or load_eval_task_fixture(plan.fixture_instance.fixture_id)
|
|
1368
|
+
if fixture.fixture_id != plan.fixture_instance.fixture_id:
|
|
1369
|
+
raise ValueError("offline fake runner fixture_id must match the planned trial")
|
|
1370
|
+
if fixture.fixture_hash != plan.fixture_instance.fixture_hash:
|
|
1371
|
+
raise ValueError("offline fake runner fixture_hash must match the trial plan")
|
|
1372
|
+
|
|
1373
|
+
bundles_by_arm: dict[EvalTrialArmId, EvalFakeRunnerArtifactBundle] = {}
|
|
1374
|
+
results_by_arm: dict[EvalTrialArmId, EvalTrialExecutionResult] = {}
|
|
1375
|
+
for arm_plan in plan.arm_plans:
|
|
1376
|
+
bundle = _fake_runner_artifact_bundle(plan=plan, arm_plan=arm_plan)
|
|
1377
|
+
scorer_input = _fake_runner_scorer_input(
|
|
1378
|
+
plan=plan,
|
|
1379
|
+
fixture=fixture,
|
|
1380
|
+
arm_id=arm_plan.arm_id,
|
|
1381
|
+
artifact_hashes=bundle.artifact_hashes,
|
|
1382
|
+
)
|
|
1383
|
+
scorer_result = score_eval_trial(fixture, scorer_input)
|
|
1384
|
+
_validate_fake_script_outcome(plan.fake_runner_script, scorer_result)
|
|
1385
|
+
result = EvalTrialExecutionResult(
|
|
1386
|
+
trial_id=plan.trial_id,
|
|
1387
|
+
trial_plan_hash=plan.plan_hash,
|
|
1388
|
+
arm_id=arm_plan.arm_id,
|
|
1389
|
+
terminal_results=plan.fake_runner_script.terminal_results,
|
|
1390
|
+
artifact_bundle_hash=bundle.artifact_bundle_hash,
|
|
1391
|
+
scorer_input=scorer_input,
|
|
1392
|
+
scorer_result=scorer_result,
|
|
1393
|
+
live_execution_admitted=False,
|
|
1394
|
+
diagnostics=(),
|
|
1395
|
+
)
|
|
1396
|
+
bundles_by_arm[arm_plan.arm_id] = bundle
|
|
1397
|
+
results_by_arm[arm_plan.arm_id] = result
|
|
1398
|
+
|
|
1399
|
+
ordered_bundles = (
|
|
1400
|
+
bundles_by_arm[EvalTrialArmId.EVAL_SMALL_PI],
|
|
1401
|
+
bundles_by_arm[EvalTrialArmId.EVAL_SMALL_MILLFORGE],
|
|
1402
|
+
)
|
|
1403
|
+
ordered_results = (
|
|
1404
|
+
results_by_arm[EvalTrialArmId.EVAL_SMALL_PI],
|
|
1405
|
+
results_by_arm[EvalTrialArmId.EVAL_SMALL_MILLFORGE],
|
|
1406
|
+
)
|
|
1407
|
+
record = EvalTrialRecord.model_construct(
|
|
1408
|
+
schema_version=EVAL_TRIAL_SCHEMA_VERSION,
|
|
1409
|
+
campaign_id=plan.campaign_manifest.campaign_id,
|
|
1410
|
+
task_fixture_id=plan.fixture_instance.fixture_id,
|
|
1411
|
+
task_category=fixture.category.value,
|
|
1412
|
+
trial_id=plan.trial_id,
|
|
1413
|
+
trial_index=plan.trial_index,
|
|
1414
|
+
arm=plan.arm_order[0],
|
|
1415
|
+
arm_order_index=0,
|
|
1416
|
+
arm_order=plan.arm_order,
|
|
1417
|
+
seed_marker=str(plan.paired_seed),
|
|
1418
|
+
trial_plan_hash=plan.plan_hash,
|
|
1419
|
+
campaign_manifest_hash=plan.campaign_manifest.campaign_manifest_hash,
|
|
1420
|
+
model_manifest_hash=plan.campaign_manifest.model_manifest_hash,
|
|
1421
|
+
workflow_graph_hash=plan.campaign_manifest.workflow_graph_hash,
|
|
1422
|
+
fixture_pack_hash=plan.fixture_instance.fixture_pack_hash,
|
|
1423
|
+
fixture_instance_id=plan.fixture_instance.fixture_instance_id,
|
|
1424
|
+
fixture_id=plan.fixture_instance.fixture_id,
|
|
1425
|
+
fixture_hash=plan.fixture_instance.fixture_hash,
|
|
1426
|
+
fixture_snapshot_hash=plan.fixture_instance.fixture_snapshot_hash,
|
|
1427
|
+
runner_summaries=_trial_runner_summaries(plan),
|
|
1428
|
+
compiled_harness_hashes=_compiled_harness_hashes_by_stage(
|
|
1429
|
+
plan.spec_07_readiness
|
|
1430
|
+
),
|
|
1431
|
+
arm_results=ordered_results,
|
|
1432
|
+
final_outcomes={
|
|
1433
|
+
result.arm_id: result.scorer_result.final_outcome
|
|
1434
|
+
for result in ordered_results
|
|
1435
|
+
},
|
|
1436
|
+
scorer_result_hashes={
|
|
1437
|
+
result.arm_id: result.scorer_result.result_hash
|
|
1438
|
+
for result in ordered_results
|
|
1439
|
+
},
|
|
1440
|
+
scorer_public_summaries=_trial_scorer_public_summaries(ordered_results),
|
|
1441
|
+
artifact_manifest_hashes={
|
|
1442
|
+
bundle.arm_id: calculate_eval_artifact_manifest_sha256(
|
|
1443
|
+
bundle.artifact_manifest
|
|
1444
|
+
)
|
|
1445
|
+
for bundle in ordered_bundles
|
|
1446
|
+
},
|
|
1447
|
+
artifact_roots={
|
|
1448
|
+
bundle.arm_id: bundle.artifact_root for bundle in ordered_bundles
|
|
1449
|
+
},
|
|
1450
|
+
resource_summary=_trial_resource_summary(ordered_bundles),
|
|
1451
|
+
model_usage_summary=EvalTrialModelUsageSummary(),
|
|
1452
|
+
invalid_trial_explanations={
|
|
1453
|
+
result.arm_id: result.scorer_result.invalid_trial_explanation
|
|
1454
|
+
for result in ordered_results
|
|
1455
|
+
if result.scorer_result.invalid_trial_explanation
|
|
1456
|
+
},
|
|
1457
|
+
started_at=plan.created_at,
|
|
1458
|
+
ended_at=plan.created_at,
|
|
1459
|
+
created_at=EVAL_TRIAL_DEFAULT_CREATED_AT,
|
|
1460
|
+
record_hash_kind=EVAL_TRIAL_RECORD_HASH_KIND,
|
|
1461
|
+
record_hash="0" * 64,
|
|
1462
|
+
)
|
|
1463
|
+
record = EvalTrialRecord.model_validate(
|
|
1464
|
+
record.model_copy(
|
|
1465
|
+
update={"record_hash": calculate_eval_trial_record_hash(record)}
|
|
1466
|
+
)
|
|
1467
|
+
)
|
|
1468
|
+
return EvalOfflineFakeTrialRun(
|
|
1469
|
+
trial_id=plan.trial_id,
|
|
1470
|
+
trial_plan_hash=plan.plan_hash,
|
|
1471
|
+
artifact_bundles=ordered_bundles,
|
|
1472
|
+
execution_results=ordered_results,
|
|
1473
|
+
trial_record=record,
|
|
1474
|
+
live_execution_admitted=False,
|
|
1475
|
+
diagnostics=(),
|
|
1476
|
+
)
|
|
1477
|
+
|
|
1478
|
+
|
|
1479
|
+
def append_eval_trial_record_to_campaign_store(
|
|
1480
|
+
store_root: str | Path,
|
|
1481
|
+
*,
|
|
1482
|
+
plan: EvalTrialPlan,
|
|
1483
|
+
record: EvalTrialRecord,
|
|
1484
|
+
plans: tuple[EvalTrialPlan, ...] | None = None,
|
|
1485
|
+
) -> EvalTrialCampaignStoreAppendResult:
|
|
1486
|
+
"""Append one complete newline-terminated record to a caller-selected store."""
|
|
1487
|
+
_validate_store_record_matches_plan(plan=plan, record=record)
|
|
1488
|
+
plan_set = _campaign_store_plan_set(plan=plan, plans=plans)
|
|
1489
|
+
campaign_dir = _campaign_store_dir(store_root, plan)
|
|
1490
|
+
manifest_path = campaign_dir / "manifest.json"
|
|
1491
|
+
plan_path = campaign_dir / "plan.json"
|
|
1492
|
+
trials_path = campaign_dir / "trials.jsonl"
|
|
1493
|
+
|
|
1494
|
+
manifest = _campaign_store_manifest(plan=plan, record_hashes=())
|
|
1495
|
+
manifest_bytes = canonical_eval_trial_store_manifest_bytes(manifest)
|
|
1496
|
+
plan_bytes = _canonical_campaign_store_plan_set_bytes(plan_set)
|
|
1497
|
+
|
|
1498
|
+
if manifest_path.exists() and manifest_path.read_bytes() != manifest_bytes:
|
|
1499
|
+
raise ValueError("campaign manifest mismatch; refusing to overwrite")
|
|
1500
|
+
if plan_path.exists() and plan_path.read_bytes() != plan_bytes:
|
|
1501
|
+
raise ValueError("campaign plan mismatch; refusing to overwrite")
|
|
1502
|
+
|
|
1503
|
+
existing_records, diagnostics = _read_campaign_trial_log(trials_path)
|
|
1504
|
+
if diagnostics:
|
|
1505
|
+
raise ValueError(diagnostics[0].summary)
|
|
1506
|
+
if record.trial_id in {existing.trial_id for existing in existing_records}:
|
|
1507
|
+
raise ValueError("duplicate trial IDs are rejected by append-only stores")
|
|
1508
|
+
|
|
1509
|
+
campaign_dir.mkdir(parents=True, exist_ok=True)
|
|
1510
|
+
(campaign_dir / "artifacts").mkdir(exist_ok=True)
|
|
1511
|
+
if not manifest_path.exists():
|
|
1512
|
+
manifest_path.write_bytes(manifest_bytes)
|
|
1513
|
+
if not plan_path.exists():
|
|
1514
|
+
plan_path.write_bytes(plan_bytes)
|
|
1515
|
+
with trials_path.open("ab") as handle:
|
|
1516
|
+
handle.write(canonical_eval_trial_record_bytes(record))
|
|
1517
|
+
|
|
1518
|
+
records = existing_records + (_record_summary(record),)
|
|
1519
|
+
index = _campaign_resume_index(plan=plan, records=records, plans=plan_set)
|
|
1520
|
+
return EvalTrialCampaignStoreAppendResult(
|
|
1521
|
+
campaign_store_root=plan.campaign_store_root,
|
|
1522
|
+
manifest=manifest,
|
|
1523
|
+
plan=plan,
|
|
1524
|
+
appended_trial_id=record.trial_id,
|
|
1525
|
+
appended_record_hash=record.record_hash,
|
|
1526
|
+
completed_trial_ids=tuple(item.trial_id for item in records),
|
|
1527
|
+
resume_index=index,
|
|
1528
|
+
)
|
|
1529
|
+
|
|
1530
|
+
|
|
1531
|
+
def resume_eval_trial_campaign_store(
|
|
1532
|
+
store_root: str | Path,
|
|
1533
|
+
*,
|
|
1534
|
+
plan: EvalTrialPlan,
|
|
1535
|
+
plans: tuple[EvalTrialPlan, ...] | None = None,
|
|
1536
|
+
) -> EvalTrialCampaignStoreResumeResult:
|
|
1537
|
+
"""Reconstruct append-only campaign progress without mutating trial records."""
|
|
1538
|
+
plan_set = _campaign_store_plan_set(plan=plan, plans=plans)
|
|
1539
|
+
campaign_dir = _campaign_store_dir(store_root, plan)
|
|
1540
|
+
manifest_path = campaign_dir / "manifest.json"
|
|
1541
|
+
plan_path = campaign_dir / "plan.json"
|
|
1542
|
+
trials_path = campaign_dir / "trials.jsonl"
|
|
1543
|
+
index_path = campaign_dir / "index.json"
|
|
1544
|
+
|
|
1545
|
+
diagnostics: list[EvalTrialInvalidDiagnostic] = []
|
|
1546
|
+
manifest = _campaign_store_manifest(plan=plan, record_hashes=())
|
|
1547
|
+
if manifest_path.exists() and manifest_path.read_bytes() != (
|
|
1548
|
+
canonical_eval_trial_store_manifest_bytes(manifest)
|
|
1549
|
+
):
|
|
1550
|
+
diagnostics.append(
|
|
1551
|
+
_store_diagnostic(
|
|
1552
|
+
EvalTrialInvalidDiagnosticCode.STORE_MANIFEST_HASH_MISMATCH,
|
|
1553
|
+
"eval_trials.store.manifest_mismatch",
|
|
1554
|
+
"campaign manifest does not match the expected immutable manifest",
|
|
1555
|
+
)
|
|
1556
|
+
)
|
|
1557
|
+
if plan_path.exists() and plan_path.read_bytes() != (
|
|
1558
|
+
_canonical_campaign_store_plan_set_bytes(plan_set)
|
|
1559
|
+
):
|
|
1560
|
+
diagnostics.append(
|
|
1561
|
+
_store_diagnostic(
|
|
1562
|
+
EvalTrialInvalidDiagnosticCode.PLAN_HASH_MISMATCH,
|
|
1563
|
+
"eval_trials.store.plan_mismatch",
|
|
1564
|
+
"campaign plan does not match the expected immutable plan",
|
|
1565
|
+
)
|
|
1566
|
+
)
|
|
1567
|
+
records, log_diagnostics = _read_campaign_trial_log(trials_path)
|
|
1568
|
+
diagnostics.extend(log_diagnostics)
|
|
1569
|
+
duplicate_ids = _duplicate_trial_ids(records)
|
|
1570
|
+
diagnostics.extend(
|
|
1571
|
+
_store_diagnostic(
|
|
1572
|
+
EvalTrialInvalidDiagnosticCode.TRIAL_LOG_DUPLICATE_TRIAL_ID,
|
|
1573
|
+
"eval_trials.store.duplicate_trial_id",
|
|
1574
|
+
f"duplicate trial ID in append-only log: {trial_id}",
|
|
1575
|
+
)
|
|
1576
|
+
for trial_id in duplicate_ids
|
|
1577
|
+
)
|
|
1578
|
+
|
|
1579
|
+
index: EvalTrialResumeIndex | None = None
|
|
1580
|
+
if not diagnostics:
|
|
1581
|
+
campaign_dir.mkdir(parents=True, exist_ok=True)
|
|
1582
|
+
(campaign_dir / "artifacts").mkdir(exist_ok=True)
|
|
1583
|
+
if not manifest_path.exists():
|
|
1584
|
+
manifest_path.write_bytes(
|
|
1585
|
+
canonical_eval_trial_store_manifest_bytes(manifest)
|
|
1586
|
+
)
|
|
1587
|
+
if not plan_path.exists():
|
|
1588
|
+
plan_path.write_bytes(_canonical_campaign_store_plan_set_bytes(plan_set))
|
|
1589
|
+
index = _campaign_resume_index(plan=plan, records=records, plans=plan_set)
|
|
1590
|
+
index_bytes = canonical_eval_trial_resume_index_bytes(index)
|
|
1591
|
+
if index_path.exists():
|
|
1592
|
+
if index_path.read_bytes() != index_bytes:
|
|
1593
|
+
diagnostics.append(
|
|
1594
|
+
_store_diagnostic(
|
|
1595
|
+
EvalTrialInvalidDiagnosticCode.RESUME_INDEX_HASH_MISMATCH,
|
|
1596
|
+
"eval_trials.store.resume_index_mismatch",
|
|
1597
|
+
"campaign resume index does not match reconstructed records",
|
|
1598
|
+
)
|
|
1599
|
+
)
|
|
1600
|
+
index = None
|
|
1601
|
+
else:
|
|
1602
|
+
index_path.write_bytes(index_bytes)
|
|
1603
|
+
|
|
1604
|
+
completed = tuple(record.trial_id for record in records)
|
|
1605
|
+
pending = tuple(
|
|
1606
|
+
item.trial_id for item in plan_set if item.trial_id not in completed
|
|
1607
|
+
)
|
|
1608
|
+
return EvalTrialCampaignStoreResumeResult(
|
|
1609
|
+
campaign_store_root=plan.campaign_store_root,
|
|
1610
|
+
plan=plan,
|
|
1611
|
+
completed_trial_ids=completed,
|
|
1612
|
+
pending_trial_ids=pending,
|
|
1613
|
+
records=records,
|
|
1614
|
+
resume_index=index,
|
|
1615
|
+
diagnostics=tuple(diagnostics),
|
|
1616
|
+
)
|
|
1617
|
+
|
|
1618
|
+
|
|
1619
|
+
def canonical_eval_trial_plan_bytes(plan: EvalTrialPlan) -> bytes:
|
|
1620
|
+
"""Return canonical ASCII JSON bytes for a trial plan."""
|
|
1621
|
+
return _canonical_eval_trial_bytes(plan.model_dump(mode="json"))
|
|
1622
|
+
|
|
1623
|
+
|
|
1624
|
+
def canonical_eval_fake_runner_artifact_bundle_bytes(
|
|
1625
|
+
bundle: EvalFakeRunnerArtifactBundle,
|
|
1626
|
+
) -> bytes:
|
|
1627
|
+
"""Return canonical ASCII JSON bytes for an artifact bundle."""
|
|
1628
|
+
return _canonical_eval_trial_bytes(bundle.model_dump(mode="json"))
|
|
1629
|
+
|
|
1630
|
+
|
|
1631
|
+
def canonical_eval_trial_record_bytes(record: EvalTrialRecord) -> bytes:
|
|
1632
|
+
"""Return canonical ASCII JSON bytes for a trial record."""
|
|
1633
|
+
return _canonical_eval_trial_bytes(record.model_dump(mode="json"))
|
|
1634
|
+
|
|
1635
|
+
|
|
1636
|
+
def canonical_eval_trial_store_manifest_bytes(
|
|
1637
|
+
manifest: EvalTrialStoreManifest,
|
|
1638
|
+
) -> bytes:
|
|
1639
|
+
"""Return canonical ASCII JSON bytes for a store manifest."""
|
|
1640
|
+
return _canonical_eval_trial_bytes(manifest.model_dump(mode="json"))
|
|
1641
|
+
|
|
1642
|
+
|
|
1643
|
+
def canonical_eval_trial_resume_index_bytes(index: EvalTrialResumeIndex) -> bytes:
|
|
1644
|
+
"""Return canonical ASCII JSON bytes for a resume index."""
|
|
1645
|
+
return _canonical_eval_trial_bytes(index.model_dump(mode="json"))
|
|
1646
|
+
|
|
1647
|
+
|
|
1648
|
+
def calculate_eval_trial_plan_hash(plan: EvalTrialPlan) -> str:
|
|
1649
|
+
payload = plan.model_dump(mode="json")
|
|
1650
|
+
payload.pop("plan_hash", None)
|
|
1651
|
+
return hashlib.sha256(_canonical_eval_trial_bytes(payload)).hexdigest()
|
|
1652
|
+
|
|
1653
|
+
|
|
1654
|
+
def calculate_eval_fake_runner_artifact_bundle_hash(
|
|
1655
|
+
bundle: EvalFakeRunnerArtifactBundle,
|
|
1656
|
+
) -> str:
|
|
1657
|
+
payload = bundle.model_dump(mode="json")
|
|
1658
|
+
payload.pop("artifact_bundle_hash", None)
|
|
1659
|
+
return hashlib.sha256(_canonical_eval_trial_bytes(payload)).hexdigest()
|
|
1660
|
+
|
|
1661
|
+
|
|
1662
|
+
def calculate_eval_trial_record_hash(record: EvalTrialRecord) -> str:
|
|
1663
|
+
payload = record.model_dump(mode="json")
|
|
1664
|
+
payload.pop("record_hash", None)
|
|
1665
|
+
return hashlib.sha256(_canonical_eval_trial_bytes(payload)).hexdigest()
|
|
1666
|
+
|
|
1667
|
+
|
|
1668
|
+
def calculate_eval_trial_store_manifest_hash(
|
|
1669
|
+
manifest: EvalTrialStoreManifest,
|
|
1670
|
+
) -> str:
|
|
1671
|
+
payload = manifest.model_dump(mode="json")
|
|
1672
|
+
payload.pop("store_manifest_hash", None)
|
|
1673
|
+
return hashlib.sha256(_canonical_eval_trial_bytes(payload)).hexdigest()
|
|
1674
|
+
|
|
1675
|
+
|
|
1676
|
+
def calculate_eval_trial_resume_index_hash(index: EvalTrialResumeIndex) -> str:
|
|
1677
|
+
payload = index.model_dump(mode="json")
|
|
1678
|
+
payload.pop("resume_index_hash", None)
|
|
1679
|
+
return hashlib.sha256(_canonical_eval_trial_bytes(payload)).hexdigest()
|
|
1680
|
+
|
|
1681
|
+
|
|
1682
|
+
def _campaign_store_plan_set(
|
|
1683
|
+
*,
|
|
1684
|
+
plan: EvalTrialPlan,
|
|
1685
|
+
plans: tuple[EvalTrialPlan, ...] | None,
|
|
1686
|
+
) -> tuple[EvalTrialPlan, ...]:
|
|
1687
|
+
plan_set = plans or (plan,)
|
|
1688
|
+
if not plan_set:
|
|
1689
|
+
raise ValueError("campaign plan set must include at least one trial plan")
|
|
1690
|
+
_validate_campaign_store_root(plan)
|
|
1691
|
+
if not any(
|
|
1692
|
+
item.trial_id == plan.trial_id and item.plan_hash == plan.plan_hash
|
|
1693
|
+
for item in plan_set
|
|
1694
|
+
):
|
|
1695
|
+
raise ValueError("campaign plan set must include the active trial plan")
|
|
1696
|
+
campaign_manifest_hash = plan.campaign_manifest.campaign_manifest_hash
|
|
1697
|
+
campaign_store_root = plan.campaign_store_root
|
|
1698
|
+
trial_ids = tuple(item.trial_id for item in plan_set)
|
|
1699
|
+
plan_hashes = tuple(item.plan_hash for item in plan_set)
|
|
1700
|
+
if len(set(trial_ids)) != len(trial_ids):
|
|
1701
|
+
raise ValueError("campaign plan set rejects duplicate trial IDs")
|
|
1702
|
+
if len(set(plan_hashes)) != len(plan_hashes):
|
|
1703
|
+
raise ValueError("campaign plan set rejects duplicate plan hashes")
|
|
1704
|
+
for item in plan_set:
|
|
1705
|
+
_validate_campaign_store_root(item)
|
|
1706
|
+
if item.campaign_store_root != campaign_store_root:
|
|
1707
|
+
raise ValueError("campaign plan set must share one campaign store root")
|
|
1708
|
+
if item.campaign_manifest.campaign_manifest_hash != campaign_manifest_hash:
|
|
1709
|
+
raise ValueError("campaign plan set must share one campaign manifest")
|
|
1710
|
+
return tuple(
|
|
1711
|
+
sorted(
|
|
1712
|
+
plan_set,
|
|
1713
|
+
key=lambda item: (item.trial_index, item.trial_id, item.plan_hash),
|
|
1714
|
+
)
|
|
1715
|
+
)
|
|
1716
|
+
|
|
1717
|
+
|
|
1718
|
+
def _canonical_campaign_store_plan_set_bytes(
|
|
1719
|
+
plans: tuple[EvalTrialPlan, ...],
|
|
1720
|
+
) -> bytes:
|
|
1721
|
+
first = plans[0]
|
|
1722
|
+
return _canonical_eval_trial_bytes(
|
|
1723
|
+
{
|
|
1724
|
+
"schema_version": EVAL_TRIAL_SCHEMA_VERSION,
|
|
1725
|
+
"campaign_manifest_hash": (first.campaign_manifest.campaign_manifest_hash),
|
|
1726
|
+
"campaign_store_root": first.campaign_store_root,
|
|
1727
|
+
"plan_hashes": tuple(item.plan_hash for item in plans),
|
|
1728
|
+
"plans": tuple(item.model_dump(mode="json") for item in plans),
|
|
1729
|
+
}
|
|
1730
|
+
)
|
|
1731
|
+
|
|
1732
|
+
|
|
1733
|
+
def _campaign_store_dir(store_root: str | Path, plan: EvalTrialPlan) -> Path:
|
|
1734
|
+
_validate_campaign_store_root(plan)
|
|
1735
|
+
return Path(store_root) / PurePosixPath(plan.campaign_store_root)
|
|
1736
|
+
|
|
1737
|
+
|
|
1738
|
+
def _validate_campaign_store_root(plan: EvalTrialPlan) -> None:
|
|
1739
|
+
_validate_relative_artifact_root(plan.campaign_store_root)
|
|
1740
|
+
parts = PurePosixPath(plan.campaign_store_root).parts
|
|
1741
|
+
if parts != ("eval", "campaigns", plan.campaign_manifest.campaign_id):
|
|
1742
|
+
raise ValueError("campaign store root must be eval/campaigns/<campaign-id>")
|
|
1743
|
+
|
|
1744
|
+
|
|
1745
|
+
def _validate_store_record_matches_plan(
|
|
1746
|
+
*,
|
|
1747
|
+
plan: EvalTrialPlan,
|
|
1748
|
+
record: EvalTrialRecord,
|
|
1749
|
+
) -> None:
|
|
1750
|
+
_validate_campaign_store_root(plan)
|
|
1751
|
+
if record.campaign_id != plan.campaign_manifest.campaign_id:
|
|
1752
|
+
raise ValueError("trial record campaign_id must match campaign plan")
|
|
1753
|
+
if record.trial_id != plan.trial_id:
|
|
1754
|
+
raise ValueError("trial record trial_id must match campaign plan")
|
|
1755
|
+
if record.trial_plan_hash != plan.plan_hash:
|
|
1756
|
+
raise ValueError("trial record plan hash must match campaign plan")
|
|
1757
|
+
if record.campaign_manifest_hash != plan.campaign_manifest.campaign_manifest_hash:
|
|
1758
|
+
raise ValueError("trial record campaign manifest hash must match plan")
|
|
1759
|
+
expected_record_values = {
|
|
1760
|
+
"model_manifest_hash": plan.campaign_manifest.model_manifest_hash,
|
|
1761
|
+
"workflow_graph_hash": plan.campaign_manifest.workflow_graph_hash,
|
|
1762
|
+
"fixture_pack_hash": plan.fixture_instance.fixture_pack_hash,
|
|
1763
|
+
"fixture_instance_id": plan.fixture_instance.fixture_instance_id,
|
|
1764
|
+
"fixture_id": plan.fixture_instance.fixture_id,
|
|
1765
|
+
"fixture_hash": plan.fixture_instance.fixture_hash,
|
|
1766
|
+
"fixture_snapshot_hash": plan.fixture_instance.fixture_snapshot_hash,
|
|
1767
|
+
"started_at": plan.created_at,
|
|
1768
|
+
"ended_at": plan.created_at,
|
|
1769
|
+
}
|
|
1770
|
+
for field_name, expected_value in expected_record_values.items():
|
|
1771
|
+
if getattr(record, field_name) != expected_value:
|
|
1772
|
+
raise ValueError(f"trial record {field_name} must match campaign plan")
|
|
1773
|
+
for artifact_root in record.artifact_roots.values():
|
|
1774
|
+
_validate_artifact_root_under_campaign(
|
|
1775
|
+
artifact_root,
|
|
1776
|
+
plan.campaign_store_root,
|
|
1777
|
+
)
|
|
1778
|
+
|
|
1779
|
+
|
|
1780
|
+
def _campaign_store_manifest(
|
|
1781
|
+
*,
|
|
1782
|
+
plan: EvalTrialPlan,
|
|
1783
|
+
record_hashes: tuple[str, ...],
|
|
1784
|
+
) -> EvalTrialStoreManifest:
|
|
1785
|
+
manifest = EvalTrialStoreManifest.model_construct(
|
|
1786
|
+
schema_version=EVAL_TRIAL_SCHEMA_VERSION,
|
|
1787
|
+
store_manifest_id=f"{plan.campaign_manifest.campaign_id}.store.v1",
|
|
1788
|
+
campaign_manifest_hash=plan.campaign_manifest.campaign_manifest_hash,
|
|
1789
|
+
record_hashes=record_hashes,
|
|
1790
|
+
append_only=True,
|
|
1791
|
+
store_manifest_hash_kind=EVAL_TRIAL_STORE_MANIFEST_HASH_KIND,
|
|
1792
|
+
store_manifest_hash="0" * 64,
|
|
1793
|
+
)
|
|
1794
|
+
return EvalTrialStoreManifest.model_validate(
|
|
1795
|
+
manifest.model_copy(
|
|
1796
|
+
update={
|
|
1797
|
+
"store_manifest_hash": calculate_eval_trial_store_manifest_hash(
|
|
1798
|
+
manifest
|
|
1799
|
+
)
|
|
1800
|
+
}
|
|
1801
|
+
)
|
|
1802
|
+
)
|
|
1803
|
+
|
|
1804
|
+
|
|
1805
|
+
def _campaign_resume_index(
|
|
1806
|
+
*,
|
|
1807
|
+
plan: EvalTrialPlan,
|
|
1808
|
+
records: tuple[EvalTrialCampaignStoreRecordSummary, ...],
|
|
1809
|
+
plans: tuple[EvalTrialPlan, ...],
|
|
1810
|
+
) -> EvalTrialResumeIndex:
|
|
1811
|
+
completed_plan_hashes = {record.trial_plan_hash for record in records}
|
|
1812
|
+
index = EvalTrialResumeIndex.model_construct(
|
|
1813
|
+
schema_version=EVAL_TRIAL_SCHEMA_VERSION,
|
|
1814
|
+
resume_index_id=f"{plan.campaign_manifest.campaign_id}.resume.v1",
|
|
1815
|
+
campaign_manifest_hash=plan.campaign_manifest.campaign_manifest_hash,
|
|
1816
|
+
completed_trial_record_hashes=tuple(record.record_hash for record in records),
|
|
1817
|
+
pending_trial_plan_hashes=tuple(
|
|
1818
|
+
item.plan_hash
|
|
1819
|
+
for item in plans
|
|
1820
|
+
if item.plan_hash not in completed_plan_hashes
|
|
1821
|
+
),
|
|
1822
|
+
invalid_trial_diagnostics=(),
|
|
1823
|
+
resume_index_hash_kind=EVAL_TRIAL_RESUME_INDEX_HASH_KIND,
|
|
1824
|
+
resume_index_hash="0" * 64,
|
|
1825
|
+
)
|
|
1826
|
+
return EvalTrialResumeIndex.model_validate(
|
|
1827
|
+
index.model_copy(
|
|
1828
|
+
update={"resume_index_hash": calculate_eval_trial_resume_index_hash(index)}
|
|
1829
|
+
)
|
|
1830
|
+
)
|
|
1831
|
+
|
|
1832
|
+
|
|
1833
|
+
def _read_campaign_trial_log(
|
|
1834
|
+
trials_path: Path,
|
|
1835
|
+
) -> tuple[
|
|
1836
|
+
tuple[EvalTrialCampaignStoreRecordSummary, ...],
|
|
1837
|
+
tuple[EvalTrialInvalidDiagnostic, ...],
|
|
1838
|
+
]:
|
|
1839
|
+
if not trials_path.exists():
|
|
1840
|
+
return (), ()
|
|
1841
|
+
data = trials_path.read_bytes()
|
|
1842
|
+
if data == b"":
|
|
1843
|
+
return (), ()
|
|
1844
|
+
|
|
1845
|
+
records: list[EvalTrialCampaignStoreRecordSummary] = []
|
|
1846
|
+
diagnostics: list[EvalTrialInvalidDiagnostic] = []
|
|
1847
|
+
lines = data.splitlines(keepends=True)
|
|
1848
|
+
for index, line in enumerate(lines, start=1):
|
|
1849
|
+
final_line = index == len(lines)
|
|
1850
|
+
has_newline = line.endswith(b"\n")
|
|
1851
|
+
payload = line[:-1] if has_newline else line
|
|
1852
|
+
if not payload:
|
|
1853
|
+
continue
|
|
1854
|
+
if not has_newline and final_line:
|
|
1855
|
+
diagnostics.append(
|
|
1856
|
+
_store_diagnostic(
|
|
1857
|
+
EvalTrialInvalidDiagnosticCode.TRIAL_LOG_MISSING_FINAL_NEWLINE,
|
|
1858
|
+
"eval_trials.store.missing_final_newline",
|
|
1859
|
+
"trials.jsonl is missing the required final newline",
|
|
1860
|
+
)
|
|
1861
|
+
)
|
|
1862
|
+
try:
|
|
1863
|
+
parsed = json.loads(payload.decode("utf-8"))
|
|
1864
|
+
except (UnicodeDecodeError, json.JSONDecodeError):
|
|
1865
|
+
diagnostics.append(
|
|
1866
|
+
_store_diagnostic(
|
|
1867
|
+
EvalTrialInvalidDiagnosticCode.TRIAL_LOG_PARTIAL_TRAILING_RECORD
|
|
1868
|
+
if final_line and not has_newline
|
|
1869
|
+
else EvalTrialInvalidDiagnosticCode.TRIAL_LOG_MALFORMED_TRAILING_LINE,
|
|
1870
|
+
"eval_trials.store.partial_trailing_record"
|
|
1871
|
+
if final_line and not has_newline
|
|
1872
|
+
else "eval_trials.store.malformed_trailing_line",
|
|
1873
|
+
"trials.jsonl contains a partial trailing record"
|
|
1874
|
+
if final_line and not has_newline
|
|
1875
|
+
else "trials.jsonl contains malformed JSON",
|
|
1876
|
+
)
|
|
1877
|
+
)
|
|
1878
|
+
continue
|
|
1879
|
+
if not has_newline and final_line:
|
|
1880
|
+
continue
|
|
1881
|
+
try:
|
|
1882
|
+
records.append(_public_trial_record_from_json(parsed))
|
|
1883
|
+
except ValueError:
|
|
1884
|
+
diagnostics.append(
|
|
1885
|
+
_store_diagnostic(
|
|
1886
|
+
EvalTrialInvalidDiagnosticCode.TRIAL_LOG_INVALID_RECORD,
|
|
1887
|
+
"eval_trials.store.invalid_record",
|
|
1888
|
+
"trials.jsonl contains an invalid trial record",
|
|
1889
|
+
)
|
|
1890
|
+
)
|
|
1891
|
+
return tuple(records), tuple(diagnostics)
|
|
1892
|
+
|
|
1893
|
+
|
|
1894
|
+
def _public_trial_record_from_json(value: Any) -> EvalTrialCampaignStoreRecordSummary:
|
|
1895
|
+
if not isinstance(value, dict):
|
|
1896
|
+
raise ValueError("trial log record must be a JSON object")
|
|
1897
|
+
for field_name in (
|
|
1898
|
+
"campaign_id",
|
|
1899
|
+
"campaign_manifest_hash",
|
|
1900
|
+
"record_hash",
|
|
1901
|
+
"schema_version",
|
|
1902
|
+
"trial_id",
|
|
1903
|
+
"trial_plan_hash",
|
|
1904
|
+
):
|
|
1905
|
+
if field_name not in value:
|
|
1906
|
+
raise ValueError("trial log record is missing required public fields")
|
|
1907
|
+
if value["schema_version"] != EVAL_TRIAL_SCHEMA_VERSION:
|
|
1908
|
+
raise ValueError("trial log record has unsupported schema version")
|
|
1909
|
+
for field_name in ("campaign_manifest_hash", "record_hash", "trial_plan_hash"):
|
|
1910
|
+
if not isinstance(value[field_name], str):
|
|
1911
|
+
raise ValueError("trial log record hash fields must be strings")
|
|
1912
|
+
_validate_sha256(value[field_name])
|
|
1913
|
+
payload = dict(value)
|
|
1914
|
+
record_hash = payload.pop("record_hash")
|
|
1915
|
+
if hashlib.sha256(_canonical_eval_trial_bytes(payload)).hexdigest() != record_hash:
|
|
1916
|
+
raise ValueError("trial log record hash does not match public payload")
|
|
1917
|
+
if not isinstance(value["trial_id"], str):
|
|
1918
|
+
raise ValueError("trial log trial_id must be a string")
|
|
1919
|
+
return EvalTrialCampaignStoreRecordSummary(
|
|
1920
|
+
trial_id=value["trial_id"],
|
|
1921
|
+
trial_plan_hash=value["trial_plan_hash"],
|
|
1922
|
+
record_hash=record_hash,
|
|
1923
|
+
)
|
|
1924
|
+
|
|
1925
|
+
|
|
1926
|
+
def _record_summary(record: EvalTrialRecord) -> EvalTrialCampaignStoreRecordSummary:
|
|
1927
|
+
return EvalTrialCampaignStoreRecordSummary(
|
|
1928
|
+
trial_id=record.trial_id,
|
|
1929
|
+
trial_plan_hash=record.trial_plan_hash,
|
|
1930
|
+
record_hash=record.record_hash,
|
|
1931
|
+
)
|
|
1932
|
+
|
|
1933
|
+
|
|
1934
|
+
def _duplicate_trial_ids(
|
|
1935
|
+
records: tuple[EvalTrialCampaignStoreRecordSummary, ...],
|
|
1936
|
+
) -> tuple[str, ...]:
|
|
1937
|
+
seen: set[str] = set()
|
|
1938
|
+
duplicates: list[str] = []
|
|
1939
|
+
for record in records:
|
|
1940
|
+
if record.trial_id in seen and record.trial_id not in duplicates:
|
|
1941
|
+
duplicates.append(record.trial_id)
|
|
1942
|
+
seen.add(record.trial_id)
|
|
1943
|
+
return tuple(duplicates)
|
|
1944
|
+
|
|
1945
|
+
|
|
1946
|
+
def _store_diagnostic(
|
|
1947
|
+
code: EvalTrialInvalidDiagnosticCode,
|
|
1948
|
+
rule_id: str,
|
|
1949
|
+
summary: str,
|
|
1950
|
+
) -> EvalTrialInvalidDiagnostic:
|
|
1951
|
+
return EvalTrialInvalidDiagnostic(
|
|
1952
|
+
diagnostic_code=code,
|
|
1953
|
+
rule_id=rule_id,
|
|
1954
|
+
summary=summary,
|
|
1955
|
+
)
|
|
1956
|
+
|
|
1957
|
+
|
|
1958
|
+
def _offline_fake_runner_live_denials(
|
|
1959
|
+
*,
|
|
1960
|
+
execution_mode: EvalSuiteExecutionMode,
|
|
1961
|
+
live_execution_admitted: bool,
|
|
1962
|
+
allow_live_model_call: bool,
|
|
1963
|
+
allow_pi_execution: bool,
|
|
1964
|
+
allow_millforge_harness_execution: bool,
|
|
1965
|
+
) -> tuple[EvalTrialInvalidDiagnostic, ...]:
|
|
1966
|
+
diagnostics: list[EvalTrialInvalidDiagnostic] = []
|
|
1967
|
+
if execution_mode != EvalSuiteExecutionMode.OFFLINE_FAKE:
|
|
1968
|
+
diagnostics.append(
|
|
1969
|
+
deny_eval_trial_live_execution(
|
|
1970
|
+
"Offline fake trial runner rejects live runner execution mode."
|
|
1971
|
+
)
|
|
1972
|
+
)
|
|
1973
|
+
if live_execution_admitted:
|
|
1974
|
+
diagnostics.append(
|
|
1975
|
+
deny_eval_trial_live_execution(
|
|
1976
|
+
"Offline fake trial runner rejects live execution admission."
|
|
1977
|
+
)
|
|
1978
|
+
)
|
|
1979
|
+
if allow_live_model_call:
|
|
1980
|
+
diagnostics.append(
|
|
1981
|
+
deny_eval_trial_live_execution(
|
|
1982
|
+
"Offline fake trial runner rejects model call execution."
|
|
1983
|
+
)
|
|
1984
|
+
)
|
|
1985
|
+
if allow_pi_execution:
|
|
1986
|
+
diagnostics.append(
|
|
1987
|
+
deny_eval_trial_live_execution(
|
|
1988
|
+
"Offline fake trial runner rejects Pi execution."
|
|
1989
|
+
)
|
|
1990
|
+
)
|
|
1991
|
+
if allow_millforge_harness_execution:
|
|
1992
|
+
diagnostics.append(
|
|
1993
|
+
deny_eval_trial_live_execution(
|
|
1994
|
+
"Offline fake trial runner rejects Millforge harness execution."
|
|
1995
|
+
)
|
|
1996
|
+
)
|
|
1997
|
+
return tuple(diagnostics)
|
|
1998
|
+
|
|
1999
|
+
|
|
2000
|
+
def _fake_runner_artifact_bundle(
|
|
2001
|
+
*,
|
|
2002
|
+
plan: EvalTrialPlan,
|
|
2003
|
+
arm_plan: EvalTrialArmPlan,
|
|
2004
|
+
) -> EvalFakeRunnerArtifactBundle:
|
|
2005
|
+
public_artifact_ids = _fake_runner_public_artifact_ids()
|
|
2006
|
+
entries = tuple(
|
|
2007
|
+
_fake_runner_artifact_manifest_entry(
|
|
2008
|
+
trial_id=plan.trial_id,
|
|
2009
|
+
arm_id=arm_plan.arm_id,
|
|
2010
|
+
artifact_id=artifact_id,
|
|
2011
|
+
script_kind=plan.fake_runner_script.script_kind,
|
|
2012
|
+
)
|
|
2013
|
+
for artifact_id in public_artifact_ids
|
|
2014
|
+
)
|
|
2015
|
+
artifact_manifest = EvalArtifactManifestArtifact(
|
|
2016
|
+
trial_id=plan.trial_id,
|
|
2017
|
+
created_by="offline_fake_runner",
|
|
2018
|
+
summary="offline fake runner artifact manifest",
|
|
2019
|
+
entries=entries,
|
|
2020
|
+
)
|
|
2021
|
+
manifest_hash = calculate_eval_artifact_manifest_sha256(artifact_manifest)
|
|
2022
|
+
artifact_hashes = tuple(
|
|
2023
|
+
EvalHashRecord(hash_kind="eval_artifact_sha256_v1", sha256=entry.sha256)
|
|
2024
|
+
for entry in entries
|
|
2025
|
+
) + (
|
|
2026
|
+
EvalHashRecord(
|
|
2027
|
+
hash_kind="eval_artifact_manifest_sha256_v1",
|
|
2028
|
+
sha256=manifest_hash,
|
|
2029
|
+
),
|
|
2030
|
+
)
|
|
2031
|
+
bundle = EvalFakeRunnerArtifactBundle.model_construct(
|
|
2032
|
+
schema_version=EVAL_TRIAL_SCHEMA_VERSION,
|
|
2033
|
+
artifact_bundle_id=f"{plan.trial_id}.{arm_plan.arm_id.value}.bundle.v1",
|
|
2034
|
+
trial_plan_hash=plan.plan_hash,
|
|
2035
|
+
arm_id=arm_plan.arm_id,
|
|
2036
|
+
artifact_root=arm_plan.artifact_root,
|
|
2037
|
+
artifact_manifest=artifact_manifest,
|
|
2038
|
+
artifact_hashes=artifact_hashes,
|
|
2039
|
+
zero_model_usage=True,
|
|
2040
|
+
zero_external_usage=True,
|
|
2041
|
+
artifact_bundle_hash_kind=EVAL_TRIAL_ARTIFACT_BUNDLE_HASH_KIND,
|
|
2042
|
+
artifact_bundle_hash="0" * 64,
|
|
2043
|
+
)
|
|
2044
|
+
return EvalFakeRunnerArtifactBundle.model_validate(
|
|
2045
|
+
bundle.model_copy(
|
|
2046
|
+
update={
|
|
2047
|
+
"artifact_bundle_hash": (
|
|
2048
|
+
calculate_eval_fake_runner_artifact_bundle_hash(bundle)
|
|
2049
|
+
)
|
|
2050
|
+
}
|
|
2051
|
+
)
|
|
2052
|
+
)
|
|
2053
|
+
|
|
2054
|
+
|
|
2055
|
+
def _fake_runner_artifact_manifest_entry(
|
|
2056
|
+
*,
|
|
2057
|
+
trial_id: str,
|
|
2058
|
+
arm_id: EvalTrialArmId,
|
|
2059
|
+
artifact_id: EvalArtifactId,
|
|
2060
|
+
script_kind: EvalFakeOutcomeScriptKind,
|
|
2061
|
+
) -> EvalArtifactManifestEntry:
|
|
2062
|
+
layout = eval_artifact_layout_entry(artifact_id)
|
|
2063
|
+
payload = {
|
|
2064
|
+
"artifact_id": artifact_id.value,
|
|
2065
|
+
"arm_id": arm_id.value,
|
|
2066
|
+
"script_kind": script_kind.value,
|
|
2067
|
+
"trial_id": trial_id,
|
|
2068
|
+
}
|
|
2069
|
+
artifact_bytes = _canonical_eval_trial_bytes(payload)
|
|
2070
|
+
return EvalArtifactManifestEntry(
|
|
2071
|
+
artifact_id=artifact_id,
|
|
2072
|
+
layout_path=layout.layout_path,
|
|
2073
|
+
media_type=layout.media_type,
|
|
2074
|
+
schema_id=layout.schema_id,
|
|
2075
|
+
byte_size=len(artifact_bytes),
|
|
2076
|
+
sha256=hashlib.sha256(artifact_bytes).hexdigest(),
|
|
2077
|
+
producer="offline_fake_runner",
|
|
2078
|
+
model_visible=layout.model_visible,
|
|
2079
|
+
)
|
|
2080
|
+
|
|
2081
|
+
|
|
2082
|
+
def _fake_runner_scorer_input(
|
|
2083
|
+
*,
|
|
2084
|
+
plan: EvalTrialPlan,
|
|
2085
|
+
fixture: EvalTaskFixture,
|
|
2086
|
+
arm_id: EvalTrialArmId,
|
|
2087
|
+
artifact_hashes: tuple[EvalHashRecord, ...],
|
|
2088
|
+
) -> EvalScorerInput:
|
|
2089
|
+
script_kind = plan.fake_runner_script.script_kind
|
|
2090
|
+
artifact_ids = tuple(
|
|
2091
|
+
artifact_id.value for artifact_id in _fake_runner_public_artifact_ids()
|
|
2092
|
+
)
|
|
2093
|
+
terminal_results = tuple(
|
|
2094
|
+
terminal.value for terminal in plan.fake_runner_script.terminal_results
|
|
2095
|
+
)
|
|
2096
|
+
visible_passed = script_kind not in {
|
|
2097
|
+
EvalFakeOutcomeScriptKind.FALSE_CLOSURE,
|
|
2098
|
+
EvalFakeOutcomeScriptKind.FALSE_SUCCESS_WITHOUT_CLOSURE,
|
|
2099
|
+
}
|
|
2100
|
+
hidden_passed = visible_passed
|
|
2101
|
+
scorer_input = EvalScorerInput.model_construct(
|
|
2102
|
+
trial_id=plan.trial_id,
|
|
2103
|
+
fixture_id=fixture.fixture_id,
|
|
2104
|
+
fixture_hash=fixture.fixture_hash
|
|
2105
|
+
if script_kind is not EvalFakeOutcomeScriptKind.INVALID_TRIAL
|
|
2106
|
+
else "f" * 64,
|
|
2107
|
+
final_workspace_hash=_fake_workspace_hash(
|
|
2108
|
+
trial_id=plan.trial_id,
|
|
2109
|
+
arm_id=arm_id,
|
|
2110
|
+
script_kind=script_kind,
|
|
2111
|
+
),
|
|
2112
|
+
path_limited_workspace_hashes=(),
|
|
2113
|
+
public_artifact_hashes=artifact_hashes,
|
|
2114
|
+
required_public_artifact_ids=artifact_ids,
|
|
2115
|
+
provided_public_artifact_ids=artifact_ids,
|
|
2116
|
+
malformed_artifact_ids=(),
|
|
2117
|
+
stage_terminal_results=terminal_results,
|
|
2118
|
+
capability_audit=EvalCapabilityAuditSummary(
|
|
2119
|
+
capability_violation=False,
|
|
2120
|
+
denied_capability_ids=(),
|
|
2121
|
+
summary="offline fake runner used no denied capabilities",
|
|
2122
|
+
),
|
|
2123
|
+
visible_check_results=tuple(
|
|
2124
|
+
EvalCheckResult(check_id=check.check_id, passed=visible_passed)
|
|
2125
|
+
for check in fixture.visible_checks
|
|
2126
|
+
),
|
|
2127
|
+
hidden_check_results=tuple(
|
|
2128
|
+
EvalCheckResult(check_id=check.check_id, passed=hidden_passed)
|
|
2129
|
+
for check in fixture.hidden_checks
|
|
2130
|
+
),
|
|
2131
|
+
claimed_mutation_present=True,
|
|
2132
|
+
unauthorized_mutation=False,
|
|
2133
|
+
checker_public_evidence_valid=True,
|
|
2134
|
+
runtime_failure=script_kind is EvalFakeOutcomeScriptKind.RUNTIME_FAILURE,
|
|
2135
|
+
provider_failure=script_kind is EvalFakeOutcomeScriptKind.PROVIDER_FAILURE,
|
|
2136
|
+
invalid_trial_explanation=(
|
|
2137
|
+
"offline fake runner scripted invalid trial"
|
|
2138
|
+
if script_kind is EvalFakeOutcomeScriptKind.INVALID_TRIAL
|
|
2139
|
+
else None
|
|
2140
|
+
),
|
|
2141
|
+
scorer_input_hash_kind="eval_suite_scorer_input_sha256_v1",
|
|
2142
|
+
scorer_input_hash="0" * 64,
|
|
2143
|
+
)
|
|
2144
|
+
return EvalScorerInput.model_validate(
|
|
2145
|
+
scorer_input.model_copy(
|
|
2146
|
+
update={"scorer_input_hash": calculate_eval_scorer_input_hash(scorer_input)}
|
|
2147
|
+
)
|
|
2148
|
+
)
|
|
2149
|
+
|
|
2150
|
+
|
|
2151
|
+
def _validate_fake_script_outcome(
|
|
2152
|
+
script: EvalTrialFakeRunnerScript,
|
|
2153
|
+
scorer_result: EvalScorerResult,
|
|
2154
|
+
) -> None:
|
|
2155
|
+
if scorer_result.final_outcome != script.expected_outcome:
|
|
2156
|
+
raise ValueError(
|
|
2157
|
+
"offline fake script expected_outcome does not match scorer result"
|
|
2158
|
+
)
|
|
2159
|
+
if (
|
|
2160
|
+
script.script_kind is EvalFakeOutcomeScriptKind.FALSE_SUCCESS_WITHOUT_CLOSURE
|
|
2161
|
+
and not scorer_result.false_success
|
|
2162
|
+
):
|
|
2163
|
+
raise ValueError("offline fake script did not produce false success")
|
|
2164
|
+
|
|
2165
|
+
|
|
2166
|
+
def _trial_runner_summaries(
|
|
2167
|
+
plan: EvalTrialPlan,
|
|
2168
|
+
) -> Mapping[EvalTrialArmId, EvalTrialRunnerRecordSummary]:
|
|
2169
|
+
return _freeze_trial_mapping(
|
|
2170
|
+
{
|
|
2171
|
+
arm_plan.arm_id: EvalTrialRunnerRecordSummary(
|
|
2172
|
+
arm_id=arm_plan.arm_id,
|
|
2173
|
+
runner_kind=arm_plan.runner_kind,
|
|
2174
|
+
runner_descriptor_id=arm_plan.runner_descriptor_id,
|
|
2175
|
+
runner_descriptor_version="eval-trial.runner-descriptor.v1",
|
|
2176
|
+
baseline_pi_runtime_hash=arm_plan.baseline_pi_runtime_hash,
|
|
2177
|
+
baseline_runtime_diagnostic=arm_plan.baseline_runtime_diagnostic,
|
|
2178
|
+
)
|
|
2179
|
+
for arm_plan in plan.arm_plans
|
|
2180
|
+
}
|
|
2181
|
+
)
|
|
2182
|
+
|
|
2183
|
+
|
|
2184
|
+
def _trial_scorer_public_summaries(
|
|
2185
|
+
results: tuple[EvalTrialExecutionResult, EvalTrialExecutionResult],
|
|
2186
|
+
) -> Mapping[EvalTrialArmId, EvalTrialScorerPublicSummary]:
|
|
2187
|
+
return _freeze_trial_mapping(
|
|
2188
|
+
{
|
|
2189
|
+
result.arm_id: EvalTrialScorerPublicSummary(
|
|
2190
|
+
arm_id=result.arm_id,
|
|
2191
|
+
scorer_version=result.scorer_result.scorer_version,
|
|
2192
|
+
final_outcome=result.scorer_result.final_outcome,
|
|
2193
|
+
primary_success=result.scorer_result.primary_success,
|
|
2194
|
+
false_closure=result.scorer_result.false_closure,
|
|
2195
|
+
false_success=result.scorer_result.false_success,
|
|
2196
|
+
correctly_blocked=result.scorer_result.correctly_blocked,
|
|
2197
|
+
capability_violation=result.scorer_result.capability_violation,
|
|
2198
|
+
artifact_complete=result.scorer_result.artifact_complete,
|
|
2199
|
+
missing_artifact_ids=result.scorer_result.missing_artifact_ids,
|
|
2200
|
+
malformed_artifact_ids=result.scorer_result.malformed_artifact_ids,
|
|
2201
|
+
failure_labels=tuple(
|
|
2202
|
+
label.value for label in result.scorer_result.failure_labels
|
|
2203
|
+
),
|
|
2204
|
+
public_diagnostics=result.scorer_result.public_diagnostics,
|
|
2205
|
+
invalid_trial_explanation=(
|
|
2206
|
+
result.scorer_result.invalid_trial_explanation
|
|
2207
|
+
),
|
|
2208
|
+
)
|
|
2209
|
+
for result in results
|
|
2210
|
+
}
|
|
2211
|
+
)
|
|
2212
|
+
|
|
2213
|
+
|
|
2214
|
+
def _trial_resource_summary(
|
|
2215
|
+
bundles: tuple[EvalFakeRunnerArtifactBundle, EvalFakeRunnerArtifactBundle],
|
|
2216
|
+
) -> EvalTrialResourceSummary:
|
|
2217
|
+
hashes = tuple(record for bundle in bundles for record in bundle.artifact_hashes)
|
|
2218
|
+
return EvalTrialResourceSummary(
|
|
2219
|
+
artifact_count=sum(len(bundle.artifact_hashes) for bundle in bundles),
|
|
2220
|
+
artifact_bytes=sum(
|
|
2221
|
+
entry.byte_size
|
|
2222
|
+
for bundle in bundles
|
|
2223
|
+
for entry in bundle.artifact_manifest.entries
|
|
2224
|
+
),
|
|
2225
|
+
resource_artifact_hashes=hashes,
|
|
2226
|
+
zero_external_usage=all(bundle.zero_external_usage for bundle in bundles),
|
|
2227
|
+
)
|
|
2228
|
+
|
|
2229
|
+
|
|
2230
|
+
def _fake_runner_public_artifact_ids() -> tuple[EvalArtifactId, ...]:
|
|
2231
|
+
return (
|
|
2232
|
+
EvalArtifactId.TASK,
|
|
2233
|
+
EvalArtifactId.ACCEPTANCE_CHECKS,
|
|
2234
|
+
EvalArtifactId.PLAN,
|
|
2235
|
+
EvalArtifactId.WORKSPACE_DIFF,
|
|
2236
|
+
EvalArtifactId.PATCH_SUMMARY,
|
|
2237
|
+
EvalArtifactId.TEST_RESULTS,
|
|
2238
|
+
EvalArtifactId.CHECKER_VERDICT,
|
|
2239
|
+
EvalArtifactId.ARBITER_VERDICT,
|
|
2240
|
+
EvalArtifactId.STAGE_RESULT,
|
|
2241
|
+
EvalArtifactId.EVENT_LOG,
|
|
2242
|
+
EvalArtifactId.RESOURCE_USAGE,
|
|
2243
|
+
EvalArtifactId.MODEL_USAGE,
|
|
2244
|
+
EvalArtifactId.VALIDATOR_RESULT,
|
|
2245
|
+
)
|
|
2246
|
+
|
|
2247
|
+
|
|
2248
|
+
def _fake_workspace_hash(
|
|
2249
|
+
*,
|
|
2250
|
+
trial_id: str,
|
|
2251
|
+
arm_id: EvalTrialArmId,
|
|
2252
|
+
script_kind: EvalFakeOutcomeScriptKind,
|
|
2253
|
+
) -> str:
|
|
2254
|
+
return hashlib.sha256(
|
|
2255
|
+
_canonical_eval_trial_bytes(
|
|
2256
|
+
{
|
|
2257
|
+
"arm_id": arm_id.value,
|
|
2258
|
+
"script_kind": script_kind.value,
|
|
2259
|
+
"trial_id": trial_id,
|
|
2260
|
+
}
|
|
2261
|
+
)
|
|
2262
|
+
).hexdigest()
|
|
2263
|
+
|
|
2264
|
+
|
|
2265
|
+
def _arm_definition(
|
|
2266
|
+
descriptor: EvalModeDescriptor,
|
|
2267
|
+
arm_id: EvalTrialArmId,
|
|
2268
|
+
) -> EvalTrialArmDefinition:
|
|
2269
|
+
return EvalTrialArmDefinition(
|
|
2270
|
+
arm_id=arm_id,
|
|
2271
|
+
mode_id=descriptor.mode_id,
|
|
2272
|
+
descriptor_fingerprint=descriptor.descriptor_fingerprint,
|
|
2273
|
+
fairness_fingerprint=descriptor.fairness_fingerprint,
|
|
2274
|
+
runner_kind=descriptor.runner_bindings[0].runner_kind.value,
|
|
2275
|
+
descriptor=descriptor,
|
|
2276
|
+
)
|
|
2277
|
+
|
|
2278
|
+
|
|
2279
|
+
def _default_eval_trial_arm_plans(
|
|
2280
|
+
*,
|
|
2281
|
+
trial_id: str,
|
|
2282
|
+
trial_index: int,
|
|
2283
|
+
paired_seed: int,
|
|
2284
|
+
campaign_store_root: str,
|
|
2285
|
+
campaign_manifest: EvalCampaignManifest,
|
|
2286
|
+
arms: tuple[EvalTrialArmDefinition, EvalTrialArmDefinition],
|
|
2287
|
+
arm_order: tuple[EvalTrialArmId, EvalTrialArmId],
|
|
2288
|
+
fixture: EvalTaskFixture,
|
|
2289
|
+
readiness: EvalPresetReadinessReport,
|
|
2290
|
+
artifact_roots_by_arm: Mapping[EvalTrialArmId | str, str] | None = None,
|
|
2291
|
+
) -> tuple[EvalTrialArmPlan, EvalTrialArmPlan]:
|
|
2292
|
+
roots = artifact_roots_by_arm or {}
|
|
2293
|
+
compiled_hashes = _compiled_harness_hashes_by_stage(readiness)
|
|
2294
|
+
hidden_check_set_ids = tuple(check.check_id for check in fixture.hidden_checks)
|
|
2295
|
+
hidden_check_set_hash = _hidden_check_set_hash(hidden_check_set_ids)
|
|
2296
|
+
visible_acceptance_hash = _visible_acceptance_criteria_hash(
|
|
2297
|
+
fixture.visible_acceptance_criteria
|
|
2298
|
+
)
|
|
2299
|
+
plans: list[EvalTrialArmPlan] = []
|
|
2300
|
+
for arm in arms:
|
|
2301
|
+
root = roots.get(arm.arm_id) or roots.get(arm.arm_id.value)
|
|
2302
|
+
root = root or _default_artifact_root(
|
|
2303
|
+
campaign_store_root=campaign_store_root,
|
|
2304
|
+
trial_id=trial_id,
|
|
2305
|
+
arm_id=arm.arm_id,
|
|
2306
|
+
)
|
|
2307
|
+
plans.append(
|
|
2308
|
+
EvalTrialArmPlan(
|
|
2309
|
+
trial_id=trial_id,
|
|
2310
|
+
trial_index=trial_index,
|
|
2311
|
+
arm_id=arm.arm_id,
|
|
2312
|
+
arm_order_index=arm_order.index(arm.arm_id),
|
|
2313
|
+
paired_seed=paired_seed,
|
|
2314
|
+
artifact_root=root,
|
|
2315
|
+
runner_kind=arm.runner_kind,
|
|
2316
|
+
runner_descriptor_id=arm.descriptor_fingerprint,
|
|
2317
|
+
campaign_manifest_hash=campaign_manifest.campaign_manifest_hash,
|
|
2318
|
+
model_manifest_hash=campaign_manifest.model_manifest_hash,
|
|
2319
|
+
workflow_graph_hash=campaign_manifest.workflow_graph_hash,
|
|
2320
|
+
fixture_pack_hash=campaign_manifest.fixture_pack_hash,
|
|
2321
|
+
fixture_id=fixture.fixture_id,
|
|
2322
|
+
fixture_hash=fixture.fixture_hash,
|
|
2323
|
+
visible_acceptance_criteria_hash=visible_acceptance_hash,
|
|
2324
|
+
hidden_check_set_ids=hidden_check_set_ids,
|
|
2325
|
+
hidden_check_set_hash=hidden_check_set_hash,
|
|
2326
|
+
scorer_version=campaign_manifest.scorer_version,
|
|
2327
|
+
treatment_compiled_harness_hashes=compiled_hashes
|
|
2328
|
+
if arm.arm_id is EvalTrialArmId.EVAL_SMALL_MILLFORGE
|
|
2329
|
+
else {},
|
|
2330
|
+
baseline_pi_runtime_hash=None,
|
|
2331
|
+
baseline_runtime_diagnostic=EvalTrialInvalidDiagnostic(
|
|
2332
|
+
diagnostic_code=(
|
|
2333
|
+
EvalTrialInvalidDiagnosticCode.LIVE_EXECUTION_UNAVAILABLE
|
|
2334
|
+
),
|
|
2335
|
+
rule_id="eval_trials.pi_runtime.deferred",
|
|
2336
|
+
summary="Pi runtime hash is deferred for offline planning.",
|
|
2337
|
+
)
|
|
2338
|
+
if arm.arm_id is EvalTrialArmId.EVAL_SMALL_PI
|
|
2339
|
+
else None,
|
|
2340
|
+
)
|
|
2341
|
+
)
|
|
2342
|
+
return (plans[0], plans[1])
|
|
2343
|
+
|
|
2344
|
+
|
|
2345
|
+
def _validate_admitted_planning_arms(
|
|
2346
|
+
arms: tuple[EvalTrialArmDefinition, EvalTrialArmDefinition],
|
|
2347
|
+
) -> None:
|
|
2348
|
+
if tuple(arm.arm_id.value for arm in arms) != EVAL_TRIAL_ADMITTED_ARM_IDS:
|
|
2349
|
+
raise ValueError("paired trial planning rejects unsupported arms")
|
|
2350
|
+
|
|
2351
|
+
|
|
2352
|
+
def _default_campaign_store_root(campaign_manifest: EvalCampaignManifest) -> str:
|
|
2353
|
+
return f"eval/campaigns/{campaign_manifest.campaign_id}"
|
|
2354
|
+
|
|
2355
|
+
|
|
2356
|
+
def _default_artifact_root(
|
|
2357
|
+
*,
|
|
2358
|
+
campaign_store_root: str,
|
|
2359
|
+
trial_id: str,
|
|
2360
|
+
arm_id: EvalTrialArmId,
|
|
2361
|
+
) -> str:
|
|
2362
|
+
_validate_relative_artifact_root(campaign_store_root)
|
|
2363
|
+
return f"artifacts/{trial_id}/{arm_id.value}"
|
|
2364
|
+
|
|
2365
|
+
|
|
2366
|
+
def _trial_id(
|
|
2367
|
+
*,
|
|
2368
|
+
campaign_id: str,
|
|
2369
|
+
fixture_id: str,
|
|
2370
|
+
trial_index: int,
|
|
2371
|
+
seed: int,
|
|
2372
|
+
) -> str:
|
|
2373
|
+
digest = hashlib.sha256(
|
|
2374
|
+
_canonical_eval_trial_bytes(
|
|
2375
|
+
{
|
|
2376
|
+
"campaign_id": campaign_id,
|
|
2377
|
+
"fixture_id": fixture_id,
|
|
2378
|
+
"seed": seed,
|
|
2379
|
+
"trial_index": trial_index,
|
|
2380
|
+
}
|
|
2381
|
+
)
|
|
2382
|
+
).hexdigest()[:12]
|
|
2383
|
+
safe_fixture = re.sub(r"[^a-zA-Z0-9_.-]+", "_", fixture_id)
|
|
2384
|
+
return f"{campaign_id}.trial.{trial_index:04d}.{safe_fixture}.{digest}"
|
|
2385
|
+
|
|
2386
|
+
|
|
2387
|
+
def _paired_seed(*, seed: int, fixture_id: str, trial_index: int) -> int:
|
|
2388
|
+
digest = hashlib.sha256(
|
|
2389
|
+
_canonical_eval_trial_bytes(
|
|
2390
|
+
{"fixture_id": fixture_id, "seed": seed, "trial_index": trial_index}
|
|
2391
|
+
)
|
|
2392
|
+
).hexdigest()
|
|
2393
|
+
return int(digest[:16], 16)
|
|
2394
|
+
|
|
2395
|
+
|
|
2396
|
+
def _randomized_arm_order(
|
|
2397
|
+
paired_seed: int,
|
|
2398
|
+
) -> tuple[EvalTrialArmId, EvalTrialArmId]:
|
|
2399
|
+
arms = [EvalTrialArmId.EVAL_SMALL_PI, EvalTrialArmId.EVAL_SMALL_MILLFORGE]
|
|
2400
|
+
random.Random(paired_seed).shuffle(arms)
|
|
2401
|
+
return (arms[0], arms[1])
|
|
2402
|
+
|
|
2403
|
+
|
|
2404
|
+
def _fixture_instance_id(
|
|
2405
|
+
*,
|
|
2406
|
+
trial_id: str,
|
|
2407
|
+
trial_index: int,
|
|
2408
|
+
fixture_id: str,
|
|
2409
|
+
paired_seed: int,
|
|
2410
|
+
) -> str:
|
|
2411
|
+
digest = hashlib.sha256(
|
|
2412
|
+
_canonical_eval_trial_bytes(
|
|
2413
|
+
{
|
|
2414
|
+
"fixture_id": fixture_id,
|
|
2415
|
+
"paired_seed": paired_seed,
|
|
2416
|
+
"trial_id": trial_id,
|
|
2417
|
+
"trial_index": trial_index,
|
|
2418
|
+
}
|
|
2419
|
+
)
|
|
2420
|
+
).hexdigest()[:16]
|
|
2421
|
+
return f"{trial_id}.fixture_instance.{digest}"
|
|
2422
|
+
|
|
2423
|
+
|
|
2424
|
+
def _fixture_snapshot_hash(
|
|
2425
|
+
*,
|
|
2426
|
+
fixture: EvalTaskFixture,
|
|
2427
|
+
fixture_instance_id: str,
|
|
2428
|
+
paired_seed: int,
|
|
2429
|
+
) -> str:
|
|
2430
|
+
return hashlib.sha256(
|
|
2431
|
+
_canonical_eval_trial_bytes(
|
|
2432
|
+
{
|
|
2433
|
+
"fixture_hash": fixture.fixture_hash,
|
|
2434
|
+
"fixture_id": fixture.fixture_id,
|
|
2435
|
+
"fixture_instance_id": fixture_instance_id,
|
|
2436
|
+
"paired_seed": paired_seed,
|
|
2437
|
+
}
|
|
2438
|
+
)
|
|
2439
|
+
).hexdigest()
|
|
2440
|
+
|
|
2441
|
+
|
|
2442
|
+
def _visible_acceptance_criteria_hash(criteria: tuple[str, ...]) -> str:
|
|
2443
|
+
return hashlib.sha256(
|
|
2444
|
+
_canonical_eval_trial_bytes({"visible_acceptance_criteria": criteria})
|
|
2445
|
+
).hexdigest()
|
|
2446
|
+
|
|
2447
|
+
|
|
2448
|
+
def _hidden_check_set_hash(check_ids: tuple[str, ...]) -> str:
|
|
2449
|
+
return hashlib.sha256(
|
|
2450
|
+
_canonical_eval_trial_bytes({"hidden_check_set_ids": check_ids})
|
|
2451
|
+
).hexdigest()
|
|
2452
|
+
|
|
2453
|
+
|
|
2454
|
+
def _compiled_harness_hashes_by_stage(
|
|
2455
|
+
readiness: EvalPresetReadinessReport,
|
|
2456
|
+
) -> Mapping[str, str]:
|
|
2457
|
+
required_stage_ids = tuple(stage_id.value for stage_id in EvalStageId)
|
|
2458
|
+
compiled = {
|
|
2459
|
+
plan.stage_id: plan.compiled_sha256 for plan in readiness.compiled_plans
|
|
2460
|
+
}
|
|
2461
|
+
if tuple(compiled) != required_stage_ids:
|
|
2462
|
+
raise ValueError(
|
|
2463
|
+
"Spec 07E readiness must include Planner, Builder, Checker, Arbiter"
|
|
2464
|
+
)
|
|
2465
|
+
return _freeze_trial_mapping(compiled)
|
|
2466
|
+
|
|
2467
|
+
|
|
2468
|
+
def _reject_fake_pi_runtime_hash(value: str) -> None:
|
|
2469
|
+
if value in {"0" * 64, "f" * 64} or len(set(value)) == 1:
|
|
2470
|
+
raise ValueError("Pi baseline runtime hash must not be a fake concrete hash")
|
|
2471
|
+
|
|
2472
|
+
|
|
2473
|
+
def _validate_relative_artifact_root(path: str) -> None:
|
|
2474
|
+
if (
|
|
2475
|
+
not path
|
|
2476
|
+
or _WINDOWS_ABSOLUTE_PATH.search(path)
|
|
2477
|
+
or _POSIX_ABSOLUTE_PATH.search(path)
|
|
2478
|
+
or _USER_HOME_PATH.search(path)
|
|
2479
|
+
):
|
|
2480
|
+
raise ValueError("artifact roots must be stable relative paths")
|
|
2481
|
+
pure = PurePosixPath(path)
|
|
2482
|
+
if pure.is_absolute() or any(part in {"", ".", ".."} for part in pure.parts):
|
|
2483
|
+
raise ValueError("artifact roots must be stable relative paths")
|
|
2484
|
+
if "\\" in path:
|
|
2485
|
+
raise ValueError("artifact roots must use relative POSIX paths")
|
|
2486
|
+
|
|
2487
|
+
|
|
2488
|
+
def _validate_artifact_root_under_campaign(
|
|
2489
|
+
artifact_root: str,
|
|
2490
|
+
campaign_store_root: str,
|
|
2491
|
+
) -> None:
|
|
2492
|
+
_validate_relative_artifact_root(artifact_root)
|
|
2493
|
+
_validate_relative_artifact_root(campaign_store_root)
|
|
2494
|
+
artifact_parts = PurePosixPath(artifact_root).parts
|
|
2495
|
+
if len(artifact_parts) < 2 or artifact_parts[0] != "artifacts":
|
|
2496
|
+
raise ValueError("artifact roots must stay inside the campaign store")
|
|
2497
|
+
|
|
2498
|
+
|
|
2499
|
+
def _canonical_eval_trial_bytes(value: Mapping[str, Any]) -> bytes:
|
|
2500
|
+
return (
|
|
2501
|
+
json.dumps(
|
|
2502
|
+
value,
|
|
2503
|
+
sort_keys=True,
|
|
2504
|
+
ensure_ascii=True,
|
|
2505
|
+
allow_nan=False,
|
|
2506
|
+
separators=(",", ":"),
|
|
2507
|
+
).replace("\r\n", "\n")
|
|
2508
|
+
+ "\n"
|
|
2509
|
+
).encode("ascii")
|
|
2510
|
+
|
|
2511
|
+
|
|
2512
|
+
def _validate_sha256(value: str) -> None:
|
|
2513
|
+
if not _SHA256_RE.fullmatch(value):
|
|
2514
|
+
raise ValueError("expected lowercase sha256 hex digest")
|
|
2515
|
+
|
|
2516
|
+
|
|
2517
|
+
def _reject_forbidden_material(value: Any) -> None:
|
|
2518
|
+
if isinstance(value, Mapping):
|
|
2519
|
+
for key, child in value.items():
|
|
2520
|
+
key_text = str(key)
|
|
2521
|
+
_reject_secret_like_field_name(key_text)
|
|
2522
|
+
_reject_forbidden_material(key_text)
|
|
2523
|
+
_reject_forbidden_material(child)
|
|
2524
|
+
return
|
|
2525
|
+
if isinstance(value, (tuple, list, set, frozenset)):
|
|
2526
|
+
for child in value:
|
|
2527
|
+
_reject_forbidden_material(child)
|
|
2528
|
+
return
|
|
2529
|
+
if isinstance(value, BaseModel):
|
|
2530
|
+
_reject_forbidden_material(value.model_dump(mode="json"))
|
|
2531
|
+
return
|
|
2532
|
+
if isinstance(value, Enum):
|
|
2533
|
+
_reject_forbidden_material(value.value)
|
|
2534
|
+
return
|
|
2535
|
+
if isinstance(value, str):
|
|
2536
|
+
lowered = value.lower()
|
|
2537
|
+
if any(token in lowered for token in _DENIED_TEXT_TOKENS):
|
|
2538
|
+
raise ValueError("eval-trial payload contains forbidden private material")
|
|
2539
|
+
if any(pattern.search(value) for pattern in _CREDENTIAL_VALUE_PATTERNS):
|
|
2540
|
+
raise ValueError("eval-trial payload contains credential-shaped API key")
|
|
2541
|
+
if _ENDPOINT_URL.search(value):
|
|
2542
|
+
raise ValueError("eval-trial payloads must not contain endpoint URLs")
|
|
2543
|
+
if (
|
|
2544
|
+
_WINDOWS_ABSOLUTE_PATH.search(value)
|
|
2545
|
+
or _POSIX_ABSOLUTE_PATH.search(value)
|
|
2546
|
+
or _USER_HOME_PATH.search(value)
|
|
2547
|
+
):
|
|
2548
|
+
raise ValueError("eval-trial payloads must not contain host paths")
|
|
2549
|
+
|
|
2550
|
+
|
|
2551
|
+
def _reject_secret_like_field_name(field_name: str) -> None:
|
|
2552
|
+
normalized = field_name.lower().replace("-", "_")
|
|
2553
|
+
if normalized.endswith("_free"):
|
|
2554
|
+
return
|
|
2555
|
+
if any(marker in normalized for marker in _SECRET_FIELD_MARKERS):
|
|
2556
|
+
raise ValueError("eval-trial payload contains secret-like field name")
|
|
2557
|
+
|
|
2558
|
+
|
|
2559
|
+
class _FrozenTrialDict(dict[Any, Any]):
|
|
2560
|
+
"""Dict-shaped immutable mapping that remains serializable by Pydantic."""
|
|
2561
|
+
|
|
2562
|
+
def __readonly(self, *args: Any, **kwargs: Any) -> None:
|
|
2563
|
+
raise TypeError("eval-trial mappings are immutable")
|
|
2564
|
+
|
|
2565
|
+
__setitem__ = __readonly
|
|
2566
|
+
__delitem__ = __readonly
|
|
2567
|
+
clear = __readonly
|
|
2568
|
+
pop = __readonly
|
|
2569
|
+
popitem = __readonly # type: ignore[assignment]
|
|
2570
|
+
setdefault = __readonly
|
|
2571
|
+
update = __readonly
|
|
2572
|
+
__ior__ = __readonly # type: ignore[assignment]
|
|
2573
|
+
|
|
2574
|
+
|
|
2575
|
+
def _freeze_trial_mapping(value: Mapping[Any, Any]) -> Mapping[Any, Any]:
|
|
2576
|
+
return _FrozenTrialDict({key: child for key, child in value.items()})
|
|
2577
|
+
|
|
2578
|
+
|
|
2579
|
+
__all__ = [
|
|
2580
|
+
"EVAL_TRIAL_ADMITTED_ARM_IDS",
|
|
2581
|
+
"EVAL_TRIAL_ARTIFACT_BUNDLE_HASH_KIND",
|
|
2582
|
+
"EVAL_TRIAL_DEFAULT_CREATED_AT",
|
|
2583
|
+
"EVAL_TRIAL_PLAN_HASH_KIND",
|
|
2584
|
+
"EVAL_TRIAL_RECORD_HASH_KIND",
|
|
2585
|
+
"EVAL_TRIAL_RESUME_INDEX_HASH_KIND",
|
|
2586
|
+
"EVAL_TRIAL_SCHEMA_VERSION",
|
|
2587
|
+
"EVAL_TRIAL_STORE_MANIFEST_HASH_KIND",
|
|
2588
|
+
"EvalFakeOutcomeScriptKind",
|
|
2589
|
+
"EvalFakeRunnerArtifactBundle",
|
|
2590
|
+
"EvalOfflineFakeTrialRun",
|
|
2591
|
+
"EvalTrialCampaignStoreAppendResult",
|
|
2592
|
+
"EvalTrialCampaignStoreRecordSummary",
|
|
2593
|
+
"EvalTrialCampaignStoreResumeResult",
|
|
2594
|
+
"EvalTrialArmDefinition",
|
|
2595
|
+
"EvalTrialArmId",
|
|
2596
|
+
"EvalTrialArmParityEvidence",
|
|
2597
|
+
"EvalTrialArmPlan",
|
|
2598
|
+
"EvalTrialContractModel",
|
|
2599
|
+
"EvalTrialExecutionResult",
|
|
2600
|
+
"EvalTrialFakeRunnerScript",
|
|
2601
|
+
"EvalTrialFixtureInstance",
|
|
2602
|
+
"EvalTrialInvalidDiagnostic",
|
|
2603
|
+
"EvalTrialInvalidDiagnosticCode",
|
|
2604
|
+
"EvalTrialPlan",
|
|
2605
|
+
"EvalTrialRecord",
|
|
2606
|
+
"EvalTrialModelUsageSummary",
|
|
2607
|
+
"EvalTrialResumeIndex",
|
|
2608
|
+
"EvalTrialResourceSummary",
|
|
2609
|
+
"EvalTrialRunnerRecordSummary",
|
|
2610
|
+
"EvalTrialScorerPublicSummary",
|
|
2611
|
+
"EvalTrialStoreManifest",
|
|
2612
|
+
"calculate_eval_fake_runner_artifact_bundle_hash",
|
|
2613
|
+
"calculate_eval_trial_plan_hash",
|
|
2614
|
+
"calculate_eval_trial_record_hash",
|
|
2615
|
+
"calculate_eval_trial_resume_index_hash",
|
|
2616
|
+
"calculate_eval_trial_store_manifest_hash",
|
|
2617
|
+
"canonical_eval_fake_runner_artifact_bundle_bytes",
|
|
2618
|
+
"canonical_eval_trial_plan_bytes",
|
|
2619
|
+
"canonical_eval_trial_record_bytes",
|
|
2620
|
+
"canonical_eval_trial_resume_index_bytes",
|
|
2621
|
+
"canonical_eval_trial_store_manifest_bytes",
|
|
2622
|
+
"append_eval_trial_record_to_campaign_store",
|
|
2623
|
+
"default_eval_trial_arm_definitions",
|
|
2624
|
+
"default_eval_trial_fixture_instance",
|
|
2625
|
+
"default_eval_trial_parity_evidence",
|
|
2626
|
+
"default_eval_trial_plan",
|
|
2627
|
+
"deny_eval_trial_live_execution",
|
|
2628
|
+
"fixture_copy_unavailable_diagnostic",
|
|
2629
|
+
"plan_paired_eval_trials",
|
|
2630
|
+
"resume_eval_trial_campaign_store",
|
|
2631
|
+
"run_offline_fake_eval_trial",
|
|
2632
|
+
]
|