millforge 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. millforge/__init__.py +1174 -0
  2. millforge/_forge/LICENSE +21 -0
  3. millforge/_forge/PROVENANCE.json +295 -0
  4. millforge/_forge/UPDATE_POLICY.md +24 -0
  5. millforge/_forge/__init__.py +14 -0
  6. millforge/_forge/adapter.py +2232 -0
  7. millforge/_forge/base_runner.py +121 -0
  8. millforge/_forge/clients/__init__.py +10 -0
  9. millforge/_forge/clients/base.py +200 -0
  10. millforge/_forge/context/__init__.py +23 -0
  11. millforge/_forge/context/manager.py +178 -0
  12. millforge/_forge/context/strategies.py +335 -0
  13. millforge/_forge/core/__init__.py +16 -0
  14. millforge/_forge/core/inference.py +433 -0
  15. millforge/_forge/core/messages.py +119 -0
  16. millforge/_forge/core/runner.py +479 -0
  17. millforge/_forge/core/steps.py +108 -0
  18. millforge/_forge/core/workflow.py +400 -0
  19. millforge/_forge/errors.py +222 -0
  20. millforge/_forge/guardrails/__init__.py +21 -0
  21. millforge/_forge/guardrails/error_tracker.py +71 -0
  22. millforge/_forge/guardrails/guardrails.py +194 -0
  23. millforge/_forge/guardrails/nudge.py +47 -0
  24. millforge/_forge/guardrails/response_validator.py +119 -0
  25. millforge/_forge/guardrails/step_enforcer.py +183 -0
  26. millforge/_forge/prompts/__init__.py +16 -0
  27. millforge/_forge/prompts/nudges.py +95 -0
  28. millforge/_forge/prompts/templates.py +285 -0
  29. millforge/_version.py +3 -0
  30. millforge/artifacts.py +570 -0
  31. millforge/base/__init__.py +97 -0
  32. millforge/base/composition.py +402 -0
  33. millforge/base/context.py +285 -0
  34. millforge/base/harness.py +138 -0
  35. millforge/base/identity.py +465 -0
  36. millforge/base/options.py +34 -0
  37. millforge/base/platform.py +17 -0
  38. millforge/base/prompt.py +317 -0
  39. millforge/base/runner.py +546 -0
  40. millforge/compiled_plan.py +970 -0
  41. millforge/compiler/__init__.py +231 -0
  42. millforge/compiler/artifact_validation.py +257 -0
  43. millforge/compiler/canonicalization.py +169 -0
  44. millforge/compiler/capabilities.py +66 -0
  45. millforge/compiler/catalogs.py +500 -0
  46. millforge/compiler/diagnostics.py +491 -0
  47. millforge/compiler/graph.py +678 -0
  48. millforge/compiler/lowering.py +198 -0
  49. millforge/compiler/output.py +692 -0
  50. millforge/compiler/parsing.py +1424 -0
  51. millforge/compiler/requests.py +1180 -0
  52. millforge/compiler/schema_validation.py +272 -0
  53. millforge/compiler/semantic.py +490 -0
  54. millforge/compiler/service.py +448 -0
  55. millforge/compiler/source.py +375 -0
  56. millforge/compiler/validators.py +184 -0
  57. millforge/connectors/__init__.py +95 -0
  58. millforge/connectors/admission.py +801 -0
  59. millforge/connectors/broker.py +202 -0
  60. millforge/connectors/contracts.py +1159 -0
  61. millforge/connectors/diagnostics.py +189 -0
  62. millforge/connectors/fake.py +66 -0
  63. millforge/connectors/runtime.py +236 -0
  64. millforge/contracts.py +2860 -0
  65. millforge/custom_tools/__init__.py +67 -0
  66. millforge/custom_tools/compiler.py +724 -0
  67. millforge/custom_tools/contracts.py +1093 -0
  68. millforge/custom_tools/diagnostics.py +205 -0
  69. millforge/eval_artifacts.py +952 -0
  70. millforge/eval_boundary.py +2435 -0
  71. millforge/eval_fixtures/__init__.py +1 -0
  72. millforge/eval_fixtures/default_pack/__init__.py +1 -0
  73. millforge/eval_fixtures/default_pack/fixtures/fixture.08a.bug_diagnosis.traceback.v1.json +52 -0
  74. millforge/eval_fixtures/default_pack/fixtures/fixture.08a.direct_edit.import_sort.v1.json +52 -0
  75. millforge/eval_fixtures/default_pack/fixtures/fixture.08a.evidence_discipline.no_source_change.v1.json +51 -0
  76. millforge/eval_fixtures/default_pack/fixtures/fixture.08a.false_closure.visible_green.v1.json +52 -0
  77. millforge/eval_fixtures/default_pack/fixtures/fixture.08a.multi_file.api_contract.v1.json +54 -0
  78. millforge/eval_fixtures/default_pack/fixtures/fixture.08a.recovery.malformed_artifact.v1.json +54 -0
  79. millforge/eval_fixtures/default_pack/manifest.json +12 -0
  80. millforge/eval_modes.py +1282 -0
  81. millforge/eval_presets.py +1398 -0
  82. millforge/eval_reports.py +2517 -0
  83. millforge/eval_suite.py +2429 -0
  84. millforge/eval_trials.py +2632 -0
  85. millforge/eval_workflow.py +794 -0
  86. millforge/exceptions.py +122 -0
  87. millforge/model_backend.py +2098 -0
  88. millforge/protocols.py +340 -0
  89. millforge/py.typed +0 -0
  90. millforge/runtime.py +1791 -0
  91. millforge/testing/__init__.py +1089 -0
  92. millforge/tools/__init__.py +83 -0
  93. millforge/tools/builtin_runtime.py +1339 -0
  94. millforge/tools/builtins.py +773 -0
  95. millforge/tools/execution.py +1545 -0
  96. millforge/tools/path_policy.py +155 -0
  97. millforge/tools/pi_compat/PI_LICENSE +21 -0
  98. millforge/tools/pi_compat/PROVENANCE.json +55 -0
  99. millforge/tools/pi_compat/UPDATE_POLICY.md +36 -0
  100. millforge/tools/pi_compat/__init__.py +34 -0
  101. millforge/tools/pi_compat/contracts.py +49 -0
  102. millforge/tools/pi_compat/editing.py +390 -0
  103. millforge/tools/pi_compat/mutations.py +57 -0
  104. millforge/tools/pi_compat/operations.py +401 -0
  105. millforge/tools/pi_compat/paths.py +155 -0
  106. millforge/tools/pi_compat/process.py +1375 -0
  107. millforge/tools/pi_compat/search.py +738 -0
  108. millforge/tools/pi_compat/truncation.py +267 -0
  109. millforge/tools/pi_compat_catalog.py +396 -0
  110. millforge/tools/pi_compat_runtime.py +460 -0
  111. millforge/tools/registry.py +553 -0
  112. millforge/tools/results.py +533 -0
  113. millforge-0.1.0.dist-info/METADATA +844 -0
  114. millforge-0.1.0.dist-info/RECORD +116 -0
  115. millforge-0.1.0.dist-info/WHEEL +4 -0
  116. millforge-0.1.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,2632 @@
1
+ """Public 08B offline eval-trial contract boundary."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ import random
8
+ import re
9
+ from collections.abc import Mapping
10
+ from enum import Enum
11
+ from pathlib import Path, PurePosixPath
12
+ from typing import Any
13
+
14
+ from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictInt, StrictStr
15
+ from pydantic import field_serializer
16
+ from pydantic import model_validator
17
+
18
+ from millforge.eval_artifacts import (
19
+ EvalArtifactId,
20
+ EvalArtifactManifestArtifact,
21
+ EvalArtifactManifestEntry,
22
+ calculate_eval_artifact_manifest_sha256,
23
+ eval_artifact_layout_entry,
24
+ )
25
+ from millforge.eval_modes import (
26
+ EVAL_SMALL_MILLFORGE_MODE_ID,
27
+ EVAL_SMALL_PI_MODE_ID,
28
+ EvalModeDescriptor,
29
+ default_eval_small_millforge_mode,
30
+ default_eval_small_pi_mode,
31
+ )
32
+ from millforge.eval_presets import (
33
+ EvalPresetReadinessReport,
34
+ eval_preset_readiness_report,
35
+ )
36
+ from millforge.eval_suite import (
37
+ EvalCampaignManifest,
38
+ EvalFixturePackSummary,
39
+ EvalHashRecord,
40
+ EvalPublicArtifactProjection,
41
+ EvalScorerInput,
42
+ EvalScorerResult,
43
+ EvalSuiteExecutionMode,
44
+ EvalCapabilityAuditSummary,
45
+ EvalCheckResult,
46
+ EvalTaskFixture,
47
+ EvalTrialOutcome,
48
+ calculate_eval_scorer_result_hash,
49
+ calculate_eval_scorer_input_hash,
50
+ default_eval_suite_campaign_manifest,
51
+ eval_public_artifact_projection,
52
+ load_eval_fixture_pack_summary,
53
+ load_eval_task_fixture,
54
+ load_eval_task_fixtures,
55
+ score_eval_trial,
56
+ )
57
+ from millforge.eval_workflow import EvalStageId, EvalTerminalResult
58
+
59
+ EVAL_TRIAL_SCHEMA_VERSION = 1
60
+ EVAL_TRIAL_PLAN_HASH_KIND = "eval_trial_plan_sha256_v1"
61
+ EVAL_TRIAL_ARTIFACT_BUNDLE_HASH_KIND = "eval_trial_artifact_bundle_sha256_v1"
62
+ EVAL_TRIAL_RECORD_HASH_KIND = "eval_trial_record_sha256_v1"
63
+ EVAL_TRIAL_STORE_MANIFEST_HASH_KIND = "eval_trial_store_manifest_sha256_v1"
64
+ EVAL_TRIAL_RESUME_INDEX_HASH_KIND = "eval_trial_resume_index_sha256_v1"
65
+ EVAL_TRIAL_DEFAULT_CREATED_AT = "1970-01-01T00:00:00Z"
66
+ EVAL_TRIAL_ADMITTED_ARM_IDS: tuple[str, str] = (
67
+ EVAL_SMALL_PI_MODE_ID,
68
+ EVAL_SMALL_MILLFORGE_MODE_ID,
69
+ )
70
+
71
+ _SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
72
+ _UTC_TIMESTAMP_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$")
73
+ _DENIED_TEXT_TOKENS = (
74
+ "api_key",
75
+ "authorization:",
76
+ "bearer ",
77
+ "credential",
78
+ "password",
79
+ "access token",
80
+ "auth token",
81
+ "millrace-agents",
82
+ ".millrace",
83
+ "daemon state",
84
+ "endpoint_url",
85
+ "external service",
86
+ "hidden_checks",
87
+ "hidden scorer",
88
+ "hidden answer",
89
+ "private runtime",
90
+ "network access",
91
+ "package install",
92
+ "scorer_rubric",
93
+ "ref-forge",
94
+ ".claude",
95
+ ".codex",
96
+ )
97
+ _WINDOWS_ABSOLUTE_PATH = re.compile(
98
+ r"(?:^|[\s\"'`([{<])(?:[A-Za-z]:[\\/]|\\\\)[^\s\"'`)\]}>,]*"
99
+ )
100
+ _POSIX_ABSOLUTE_PATH = re.compile(r"(?:^|[\s\"'`([{<])/(?!/)[^\s\"'`)\]}>,]*")
101
+ _USER_HOME_PATH = re.compile(r"(?:^|[\s\"'`([{<])~(?:/|\\)[^\s\"'`)\]}>,]*")
102
+ _ENDPOINT_URL = re.compile(r"https?://|localhost(?::|/|$)|127\.0\.0\.1|0\.0\.0\.0")
103
+ _CREDENTIAL_VALUE_PATTERNS = (
104
+ re.compile(r"\bsk-(?:live|proj|test)-[A-Za-z0-9_-]{10,}\b"),
105
+ re.compile(r"\b[rs]k_(?:live|test)_[A-Za-z0-9]{16,}\b"),
106
+ re.compile(r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b"),
107
+ re.compile(r"\bAIza[0-9A-Za-z_-]{20,}\b"),
108
+ re.compile(r"\bgh[opsu]_[A-Za-z0-9_]{20,}\b"),
109
+ )
110
+ _SECRET_FIELD_MARKERS = (
111
+ "api_key",
112
+ "apikey",
113
+ "auth_header",
114
+ "authorization",
115
+ "bearer",
116
+ "client_secret",
117
+ "credential",
118
+ "password",
119
+ "private_key",
120
+ "secret",
121
+ "access_token",
122
+ "auth_token",
123
+ "refresh_token",
124
+ )
125
+
126
+
127
+ class EvalTrialContractModel(BaseModel):
128
+ """Closed, frozen base for public eval-trial contracts."""
129
+
130
+ model_config = ConfigDict(extra="forbid", frozen=True)
131
+
132
+ @model_validator(mode="before")
133
+ @classmethod
134
+ def _reject_private_material(cls, data: Any) -> Any:
135
+ _reject_forbidden_material(data)
136
+ return data
137
+
138
+
139
+ class EvalTrialArmId(str, Enum):
140
+ """Exactly admitted offline comparison arms for 08B trials."""
141
+
142
+ EVAL_SMALL_PI = EVAL_SMALL_PI_MODE_ID
143
+ EVAL_SMALL_MILLFORGE = EVAL_SMALL_MILLFORGE_MODE_ID
144
+
145
+
146
+ class EvalFakeOutcomeScriptKind(str, Enum):
147
+ """Closed offline fake outcome script kinds."""
148
+
149
+ VALID_COMPLETION = "valid_completion"
150
+ CORRECT_BLOCK = "correct_block"
151
+ FALSE_CLOSURE = "false_closure"
152
+ FALSE_SUCCESS_WITHOUT_CLOSURE = "false_success_without_closure"
153
+ RUNTIME_FAILURE = "runtime_failure"
154
+ PROVIDER_FAILURE = "provider_failure"
155
+ INVALID_TRIAL = "invalid_trial"
156
+
157
+
158
+ class EvalTrialInvalidDiagnosticCode(str, Enum):
159
+ """Closed invalid-trial diagnostic codes emitted by offline contracts."""
160
+
161
+ LIVE_EXECUTION_UNAVAILABLE = "live_execution_unavailable"
162
+ ARM_NOT_ADMITTED = "arm_not_admitted"
163
+ ARM_PARITY_DRIFT = "arm_parity_drift"
164
+ FIXTURE_COPY_UNAVAILABLE = "fixture_copy_unavailable"
165
+ INVALID_PLANNING_REQUEST = "invalid_planning_request"
166
+ PLAN_HASH_MISMATCH = "plan_hash_mismatch"
167
+ ARTIFACT_BUNDLE_HASH_MISMATCH = "artifact_bundle_hash_mismatch"
168
+ RECORD_HASH_MISMATCH = "record_hash_mismatch"
169
+ RESUME_INDEX_HASH_MISMATCH = "resume_index_hash_mismatch"
170
+ STORE_MANIFEST_HASH_MISMATCH = "store_manifest_hash_mismatch"
171
+ TREATMENT_HARNESS_HASH_MISMATCH = "treatment_harness_hash_mismatch"
172
+ BASELINE_RUNTIME_HASH_UNAVAILABLE = "baseline_runtime_hash_unavailable"
173
+ TRIAL_LOG_DUPLICATE_TRIAL_ID = "trial_log_duplicate_trial_id"
174
+ TRIAL_LOG_INVALID_RECORD = "trial_log_invalid_record"
175
+ TRIAL_LOG_MALFORMED_TRAILING_LINE = "trial_log_malformed_trailing_line"
176
+ TRIAL_LOG_MISSING_FINAL_NEWLINE = "trial_log_missing_final_newline"
177
+ TRIAL_LOG_PARTIAL_TRAILING_RECORD = "trial_log_partial_trailing_record"
178
+
179
+
180
+ class EvalTrialInvalidDiagnostic(EvalTrialContractModel):
181
+ """Typed fail-closed diagnostic for invalid or unavailable trials."""
182
+
183
+ diagnostic_code: EvalTrialInvalidDiagnosticCode
184
+ rule_id: StrictStr
185
+ summary: StrictStr
186
+
187
+
188
+ class EvalTrialArmDefinition(EvalTrialContractModel):
189
+ """One admitted comparison arm bound to an existing eval mode descriptor."""
190
+
191
+ arm_id: EvalTrialArmId
192
+ mode_id: StrictStr
193
+ descriptor_fingerprint: StrictStr
194
+ fairness_fingerprint: StrictStr
195
+ runner_kind: StrictStr
196
+ descriptor: EvalModeDescriptor
197
+
198
+ @model_validator(mode="after")
199
+ def _arm_valid(self) -> EvalTrialArmDefinition:
200
+ if self.mode_id != self.arm_id.value:
201
+ raise ValueError("arm mode_id must match arm_id")
202
+ if self.descriptor.mode_id != self.arm_id.value:
203
+ raise ValueError("arm descriptor mode_id must match arm_id")
204
+ if self.descriptor.descriptor_fingerprint != self.descriptor_fingerprint:
205
+ raise ValueError("descriptor_fingerprint must match descriptor")
206
+ if self.descriptor.fairness_fingerprint != self.fairness_fingerprint:
207
+ raise ValueError("fairness_fingerprint must match descriptor")
208
+ runner_kinds = {
209
+ binding.runner_kind.value for binding in self.descriptor.runner_bindings
210
+ }
211
+ if runner_kinds != {self.runner_kind}:
212
+ raise ValueError("runner_kind must match descriptor bindings")
213
+ return self
214
+
215
+
216
+ class EvalTrialArmParityEvidence(EvalTrialContractModel):
217
+ """Hash evidence proving the two offline arms share fairness-critical inputs."""
218
+
219
+ left_arm_id: EvalTrialArmId
220
+ right_arm_id: EvalTrialArmId
221
+ shared_fairness_fingerprint: StrictStr
222
+ left_descriptor_fingerprint: StrictStr
223
+ right_descriptor_fingerprint: StrictStr
224
+ workflow_graph_hash: StrictStr
225
+ model_profile_hash: StrictStr
226
+ fixture_pack_hash: StrictStr
227
+ comparable_offline: StrictBool
228
+ diagnostics: tuple[EvalTrialInvalidDiagnostic, ...] = Field(default_factory=tuple)
229
+
230
+ @model_validator(mode="after")
231
+ def _parity_valid(self) -> EvalTrialArmParityEvidence:
232
+ if self.left_arm_id == self.right_arm_id:
233
+ raise ValueError("parity evidence requires distinct arms")
234
+ for digest in (
235
+ self.shared_fairness_fingerprint,
236
+ self.left_descriptor_fingerprint,
237
+ self.right_descriptor_fingerprint,
238
+ self.workflow_graph_hash,
239
+ self.model_profile_hash,
240
+ self.fixture_pack_hash,
241
+ ):
242
+ _validate_sha256(digest)
243
+ if self.comparable_offline and self.diagnostics:
244
+ raise ValueError("comparable offline arms must not carry diagnostics")
245
+ if not self.comparable_offline and not self.diagnostics:
246
+ raise ValueError("non-comparable parity evidence requires diagnostics")
247
+ return self
248
+
249
+
250
+ class EvalTrialFixtureInstance(EvalTrialContractModel):
251
+ """One deterministic fixture instance selected for an offline trial."""
252
+
253
+ fixture_instance_id: StrictStr
254
+ fixture_id: StrictStr
255
+ fixture_hash: StrictStr
256
+ fixture_snapshot_hash: StrictStr
257
+ fixture_pack_id: StrictStr
258
+ fixture_pack_hash: StrictStr
259
+ hidden_check_set_ids: tuple[StrictStr, ...]
260
+ hidden_check_set_hash: StrictStr
261
+ public_projection: EvalPublicArtifactProjection
262
+ fixture_pack_summary: EvalFixturePackSummary
263
+ physical_copy_available: StrictBool = False
264
+ copy_unavailable_diagnostic: EvalTrialInvalidDiagnostic | None = None
265
+
266
+ @model_validator(mode="after")
267
+ def _fixture_instance_valid(self) -> EvalTrialFixtureInstance:
268
+ if self.public_projection.fixture_id != self.fixture_id:
269
+ raise ValueError("fixture_id must match public fixture projection")
270
+ if self.fixture_pack_summary.fixture_pack_id != self.fixture_pack_id:
271
+ raise ValueError("fixture_pack_id must match fixture pack summary")
272
+ if self.fixture_pack_summary.fixture_pack_hash != self.fixture_pack_hash:
273
+ raise ValueError("fixture_pack_hash must match fixture pack summary")
274
+ pack_hashes = dict(
275
+ zip(
276
+ self.fixture_pack_summary.fixture_ids,
277
+ self.fixture_pack_summary.fixture_hashes,
278
+ )
279
+ )
280
+ fixture_hash_record = pack_hashes.get(self.fixture_id)
281
+ if fixture_hash_record is None:
282
+ raise ValueError("fixture_id must be present in fixture pack summary")
283
+ if fixture_hash_record.sha256 != self.fixture_hash:
284
+ raise ValueError("fixture_hash must match fixture pack summary")
285
+ _validate_sha256(self.fixture_hash)
286
+ _validate_sha256(self.fixture_snapshot_hash)
287
+ _validate_sha256(self.fixture_pack_hash)
288
+ _validate_sha256(self.hidden_check_set_hash)
289
+ if not self.hidden_check_set_ids:
290
+ raise ValueError("fixture instances must include hidden-check-set IDs")
291
+ if self.hidden_check_set_hash != _hidden_check_set_hash(
292
+ self.hidden_check_set_ids
293
+ ):
294
+ raise ValueError("hidden-check-set hash must match hidden-check-set IDs")
295
+ if (
296
+ not self.physical_copy_available
297
+ and self.copy_unavailable_diagnostic is None
298
+ ):
299
+ raise ValueError("deferred fixture copying requires a typed diagnostic")
300
+ if (
301
+ self.physical_copy_available
302
+ and self.copy_unavailable_diagnostic is not None
303
+ ):
304
+ raise ValueError(
305
+ "available fixture copies must not carry unavailable diagnostics"
306
+ )
307
+ return self
308
+
309
+
310
+ class EvalTrialArmPlan(EvalTrialContractModel):
311
+ """Per-arm deterministic planning evidence for one paired trial."""
312
+
313
+ trial_id: StrictStr
314
+ trial_index: StrictInt
315
+ arm_id: EvalTrialArmId
316
+ arm_order_index: StrictInt
317
+ paired_seed: StrictInt
318
+ artifact_root: StrictStr
319
+ runner_kind: StrictStr
320
+ runner_descriptor_id: StrictStr
321
+ campaign_manifest_hash: StrictStr
322
+ model_manifest_hash: StrictStr
323
+ workflow_graph_hash: StrictStr
324
+ fixture_pack_hash: StrictStr
325
+ fixture_id: StrictStr
326
+ fixture_hash: StrictStr
327
+ visible_acceptance_criteria_hash: StrictStr
328
+ hidden_check_set_ids: tuple[StrictStr, ...]
329
+ hidden_check_set_hash: StrictStr
330
+ scorer_version: StrictStr
331
+ treatment_compiled_harness_hashes: Mapping[StrictStr, StrictStr] = Field(
332
+ default_factory=dict
333
+ )
334
+ baseline_pi_runtime_hash: StrictStr | None = None
335
+ baseline_runtime_diagnostic: EvalTrialInvalidDiagnostic | None = None
336
+
337
+ @model_validator(mode="after")
338
+ def _arm_plan_valid(self) -> EvalTrialArmPlan:
339
+ if self.trial_index < 0:
340
+ raise ValueError("trial_index must be non-negative")
341
+ if self.arm_order_index not in {0, 1}:
342
+ raise ValueError("arm_order_index must be 0 or 1")
343
+ if self.paired_seed < 0:
344
+ raise ValueError("paired_seed must be non-negative")
345
+ _validate_relative_artifact_root(self.artifact_root)
346
+ for digest in (
347
+ self.campaign_manifest_hash,
348
+ self.model_manifest_hash,
349
+ self.workflow_graph_hash,
350
+ self.fixture_pack_hash,
351
+ self.fixture_hash,
352
+ self.visible_acceptance_criteria_hash,
353
+ self.hidden_check_set_hash,
354
+ ):
355
+ _validate_sha256(digest)
356
+ if self.hidden_check_set_hash != _hidden_check_set_hash(
357
+ self.hidden_check_set_ids
358
+ ):
359
+ raise ValueError("hidden-check-set hash must match hidden-check-set IDs")
360
+ object.__setattr__(
361
+ self,
362
+ "treatment_compiled_harness_hashes",
363
+ _freeze_trial_mapping(self.treatment_compiled_harness_hashes),
364
+ )
365
+ for digest in self.treatment_compiled_harness_hashes.values():
366
+ _validate_sha256(digest)
367
+ if self.arm_id is EvalTrialArmId.EVAL_SMALL_MILLFORGE:
368
+ if self.baseline_pi_runtime_hash is not None:
369
+ raise ValueError("treatment arms must not carry Pi runtime hashes")
370
+ if self.baseline_runtime_diagnostic is not None:
371
+ raise ValueError("treatment arms must not carry Pi runtime diagnostics")
372
+ if self.arm_id is EvalTrialArmId.EVAL_SMALL_PI:
373
+ if self.treatment_compiled_harness_hashes:
374
+ raise ValueError(
375
+ "baseline arms must not carry treatment harness hashes"
376
+ )
377
+ if self.baseline_pi_runtime_hash is not None:
378
+ _validate_sha256(self.baseline_pi_runtime_hash)
379
+ _reject_fake_pi_runtime_hash(self.baseline_pi_runtime_hash)
380
+ if (
381
+ self.baseline_pi_runtime_hash is None
382
+ and self.baseline_runtime_diagnostic is None
383
+ ):
384
+ raise ValueError(
385
+ "baseline arms require a Pi runtime hash or deferred diagnostic"
386
+ )
387
+ return self
388
+
389
+
390
+ class EvalTrialFakeRunnerScript(EvalTrialContractModel):
391
+ """Offline fake runner script descriptor without live execution claims."""
392
+
393
+ script_id: StrictStr
394
+ script_kind: EvalFakeOutcomeScriptKind
395
+ terminal_results: tuple[EvalTerminalResult, ...]
396
+ expected_outcome: EvalTrialOutcome
397
+ stage_result_summaries: Mapping[EvalStageId, StrictStr] = Field(
398
+ default_factory=dict
399
+ )
400
+ zero_usage: StrictBool = True
401
+
402
+ @model_validator(mode="after")
403
+ def _script_valid(self) -> EvalTrialFakeRunnerScript:
404
+ if not self.terminal_results:
405
+ raise ValueError("fake scripts must declare terminal results")
406
+ if not self.zero_usage:
407
+ raise ValueError("offline fake scripts must preserve zero-usage semantics")
408
+ object.__setattr__(
409
+ self,
410
+ "stage_result_summaries",
411
+ _freeze_trial_mapping(self.stage_result_summaries),
412
+ )
413
+ return self
414
+
415
+
416
+ class EvalTrialPlan(EvalTrialContractModel):
417
+ """Hashable paired offline eval-trial plan."""
418
+
419
+ schema_version: StrictInt = EVAL_TRIAL_SCHEMA_VERSION
420
+ trial_id: StrictStr
421
+ trial_index: StrictInt = 0
422
+ trial_plan_id: StrictStr
423
+ paired_seed: StrictInt = 0
424
+ campaign_store_root: StrictStr
425
+ campaign_manifest: EvalCampaignManifest
426
+ arms: tuple[EvalTrialArmDefinition, EvalTrialArmDefinition]
427
+ arm_order: tuple[EvalTrialArmId, EvalTrialArmId]
428
+ arm_plans: tuple[EvalTrialArmPlan, EvalTrialArmPlan]
429
+ parity_evidence: EvalTrialArmParityEvidence
430
+ fixture_instance: EvalTrialFixtureInstance
431
+ fake_runner_script: EvalTrialFakeRunnerScript
432
+ spec_07_readiness: EvalPresetReadinessReport
433
+ created_at: StrictStr = EVAL_TRIAL_DEFAULT_CREATED_AT
434
+ plan_hash_kind: StrictStr = EVAL_TRIAL_PLAN_HASH_KIND
435
+ plan_hash: StrictStr
436
+
437
+ @model_validator(mode="after")
438
+ def _plan_valid(self) -> EvalTrialPlan:
439
+ if self.schema_version != EVAL_TRIAL_SCHEMA_VERSION:
440
+ raise ValueError("unsupported eval-trial schema_version")
441
+ if not _UTC_TIMESTAMP_RE.fullmatch(self.created_at):
442
+ raise ValueError("trial plan created_at must be a UTC timestamp")
443
+ if self.trial_index < 0:
444
+ raise ValueError("trial_index must be non-negative")
445
+ if self.paired_seed < 0:
446
+ raise ValueError("paired_seed must be non-negative")
447
+ _validate_relative_artifact_root(self.campaign_store_root)
448
+ arm_ids = tuple(arm.arm_id for arm in self.arms)
449
+ if arm_ids != (
450
+ EvalTrialArmId.EVAL_SMALL_PI,
451
+ EvalTrialArmId.EVAL_SMALL_MILLFORGE,
452
+ ):
453
+ raise ValueError("trial plans must contain exactly the admitted arms")
454
+ if set(self.arm_order) != set(arm_ids) or len(set(self.arm_order)) != 2:
455
+ raise ValueError("arm_order must contain each admitted arm exactly once")
456
+ if {plan.arm_id for plan in self.arm_plans} != set(arm_ids):
457
+ raise ValueError("arm_plans must contain each admitted arm exactly once")
458
+ if self.campaign_manifest.execution_mode != EvalSuiteExecutionMode.OFFLINE_FAKE:
459
+ raise ValueError("eval-trial plans are offline fake contracts only")
460
+ if self.campaign_manifest.live_execution_admitted:
461
+ raise ValueError("eval-trial planning does not admit live execution")
462
+ if self.campaign_manifest.pi_eval_mode_id != EvalTrialArmId.EVAL_SMALL_PI.value:
463
+ raise ValueError("campaign pi arm must match admitted pi mode")
464
+ if (
465
+ self.campaign_manifest.millforge_eval_mode_id
466
+ != EvalTrialArmId.EVAL_SMALL_MILLFORGE.value
467
+ ):
468
+ raise ValueError(
469
+ "campaign millforge arm must match admitted millforge mode"
470
+ )
471
+ left_arm, right_arm = self.arms
472
+ if self.parity_evidence.left_arm_id != left_arm.arm_id:
473
+ raise ValueError("parity left arm must match trial arms")
474
+ if self.parity_evidence.right_arm_id != right_arm.arm_id:
475
+ raise ValueError("parity right arm must match trial arms")
476
+ if (
477
+ self.parity_evidence.left_descriptor_fingerprint
478
+ != left_arm.descriptor_fingerprint
479
+ ):
480
+ raise ValueError("parity left descriptor fingerprint must match arm")
481
+ if (
482
+ self.parity_evidence.right_descriptor_fingerprint
483
+ != right_arm.descriptor_fingerprint
484
+ ):
485
+ raise ValueError("parity right descriptor fingerprint must match arm")
486
+ if (
487
+ self.parity_evidence.shared_fairness_fingerprint
488
+ not in {
489
+ left_arm.fairness_fingerprint,
490
+ right_arm.fairness_fingerprint,
491
+ }
492
+ or left_arm.fairness_fingerprint != right_arm.fairness_fingerprint
493
+ ):
494
+ raise ValueError("parity fairness fingerprint must match both arms")
495
+ if self.parity_evidence.workflow_graph_hash not in {
496
+ left_arm.descriptor.graph_sha256,
497
+ right_arm.descriptor.graph_sha256,
498
+ self.campaign_manifest.workflow_graph_hash,
499
+ } or not (
500
+ left_arm.descriptor.graph_sha256
501
+ == right_arm.descriptor.graph_sha256
502
+ == self.campaign_manifest.workflow_graph_hash
503
+ ):
504
+ raise ValueError("parity workflow graph hash must match arms and campaign")
505
+ if self.parity_evidence.model_profile_hash not in {
506
+ left_arm.descriptor.model_profile.model_profile_hash,
507
+ right_arm.descriptor.model_profile.model_profile_hash,
508
+ } or (
509
+ left_arm.descriptor.model_profile.model_profile_hash
510
+ != right_arm.descriptor.model_profile.model_profile_hash
511
+ ):
512
+ raise ValueError("parity model profile hash must match both arms")
513
+ if (
514
+ self.parity_evidence.fixture_pack_hash
515
+ != self.campaign_manifest.fixture_pack_hash
516
+ or self.parity_evidence.fixture_pack_hash
517
+ != self.fixture_instance.fixture_pack_hash
518
+ ):
519
+ raise ValueError(
520
+ "parity fixture pack hash must match campaign and fixture instance"
521
+ )
522
+ expected_treatment_hashes = _compiled_harness_hashes_by_stage(
523
+ self.spec_07_readiness
524
+ )
525
+ shared_arm_values = {
526
+ "trial_id": self.trial_id,
527
+ "trial_index": self.trial_index,
528
+ "paired_seed": self.paired_seed,
529
+ "campaign_manifest_hash": self.campaign_manifest.campaign_manifest_hash,
530
+ "model_manifest_hash": self.campaign_manifest.model_manifest_hash,
531
+ "workflow_graph_hash": self.campaign_manifest.workflow_graph_hash,
532
+ "fixture_pack_hash": self.fixture_instance.fixture_pack_hash,
533
+ "fixture_id": self.fixture_instance.fixture_id,
534
+ "fixture_hash": self.fixture_instance.fixture_hash,
535
+ "visible_acceptance_criteria_hash": _visible_acceptance_criteria_hash(
536
+ self.fixture_instance.public_projection.visible_acceptance_criteria
537
+ ),
538
+ "hidden_check_set_hash": self.fixture_instance.hidden_check_set_hash,
539
+ "scorer_version": self.campaign_manifest.scorer_version,
540
+ }
541
+ arms_by_id = {arm.arm_id: arm for arm in self.arms}
542
+ for arm_plan in self.arm_plans:
543
+ _validate_artifact_root_under_campaign(
544
+ arm_plan.artifact_root,
545
+ self.campaign_store_root,
546
+ )
547
+ for field_name, expected_value in shared_arm_values.items():
548
+ if getattr(arm_plan, field_name) != expected_value:
549
+ raise ValueError(
550
+ f"arm plan {field_name} must match paired trial inputs"
551
+ )
552
+ if (
553
+ arm_plan.hidden_check_set_ids
554
+ != self.fixture_instance.hidden_check_set_ids
555
+ ):
556
+ raise ValueError("arm plan hidden-check-set IDs must match fixture")
557
+ descriptor = arms_by_id[arm_plan.arm_id]
558
+ if arm_plan.runner_kind != descriptor.runner_kind:
559
+ raise ValueError("arm plan runner_kind must match descriptor")
560
+ if arm_plan.runner_descriptor_id != descriptor.descriptor_fingerprint:
561
+ raise ValueError("arm plan runner descriptor must match descriptor")
562
+ if arm_plan.arm_order_index != self.arm_order.index(arm_plan.arm_id):
563
+ raise ValueError("arm plan order index must match arm_order")
564
+ if arm_plan.arm_id is EvalTrialArmId.EVAL_SMALL_MILLFORGE and dict(
565
+ arm_plan.treatment_compiled_harness_hashes
566
+ ) != dict(expected_treatment_hashes):
567
+ raise ValueError(
568
+ "treatment compiled harness hashes must match Spec 07E readiness"
569
+ )
570
+ pairings = {
571
+ (plan.fixture_id, plan.arm_id.value, plan.trial_index)
572
+ for plan in self.arm_plans
573
+ }
574
+ if len(pairings) != len(self.arm_plans):
575
+ raise ValueError("duplicate fixture/arm/trial-index pairings are invalid")
576
+ if not self.parity_evidence.comparable_offline:
577
+ raise ValueError("trial plans require comparable offline parity evidence")
578
+ if self.plan_hash_kind != EVAL_TRIAL_PLAN_HASH_KIND:
579
+ raise ValueError("unsupported trial plan hash kind")
580
+ _validate_sha256(self.plan_hash)
581
+ expected = calculate_eval_trial_plan_hash(self)
582
+ if self.plan_hash != expected:
583
+ raise ValueError("plan_hash does not match trial plan payload")
584
+ return self
585
+
586
+
587
+ class EvalFakeRunnerArtifactBundle(EvalTrialContractModel):
588
+ """Offline fake-runner artifact bundle manifest for one arm attempt."""
589
+
590
+ schema_version: StrictInt = EVAL_TRIAL_SCHEMA_VERSION
591
+ artifact_bundle_id: StrictStr
592
+ trial_plan_hash: StrictStr
593
+ arm_id: EvalTrialArmId
594
+ artifact_root: StrictStr
595
+ artifact_manifest: EvalArtifactManifestArtifact
596
+ artifact_hashes: tuple[EvalHashRecord, ...] = Field(default_factory=tuple)
597
+ zero_model_usage: StrictBool = True
598
+ zero_external_usage: StrictBool = True
599
+ artifact_bundle_hash_kind: StrictStr = EVAL_TRIAL_ARTIFACT_BUNDLE_HASH_KIND
600
+ artifact_bundle_hash: StrictStr
601
+
602
+ @model_validator(mode="after")
603
+ def _bundle_valid(self) -> EvalFakeRunnerArtifactBundle:
604
+ if self.schema_version != EVAL_TRIAL_SCHEMA_VERSION:
605
+ raise ValueError("unsupported eval-trial artifact schema_version")
606
+ _validate_sha256(self.trial_plan_hash)
607
+ _validate_relative_artifact_root(self.artifact_root)
608
+ if not self.zero_model_usage or not self.zero_external_usage:
609
+ raise ValueError("offline fake bundles must preserve zero-usage semantics")
610
+ if len({record.sha256 for record in self.artifact_hashes}) != len(
611
+ self.artifact_hashes
612
+ ):
613
+ raise ValueError("artifact bundle hashes must be unique")
614
+ if self.artifact_bundle_hash_kind != EVAL_TRIAL_ARTIFACT_BUNDLE_HASH_KIND:
615
+ raise ValueError("unsupported artifact bundle hash kind")
616
+ _validate_sha256(self.artifact_bundle_hash)
617
+ expected = calculate_eval_fake_runner_artifact_bundle_hash(self)
618
+ if self.artifact_bundle_hash != expected:
619
+ raise ValueError("artifact_bundle_hash does not match payload")
620
+ return self
621
+
622
+
623
+ class EvalOfflineFakeTrialRun(EvalTrialContractModel):
624
+ """Deterministic paired fake-runner output for one planned offline trial."""
625
+
626
+ schema_version: StrictInt = EVAL_TRIAL_SCHEMA_VERSION
627
+ trial_id: StrictStr
628
+ trial_plan_hash: StrictStr
629
+ artifact_bundles: tuple[EvalFakeRunnerArtifactBundle, EvalFakeRunnerArtifactBundle]
630
+ execution_results: tuple[EvalTrialExecutionResult, EvalTrialExecutionResult]
631
+ trial_record: EvalTrialRecord
632
+ live_execution_admitted: StrictBool = False
633
+ diagnostics: tuple[EvalTrialInvalidDiagnostic, ...] = Field(default_factory=tuple)
634
+
635
+ @model_validator(mode="after")
636
+ def _run_valid(self) -> EvalOfflineFakeTrialRun:
637
+ if self.schema_version != EVAL_TRIAL_SCHEMA_VERSION:
638
+ raise ValueError("unsupported eval-trial fake-run schema_version")
639
+ _validate_sha256(self.trial_plan_hash)
640
+ if self.live_execution_admitted:
641
+ raise ValueError("offline fake trial runs do not admit live execution")
642
+ bundle_arm_ids = tuple(bundle.arm_id for bundle in self.artifact_bundles)
643
+ result_arm_ids = tuple(result.arm_id for result in self.execution_results)
644
+ expected_arm_ids = (
645
+ EvalTrialArmId.EVAL_SMALL_PI,
646
+ EvalTrialArmId.EVAL_SMALL_MILLFORGE,
647
+ )
648
+ if bundle_arm_ids != expected_arm_ids or result_arm_ids != expected_arm_ids:
649
+ raise ValueError("fake trial runs must contain exactly the admitted arms")
650
+ if self.trial_record.arm_results != self.execution_results:
651
+ raise ValueError("trial_record must mirror execution_results")
652
+ return self
653
+
654
+
655
+ class EvalTrialExecutionResult(EvalTrialContractModel):
656
+ """Scorer-facing execution result for one offline arm."""
657
+
658
+ schema_version: StrictInt = EVAL_TRIAL_SCHEMA_VERSION
659
+ trial_id: StrictStr
660
+ trial_plan_hash: StrictStr
661
+ arm_id: EvalTrialArmId
662
+ terminal_results: tuple[EvalTerminalResult, ...]
663
+ artifact_bundle_hash: StrictStr
664
+ scorer_input: EvalScorerInput
665
+ scorer_result: EvalScorerResult
666
+ live_execution_admitted: StrictBool = False
667
+ diagnostics: tuple[EvalTrialInvalidDiagnostic, ...] = Field(default_factory=tuple)
668
+
669
+ @model_validator(mode="after")
670
+ def _execution_result_valid(self) -> EvalTrialExecutionResult:
671
+ if self.schema_version != EVAL_TRIAL_SCHEMA_VERSION:
672
+ raise ValueError("unsupported eval-trial result schema_version")
673
+ _validate_sha256(self.trial_plan_hash)
674
+ _validate_sha256(self.artifact_bundle_hash)
675
+ if self.scorer_input.trial_id != self.trial_id:
676
+ raise ValueError("scorer_input trial_id must match result")
677
+ if self.scorer_result.trial_id != self.trial_id:
678
+ raise ValueError("scorer_result trial_id must match result")
679
+ if self.live_execution_admitted:
680
+ raise ValueError("eval-trial contracts do not admit live execution")
681
+ return self
682
+
683
+
684
+ class EvalTrialRunnerRecordSummary(EvalTrialContractModel):
685
+ """Public runner identity and runtime diagnostic for one arm record."""
686
+
687
+ arm_id: EvalTrialArmId
688
+ runner_kind: StrictStr
689
+ runner_descriptor_id: StrictStr
690
+ runner_descriptor_version: StrictStr
691
+ baseline_pi_runtime_hash: StrictStr | None = None
692
+ baseline_runtime_diagnostic: EvalTrialInvalidDiagnostic | None = None
693
+
694
+ @model_validator(mode="after")
695
+ def _runner_summary_valid(self) -> EvalTrialRunnerRecordSummary:
696
+ if self.arm_id is EvalTrialArmId.EVAL_SMALL_PI:
697
+ if self.baseline_pi_runtime_hash is not None:
698
+ _validate_sha256(self.baseline_pi_runtime_hash)
699
+ _reject_fake_pi_runtime_hash(self.baseline_pi_runtime_hash)
700
+ if (
701
+ self.baseline_pi_runtime_hash is None
702
+ and self.baseline_runtime_diagnostic is None
703
+ ):
704
+ raise ValueError(
705
+ "Pi baseline records require a runtime hash or deferred diagnostic"
706
+ )
707
+ else:
708
+ if self.baseline_pi_runtime_hash is not None:
709
+ raise ValueError("treatment runner summaries must not carry Pi hashes")
710
+ if self.baseline_runtime_diagnostic is not None:
711
+ raise ValueError(
712
+ "treatment runner summaries must not carry Pi diagnostics"
713
+ )
714
+ return self
715
+
716
+
717
+ class EvalTrialScorerPublicSummary(EvalTrialContractModel):
718
+ """Public scorer result digest without hidden scorer material."""
719
+
720
+ arm_id: EvalTrialArmId
721
+ scorer_version: StrictStr
722
+ final_outcome: EvalTrialOutcome
723
+ primary_success: StrictBool
724
+ false_closure: StrictBool
725
+ false_success: StrictBool
726
+ correctly_blocked: StrictBool
727
+ capability_violation: StrictBool
728
+ artifact_complete: StrictBool
729
+ missing_artifact_ids: tuple[StrictStr, ...] = Field(default_factory=tuple)
730
+ malformed_artifact_ids: tuple[StrictStr, ...] = Field(default_factory=tuple)
731
+ failure_labels: tuple[StrictStr, ...] = Field(default_factory=tuple)
732
+ public_diagnostics: tuple[StrictStr, ...] = Field(default_factory=tuple)
733
+ invalid_trial_explanation: StrictStr | None = None
734
+
735
+
736
+ class EvalTrialResourceSummary(EvalTrialContractModel):
737
+ """Public resource-use summary for deterministic offline records."""
738
+
739
+ artifact_count: StrictInt
740
+ artifact_bytes: StrictInt = 0
741
+ turn_count: StrictInt = 0
742
+ invalid_tool_call_count: StrictInt = 0
743
+ malformed_argument_count: StrictInt = 0
744
+ prerequisite_violation_count: StrictInt = 0
745
+ premature_terminal_count: StrictInt = 0
746
+ tool_recovery_count: StrictInt = 0
747
+ resource_artifact_hashes: tuple[EvalHashRecord, ...] = Field(default_factory=tuple)
748
+ zero_external_usage: StrictBool = True
749
+
750
+ @model_validator(mode="after")
751
+ def _resource_summary_valid(self) -> EvalTrialResourceSummary:
752
+ for field_name in (
753
+ "artifact_count",
754
+ "artifact_bytes",
755
+ "turn_count",
756
+ "invalid_tool_call_count",
757
+ "malformed_argument_count",
758
+ "prerequisite_violation_count",
759
+ "premature_terminal_count",
760
+ "tool_recovery_count",
761
+ ):
762
+ if getattr(self, field_name) < 0:
763
+ raise ValueError(f"{field_name} must be non-negative")
764
+ if not self.zero_external_usage:
765
+ raise ValueError("offline trial records must preserve zero external usage")
766
+ return self
767
+
768
+
769
+ class EvalTrialModelUsageSummary(EvalTrialContractModel):
770
+ """Public model-use summary for deterministic offline records."""
771
+
772
+ zero_model_usage: StrictBool = True
773
+ input_tokens: StrictInt = 0
774
+ output_tokens: StrictInt = 0
775
+ model_call_count: StrictInt = 0
776
+
777
+ @model_validator(mode="after")
778
+ def _model_usage_summary_valid(self) -> EvalTrialModelUsageSummary:
779
+ if not self.zero_model_usage:
780
+ raise ValueError("offline trial records must preserve zero model usage")
781
+ if (
782
+ self.input_tokens != 0
783
+ or self.output_tokens != 0
784
+ or self.model_call_count != 0
785
+ ):
786
+ raise ValueError("offline trial records must report zero model usage")
787
+ return self
788
+
789
+
790
+ class EvalTrialRecord(EvalTrialContractModel):
791
+ """Append-only paired trial record."""
792
+
793
+ schema_version: StrictInt = EVAL_TRIAL_SCHEMA_VERSION
794
+ campaign_id: StrictStr
795
+ task_fixture_id: StrictStr
796
+ task_category: StrictStr
797
+ trial_id: StrictStr
798
+ trial_index: StrictInt
799
+ arm: EvalTrialArmId
800
+ arm_order_index: StrictInt
801
+ arm_order: tuple[EvalTrialArmId, EvalTrialArmId]
802
+ seed_marker: StrictStr
803
+ trial_plan_hash: StrictStr
804
+ campaign_manifest_hash: StrictStr
805
+ model_manifest_hash: StrictStr
806
+ workflow_graph_hash: StrictStr
807
+ fixture_pack_hash: StrictStr
808
+ fixture_instance_id: StrictStr
809
+ fixture_id: StrictStr
810
+ fixture_hash: StrictStr
811
+ fixture_snapshot_hash: StrictStr
812
+ runner_summaries: Mapping[EvalTrialArmId, EvalTrialRunnerRecordSummary]
813
+ compiled_harness_hashes: Mapping[StrictStr, StrictStr]
814
+ arm_results: tuple[EvalTrialExecutionResult, EvalTrialExecutionResult]
815
+ final_outcomes: Mapping[EvalTrialArmId, EvalTrialOutcome]
816
+ scorer_result_hashes: Mapping[EvalTrialArmId, StrictStr]
817
+ scorer_public_summaries: Mapping[EvalTrialArmId, EvalTrialScorerPublicSummary]
818
+ artifact_manifest_hashes: Mapping[EvalTrialArmId, StrictStr]
819
+ artifact_roots: Mapping[EvalTrialArmId, StrictStr]
820
+ resource_summary: EvalTrialResourceSummary
821
+ model_usage_summary: EvalTrialModelUsageSummary
822
+ invalid_trial_explanations: Mapping[EvalTrialArmId, StrictStr] = Field(
823
+ default_factory=dict
824
+ )
825
+ started_at: StrictStr
826
+ ended_at: StrictStr
827
+ created_at: StrictStr = EVAL_TRIAL_DEFAULT_CREATED_AT
828
+ record_hash_kind: StrictStr = EVAL_TRIAL_RECORD_HASH_KIND
829
+ record_hash: StrictStr
830
+
831
+ @field_serializer("arm_results")
832
+ def _serialize_public_arm_results(
833
+ self,
834
+ value: tuple[EvalTrialExecutionResult, EvalTrialExecutionResult],
835
+ ) -> tuple[dict[str, Any], ...]:
836
+ rendered: list[dict[str, Any]] = []
837
+ for result in value:
838
+ payload = result.model_dump(mode="json")
839
+ scorer_input = dict(payload["scorer_input"])
840
+ scorer_input.pop("hidden_check_results", None)
841
+ payload["scorer_input"] = scorer_input
842
+ scorer_result = dict(payload["scorer_result"])
843
+ scorer_result.pop("scorer_only_diagnostics", None)
844
+ payload["scorer_result"] = scorer_result
845
+ rendered.append(payload)
846
+ return tuple(rendered)
847
+
848
+ @model_validator(mode="after")
849
+ def _record_valid(self) -> EvalTrialRecord:
850
+ if self.schema_version != EVAL_TRIAL_SCHEMA_VERSION:
851
+ raise ValueError("unsupported eval-trial record schema_version")
852
+ for field_name in ("created_at", "started_at", "ended_at"):
853
+ if not _UTC_TIMESTAMP_RE.fullmatch(getattr(self, field_name)):
854
+ raise ValueError(f"trial record {field_name} must be a UTC timestamp")
855
+ if self.ended_at < self.started_at:
856
+ raise ValueError("trial record ended_at must not precede started_at")
857
+ if self.task_fixture_id != self.fixture_id:
858
+ raise ValueError("task_fixture_id must match fixture_id")
859
+ if not self.fixture_instance_id:
860
+ raise ValueError("fixture_instance_id must not be empty")
861
+ if self.trial_index < 0:
862
+ raise ValueError("trial_index must be non-negative")
863
+ if self.arm_order_index not in {0, 1}:
864
+ raise ValueError("arm_order_index must be 0 or 1")
865
+ if self.arm_order_index != self.arm_order.index(self.arm):
866
+ raise ValueError("arm_order_index must match record arm")
867
+ for digest in (
868
+ self.trial_plan_hash,
869
+ self.campaign_manifest_hash,
870
+ self.model_manifest_hash,
871
+ self.workflow_graph_hash,
872
+ self.fixture_pack_hash,
873
+ self.fixture_hash,
874
+ self.fixture_snapshot_hash,
875
+ ):
876
+ _validate_sha256(digest)
877
+ result_arm_ids = tuple(result.arm_id for result in self.arm_results)
878
+ if result_arm_ids != (
879
+ EvalTrialArmId.EVAL_SMALL_PI,
880
+ EvalTrialArmId.EVAL_SMALL_MILLFORGE,
881
+ ):
882
+ raise ValueError("trial records must contain exactly the admitted arms")
883
+ expected_outcomes = {
884
+ result.arm_id: result.scorer_result.final_outcome
885
+ for result in self.arm_results
886
+ }
887
+ if dict(self.final_outcomes) != expected_outcomes:
888
+ raise ValueError("final_outcomes must mirror scorer results")
889
+ expected_hashes = {
890
+ result.arm_id: result.scorer_result.result_hash
891
+ for result in self.arm_results
892
+ }
893
+ if dict(self.scorer_result_hashes) != expected_hashes:
894
+ raise ValueError("scorer_result_hashes must mirror scorer results")
895
+ for result in self.arm_results:
896
+ if result.scorer_result.result_hash != calculate_eval_scorer_result_hash(
897
+ result.scorer_result
898
+ ):
899
+ raise ValueError("scorer result hash must match scorer payload")
900
+ expected_invalid = {
901
+ result.arm_id: result.scorer_result.invalid_trial_explanation
902
+ for result in self.arm_results
903
+ if result.scorer_result.invalid_trial_explanation
904
+ }
905
+ if dict(self.invalid_trial_explanations) != expected_invalid:
906
+ raise ValueError(
907
+ "invalid_trial_explanations must mirror public scorer explanations"
908
+ )
909
+ for arm_id, summary in self.scorer_public_summaries.items():
910
+ matching_result = next(
911
+ (
912
+ candidate
913
+ for candidate in self.arm_results
914
+ if candidate.arm_id == arm_id
915
+ ),
916
+ None,
917
+ )
918
+ if (
919
+ matching_result is None
920
+ or summary.final_outcome != matching_result.scorer_result.final_outcome
921
+ ):
922
+ raise ValueError("scorer public summaries must mirror scorer results")
923
+ expected_treatment_hashes = _compiled_harness_hashes_by_stage(
924
+ eval_preset_readiness_report()
925
+ )
926
+ if dict(self.compiled_harness_hashes) != dict(expected_treatment_hashes):
927
+ raise ValueError(
928
+ "treatment compiled harness hashes must match current Spec 07E readiness"
929
+ )
930
+ if set(self.runner_summaries) != set(result_arm_ids):
931
+ raise ValueError("runner summaries must cover all trial arms")
932
+ if set(self.artifact_manifest_hashes) != set(result_arm_ids):
933
+ raise ValueError("artifact manifest hashes must cover all trial arms")
934
+ if set(self.artifact_roots) != set(result_arm_ids):
935
+ raise ValueError("artifact roots must cover all trial arms")
936
+ for digest in self.artifact_manifest_hashes.values():
937
+ _validate_sha256(digest)
938
+ for path in self.artifact_roots.values():
939
+ _validate_relative_artifact_root(path)
940
+ object.__setattr__(
941
+ self, "final_outcomes", _freeze_trial_mapping(self.final_outcomes)
942
+ )
943
+ for field_name in (
944
+ "runner_summaries",
945
+ "compiled_harness_hashes",
946
+ "scorer_result_hashes",
947
+ "scorer_public_summaries",
948
+ "artifact_manifest_hashes",
949
+ "artifact_roots",
950
+ "invalid_trial_explanations",
951
+ ):
952
+ object.__setattr__(
953
+ self, field_name, _freeze_trial_mapping(getattr(self, field_name))
954
+ )
955
+ if self.record_hash_kind != EVAL_TRIAL_RECORD_HASH_KIND:
956
+ raise ValueError("unsupported trial record hash kind")
957
+ _validate_sha256(self.record_hash)
958
+ expected = calculate_eval_trial_record_hash(self)
959
+ if self.record_hash != expected:
960
+ raise ValueError("record_hash does not match trial record payload")
961
+ return self
962
+
963
+
964
+ class EvalTrialStoreManifest(EvalTrialContractModel):
965
+ """Append-only campaign store manifest."""
966
+
967
+ schema_version: StrictInt = EVAL_TRIAL_SCHEMA_VERSION
968
+ store_manifest_id: StrictStr
969
+ campaign_manifest_hash: StrictStr
970
+ record_hashes: tuple[StrictStr, ...] = Field(default_factory=tuple)
971
+ append_only: StrictBool = True
972
+ store_manifest_hash_kind: StrictStr = EVAL_TRIAL_STORE_MANIFEST_HASH_KIND
973
+ store_manifest_hash: StrictStr
974
+
975
+ @model_validator(mode="after")
976
+ def _store_manifest_valid(self) -> EvalTrialStoreManifest:
977
+ if self.schema_version != EVAL_TRIAL_SCHEMA_VERSION:
978
+ raise ValueError("unsupported eval-trial store schema_version")
979
+ _validate_sha256(self.campaign_manifest_hash)
980
+ for digest in self.record_hashes:
981
+ _validate_sha256(digest)
982
+ if len(set(self.record_hashes)) != len(self.record_hashes):
983
+ raise ValueError("record_hashes must be unique")
984
+ if not self.append_only:
985
+ raise ValueError("eval-trial stores are append-only")
986
+ if self.store_manifest_hash_kind != EVAL_TRIAL_STORE_MANIFEST_HASH_KIND:
987
+ raise ValueError("unsupported store manifest hash kind")
988
+ _validate_sha256(self.store_manifest_hash)
989
+ expected = calculate_eval_trial_store_manifest_hash(self)
990
+ if self.store_manifest_hash != expected:
991
+ raise ValueError("store_manifest_hash does not match payload")
992
+ return self
993
+
994
+
995
+ class EvalTrialResumeIndex(EvalTrialContractModel):
996
+ """Deterministic resume index for append-only offline campaigns."""
997
+
998
+ schema_version: StrictInt = EVAL_TRIAL_SCHEMA_VERSION
999
+ resume_index_id: StrictStr
1000
+ campaign_manifest_hash: StrictStr
1001
+ completed_trial_record_hashes: tuple[StrictStr, ...] = Field(default_factory=tuple)
1002
+ pending_trial_plan_hashes: tuple[StrictStr, ...] = Field(default_factory=tuple)
1003
+ invalid_trial_diagnostics: tuple[EvalTrialInvalidDiagnostic, ...] = Field(
1004
+ default_factory=tuple
1005
+ )
1006
+ resume_index_hash_kind: StrictStr = EVAL_TRIAL_RESUME_INDEX_HASH_KIND
1007
+ resume_index_hash: StrictStr
1008
+
1009
+ @model_validator(mode="after")
1010
+ def _resume_index_valid(self) -> EvalTrialResumeIndex:
1011
+ if self.schema_version != EVAL_TRIAL_SCHEMA_VERSION:
1012
+ raise ValueError("unsupported eval-trial resume schema_version")
1013
+ _validate_sha256(self.campaign_manifest_hash)
1014
+ for digest in (
1015
+ self.completed_trial_record_hashes + self.pending_trial_plan_hashes
1016
+ ):
1017
+ _validate_sha256(digest)
1018
+ if len(set(self.completed_trial_record_hashes)) != len(
1019
+ self.completed_trial_record_hashes
1020
+ ):
1021
+ raise ValueError("completed trial record hashes must be unique")
1022
+ if len(set(self.pending_trial_plan_hashes)) != len(
1023
+ self.pending_trial_plan_hashes
1024
+ ):
1025
+ raise ValueError("pending trial plan hashes must be unique")
1026
+ if self.resume_index_hash_kind != EVAL_TRIAL_RESUME_INDEX_HASH_KIND:
1027
+ raise ValueError("unsupported resume index hash kind")
1028
+ _validate_sha256(self.resume_index_hash)
1029
+ expected = calculate_eval_trial_resume_index_hash(self)
1030
+ if self.resume_index_hash != expected:
1031
+ raise ValueError("resume_index_hash does not match payload")
1032
+ return self
1033
+
1034
+
1035
+ class EvalTrialCampaignStoreAppendResult(EvalTrialContractModel):
1036
+ """Result from one append-only campaign-store record write."""
1037
+
1038
+ campaign_store_root: StrictStr
1039
+ manifest: EvalTrialStoreManifest
1040
+ plan: EvalTrialPlan
1041
+ appended_trial_id: StrictStr
1042
+ appended_record_hash: StrictStr
1043
+ completed_trial_ids: tuple[StrictStr, ...]
1044
+ resume_index: EvalTrialResumeIndex
1045
+
1046
+
1047
+ class EvalTrialCampaignStoreRecordSummary(EvalTrialContractModel):
1048
+ """Public record identity reconstructed from an append-only trial log."""
1049
+
1050
+ trial_id: StrictStr
1051
+ trial_plan_hash: StrictStr
1052
+ record_hash: StrictStr
1053
+
1054
+ @model_validator(mode="after")
1055
+ def _summary_valid(self) -> EvalTrialCampaignStoreRecordSummary:
1056
+ _validate_sha256(self.trial_plan_hash)
1057
+ _validate_sha256(self.record_hash)
1058
+ return self
1059
+
1060
+
1061
+ class EvalTrialCampaignStoreResumeResult(EvalTrialContractModel):
1062
+ """Non-mutating resume view plus diagnostics for one campaign store."""
1063
+
1064
+ campaign_store_root: StrictStr
1065
+ plan: EvalTrialPlan
1066
+ completed_trial_ids: tuple[StrictStr, ...]
1067
+ pending_trial_ids: tuple[StrictStr, ...]
1068
+ records: tuple[EvalTrialCampaignStoreRecordSummary, ...]
1069
+ resume_index: EvalTrialResumeIndex | None = None
1070
+ diagnostics: tuple[EvalTrialInvalidDiagnostic, ...] = Field(default_factory=tuple)
1071
+
1072
+
1073
+ def default_eval_trial_arm_definitions() -> tuple[
1074
+ EvalTrialArmDefinition, EvalTrialArmDefinition
1075
+ ]:
1076
+ """Return the two admitted offline comparison arm definitions."""
1077
+ return (
1078
+ _arm_definition(default_eval_small_pi_mode(), EvalTrialArmId.EVAL_SMALL_PI),
1079
+ _arm_definition(
1080
+ default_eval_small_millforge_mode(spec_07_static_presets_ready=True),
1081
+ EvalTrialArmId.EVAL_SMALL_MILLFORGE,
1082
+ ),
1083
+ )
1084
+
1085
+
1086
+ def default_eval_trial_parity_evidence(
1087
+ arms: tuple[EvalTrialArmDefinition, EvalTrialArmDefinition] | None = None,
1088
+ ) -> EvalTrialArmParityEvidence:
1089
+ """Return deterministic offline parity evidence for the admitted arms."""
1090
+ arms = arms or default_eval_trial_arm_definitions()
1091
+ fixture_pack_hash = load_eval_fixture_pack_summary().fixture_pack_hash
1092
+ left, right = arms
1093
+ comparable = left.fairness_fingerprint == right.fairness_fingerprint
1094
+ return EvalTrialArmParityEvidence(
1095
+ left_arm_id=left.arm_id,
1096
+ right_arm_id=right.arm_id,
1097
+ shared_fairness_fingerprint=left.fairness_fingerprint,
1098
+ left_descriptor_fingerprint=left.descriptor_fingerprint,
1099
+ right_descriptor_fingerprint=right.descriptor_fingerprint,
1100
+ workflow_graph_hash=left.descriptor.graph_sha256,
1101
+ model_profile_hash=left.descriptor.model_profile.model_profile_hash,
1102
+ fixture_pack_hash=fixture_pack_hash,
1103
+ comparable_offline=comparable,
1104
+ diagnostics=()
1105
+ if comparable
1106
+ else (
1107
+ EvalTrialInvalidDiagnostic(
1108
+ diagnostic_code=EvalTrialInvalidDiagnosticCode.ARM_PARITY_DRIFT,
1109
+ rule_id="eval_trials.arm_parity.fairness_fingerprint",
1110
+ summary="admitted arms do not share fairness-critical fingerprints",
1111
+ ),
1112
+ ),
1113
+ )
1114
+
1115
+
1116
+ def default_eval_trial_fixture_instance(
1117
+ fixture: EvalTaskFixture,
1118
+ *,
1119
+ fixture_instance_id: str | None = None,
1120
+ trial_id: str = "trial.08b.default.v1",
1121
+ trial_index: int = 0,
1122
+ paired_seed: int = 0,
1123
+ ) -> EvalTrialFixtureInstance:
1124
+ """Bind one 08A fixture to a deterministic trial fixture instance."""
1125
+ pack = load_eval_fixture_pack_summary()
1126
+ instance_id = fixture_instance_id or _fixture_instance_id(
1127
+ trial_id=trial_id,
1128
+ trial_index=trial_index,
1129
+ fixture_id=fixture.fixture_id,
1130
+ paired_seed=paired_seed,
1131
+ )
1132
+ hidden_check_ids = tuple(check.check_id for check in fixture.hidden_checks)
1133
+ return EvalTrialFixtureInstance(
1134
+ fixture_instance_id=instance_id,
1135
+ fixture_id=fixture.fixture_id,
1136
+ fixture_hash=fixture.fixture_hash,
1137
+ fixture_snapshot_hash=_fixture_snapshot_hash(
1138
+ fixture=fixture,
1139
+ fixture_instance_id=instance_id,
1140
+ paired_seed=paired_seed,
1141
+ ),
1142
+ fixture_pack_id=pack.fixture_pack_id,
1143
+ fixture_pack_hash=pack.fixture_pack_hash,
1144
+ hidden_check_set_ids=hidden_check_ids,
1145
+ hidden_check_set_hash=_hidden_check_set_hash(hidden_check_ids),
1146
+ public_projection=eval_public_artifact_projection(fixture),
1147
+ fixture_pack_summary=pack,
1148
+ physical_copy_available=False,
1149
+ copy_unavailable_diagnostic=fixture_copy_unavailable_diagnostic(),
1150
+ )
1151
+
1152
+
1153
+ def default_eval_trial_plan(
1154
+ *,
1155
+ trial_plan_id: str,
1156
+ fixture: EvalTaskFixture,
1157
+ fake_runner_script: EvalTrialFakeRunnerScript,
1158
+ trial_id: str | None = None,
1159
+ trial_index: int = 0,
1160
+ paired_seed: int = 0,
1161
+ arm_order: tuple[EvalTrialArmId, EvalTrialArmId] | None = None,
1162
+ campaign_store_root: str | None = None,
1163
+ artifact_roots_by_arm: Mapping[EvalTrialArmId | str, str] | None = None,
1164
+ campaign_manifest: EvalCampaignManifest | None = None,
1165
+ created_at: str = EVAL_TRIAL_DEFAULT_CREATED_AT,
1166
+ ) -> EvalTrialPlan:
1167
+ """Build a hash-finalized offline paired trial plan."""
1168
+ campaign_manifest = campaign_manifest or default_eval_suite_campaign_manifest()
1169
+ trial_id = trial_id or trial_plan_id.removesuffix(".plan")
1170
+ campaign_store_root = campaign_store_root or _default_campaign_store_root(
1171
+ campaign_manifest
1172
+ )
1173
+ arm_order = arm_order or (
1174
+ EvalTrialArmId.EVAL_SMALL_PI,
1175
+ EvalTrialArmId.EVAL_SMALL_MILLFORGE,
1176
+ )
1177
+ arms = default_eval_trial_arm_definitions()
1178
+ readiness = eval_preset_readiness_report()
1179
+ fixture_instance = default_eval_trial_fixture_instance(
1180
+ fixture,
1181
+ trial_id=trial_id,
1182
+ trial_index=trial_index,
1183
+ paired_seed=paired_seed,
1184
+ )
1185
+ plan = EvalTrialPlan.model_construct(
1186
+ schema_version=EVAL_TRIAL_SCHEMA_VERSION,
1187
+ trial_id=trial_id,
1188
+ trial_index=trial_index,
1189
+ trial_plan_id=trial_plan_id,
1190
+ paired_seed=paired_seed,
1191
+ campaign_store_root=campaign_store_root,
1192
+ campaign_manifest=campaign_manifest,
1193
+ arms=arms,
1194
+ arm_order=arm_order,
1195
+ arm_plans=_default_eval_trial_arm_plans(
1196
+ trial_id=trial_id,
1197
+ trial_index=trial_index,
1198
+ paired_seed=paired_seed,
1199
+ campaign_store_root=campaign_store_root,
1200
+ campaign_manifest=campaign_manifest,
1201
+ arms=arms,
1202
+ arm_order=arm_order,
1203
+ fixture=fixture,
1204
+ readiness=readiness,
1205
+ artifact_roots_by_arm=artifact_roots_by_arm,
1206
+ ),
1207
+ parity_evidence=default_eval_trial_parity_evidence(arms),
1208
+ fixture_instance=fixture_instance,
1209
+ fake_runner_script=fake_runner_script,
1210
+ spec_07_readiness=readiness,
1211
+ created_at=created_at,
1212
+ plan_hash_kind=EVAL_TRIAL_PLAN_HASH_KIND,
1213
+ plan_hash="0" * 64,
1214
+ )
1215
+ return EvalTrialPlan.model_validate(
1216
+ plan.model_copy(update={"plan_hash": calculate_eval_trial_plan_hash(plan)})
1217
+ )
1218
+
1219
+
1220
+ def plan_paired_eval_trials(
1221
+ *,
1222
+ fixtures: tuple[EvalTaskFixture, ...] | None = None,
1223
+ fake_runner_script: EvalTrialFakeRunnerScript,
1224
+ seed: int = 0,
1225
+ campaign_manifest: EvalCampaignManifest | None = None,
1226
+ arms: tuple[EvalTrialArmDefinition, EvalTrialArmDefinition] | None = None,
1227
+ trial_indexes: tuple[int, ...] | None = None,
1228
+ trial_ids: tuple[str, ...] | None = None,
1229
+ campaign_store_root: str | None = None,
1230
+ artifact_roots_by_trial_arm: Mapping[tuple[str, EvalTrialArmId | str], str]
1231
+ | None = None,
1232
+ created_at: str = EVAL_TRIAL_DEFAULT_CREATED_AT,
1233
+ ) -> tuple[EvalTrialPlan, ...]:
1234
+ """Generate deterministic paired Pi-vs-Millforge trial plans."""
1235
+ campaign_manifest = campaign_manifest or default_eval_suite_campaign_manifest()
1236
+ if campaign_manifest.execution_mode != EvalSuiteExecutionMode.OFFLINE_FAKE:
1237
+ raise ValueError("eval-trial planning rejects live execution requests")
1238
+ if campaign_manifest.live_execution_admitted:
1239
+ raise ValueError("eval-trial planning rejects live execution requests")
1240
+ arms = arms or default_eval_trial_arm_definitions()
1241
+ _validate_admitted_planning_arms(arms)
1242
+ fixtures = fixtures or load_eval_task_fixtures()
1243
+ if not fixtures:
1244
+ raise ValueError("paired trial planning requires at least one fixture")
1245
+ if any(not fixture.fixture_hash for fixture in fixtures):
1246
+ raise ValueError("paired trial planning rejects missing fixture hashes")
1247
+ if trial_indexes is None:
1248
+ trial_indexes = tuple(range(len(fixtures)))
1249
+ if len(trial_indexes) != len(fixtures):
1250
+ raise ValueError("trial_indexes must match fixtures")
1251
+ if len(set(trial_indexes)) != len(trial_indexes):
1252
+ raise ValueError("duplicate fixture/arm/trial-index pairings are invalid")
1253
+ campaign_store_root = campaign_store_root or _default_campaign_store_root(
1254
+ campaign_manifest
1255
+ )
1256
+ _validate_relative_artifact_root(campaign_store_root)
1257
+
1258
+ planned_trial_ids = trial_ids or tuple(
1259
+ _trial_id(
1260
+ campaign_id=campaign_manifest.campaign_id,
1261
+ fixture_id=fixture.fixture_id,
1262
+ trial_index=trial_index,
1263
+ seed=seed,
1264
+ )
1265
+ for fixture, trial_index in zip(fixtures, trial_indexes)
1266
+ )
1267
+ if len(planned_trial_ids) != len(fixtures):
1268
+ raise ValueError("trial_ids must match fixtures")
1269
+ if len(set(planned_trial_ids)) != len(planned_trial_ids):
1270
+ raise ValueError("duplicate trial IDs are invalid")
1271
+
1272
+ plans: list[EvalTrialPlan] = []
1273
+ seen_pairings: set[tuple[str, str, int]] = set()
1274
+ for fixture, trial_index, planned_trial_id in zip(
1275
+ fixtures, trial_indexes, planned_trial_ids
1276
+ ):
1277
+ paired_seed = _paired_seed(
1278
+ seed=seed,
1279
+ fixture_id=fixture.fixture_id,
1280
+ trial_index=trial_index,
1281
+ )
1282
+ arm_order = _randomized_arm_order(paired_seed)
1283
+ per_trial_roots = {
1284
+ arm_id: root
1285
+ for (root_trial_id, arm_id), root in (
1286
+ artifact_roots_by_trial_arm or {}
1287
+ ).items()
1288
+ if root_trial_id == planned_trial_id
1289
+ }
1290
+ for arm_id in arm_order:
1291
+ pairing = (fixture.fixture_id, arm_id.value, trial_index)
1292
+ if pairing in seen_pairings:
1293
+ raise ValueError(
1294
+ "duplicate fixture/arm/trial-index pairings are invalid"
1295
+ )
1296
+ seen_pairings.add(pairing)
1297
+ plans.append(
1298
+ default_eval_trial_plan(
1299
+ trial_plan_id=f"{planned_trial_id}.plan",
1300
+ trial_id=planned_trial_id,
1301
+ trial_index=trial_index,
1302
+ paired_seed=paired_seed,
1303
+ arm_order=arm_order,
1304
+ campaign_store_root=campaign_store_root,
1305
+ artifact_roots_by_arm=per_trial_roots,
1306
+ fixture=fixture,
1307
+ fake_runner_script=fake_runner_script,
1308
+ campaign_manifest=campaign_manifest,
1309
+ created_at=created_at,
1310
+ )
1311
+ )
1312
+ return tuple(plans)
1313
+
1314
+
1315
+ def deny_eval_trial_live_execution(
1316
+ summary: str | None = None,
1317
+ ) -> EvalTrialInvalidDiagnostic:
1318
+ """Return the stable fail-closed diagnostic for live trial execution attempts."""
1319
+ return EvalTrialInvalidDiagnostic(
1320
+ diagnostic_code=EvalTrialInvalidDiagnosticCode.LIVE_EXECUTION_UNAVAILABLE,
1321
+ rule_id="eval_trials.live_execution.unavailable",
1322
+ summary=summary
1323
+ or "08B eval-trial contracts are offline-only and do not admit live execution.",
1324
+ )
1325
+
1326
+
1327
+ def fixture_copy_unavailable_diagnostic(
1328
+ summary: str | None = None,
1329
+ ) -> EvalTrialInvalidDiagnostic:
1330
+ """Return the typed diagnostic for deferred physical fixture copying."""
1331
+ return EvalTrialInvalidDiagnostic(
1332
+ diagnostic_code=EvalTrialInvalidDiagnosticCode.FIXTURE_COPY_UNAVAILABLE,
1333
+ rule_id="eval_trials.fixture_copy.unavailable",
1334
+ summary=summary
1335
+ or (
1336
+ "Physical fixture copying is deferred; planning records a fresh "
1337
+ "fixture instance identity and snapshot hash without admitting live execution."
1338
+ ),
1339
+ )
1340
+
1341
+
1342
+ def run_offline_fake_eval_trial(
1343
+ plan: EvalTrialPlan,
1344
+ *,
1345
+ fixture: EvalTaskFixture | None = None,
1346
+ execution_mode: EvalSuiteExecutionMode = EvalSuiteExecutionMode.OFFLINE_FAKE,
1347
+ live_execution_admitted: bool = False,
1348
+ allow_live_model_call: bool = False,
1349
+ allow_pi_execution: bool = False,
1350
+ allow_millforge_harness_execution: bool = False,
1351
+ ) -> EvalOfflineFakeTrialRun:
1352
+ """Execute one planned trial through the deterministic offline fake runner."""
1353
+ diagnostics = _offline_fake_runner_live_denials(
1354
+ execution_mode=execution_mode,
1355
+ live_execution_admitted=live_execution_admitted,
1356
+ allow_live_model_call=allow_live_model_call,
1357
+ allow_pi_execution=allow_pi_execution,
1358
+ allow_millforge_harness_execution=allow_millforge_harness_execution,
1359
+ )
1360
+ if diagnostics:
1361
+ raise ValueError(diagnostics[0].summary)
1362
+ if plan.campaign_manifest.execution_mode != EvalSuiteExecutionMode.OFFLINE_FAKE:
1363
+ raise ValueError("offline fake runner accepts only offline fake trial plans")
1364
+ if plan.campaign_manifest.live_execution_admitted:
1365
+ raise ValueError("offline fake runner rejects live execution admission")
1366
+
1367
+ fixture = fixture or load_eval_task_fixture(plan.fixture_instance.fixture_id)
1368
+ if fixture.fixture_id != plan.fixture_instance.fixture_id:
1369
+ raise ValueError("offline fake runner fixture_id must match the planned trial")
1370
+ if fixture.fixture_hash != plan.fixture_instance.fixture_hash:
1371
+ raise ValueError("offline fake runner fixture_hash must match the trial plan")
1372
+
1373
+ bundles_by_arm: dict[EvalTrialArmId, EvalFakeRunnerArtifactBundle] = {}
1374
+ results_by_arm: dict[EvalTrialArmId, EvalTrialExecutionResult] = {}
1375
+ for arm_plan in plan.arm_plans:
1376
+ bundle = _fake_runner_artifact_bundle(plan=plan, arm_plan=arm_plan)
1377
+ scorer_input = _fake_runner_scorer_input(
1378
+ plan=plan,
1379
+ fixture=fixture,
1380
+ arm_id=arm_plan.arm_id,
1381
+ artifact_hashes=bundle.artifact_hashes,
1382
+ )
1383
+ scorer_result = score_eval_trial(fixture, scorer_input)
1384
+ _validate_fake_script_outcome(plan.fake_runner_script, scorer_result)
1385
+ result = EvalTrialExecutionResult(
1386
+ trial_id=plan.trial_id,
1387
+ trial_plan_hash=plan.plan_hash,
1388
+ arm_id=arm_plan.arm_id,
1389
+ terminal_results=plan.fake_runner_script.terminal_results,
1390
+ artifact_bundle_hash=bundle.artifact_bundle_hash,
1391
+ scorer_input=scorer_input,
1392
+ scorer_result=scorer_result,
1393
+ live_execution_admitted=False,
1394
+ diagnostics=(),
1395
+ )
1396
+ bundles_by_arm[arm_plan.arm_id] = bundle
1397
+ results_by_arm[arm_plan.arm_id] = result
1398
+
1399
+ ordered_bundles = (
1400
+ bundles_by_arm[EvalTrialArmId.EVAL_SMALL_PI],
1401
+ bundles_by_arm[EvalTrialArmId.EVAL_SMALL_MILLFORGE],
1402
+ )
1403
+ ordered_results = (
1404
+ results_by_arm[EvalTrialArmId.EVAL_SMALL_PI],
1405
+ results_by_arm[EvalTrialArmId.EVAL_SMALL_MILLFORGE],
1406
+ )
1407
+ record = EvalTrialRecord.model_construct(
1408
+ schema_version=EVAL_TRIAL_SCHEMA_VERSION,
1409
+ campaign_id=plan.campaign_manifest.campaign_id,
1410
+ task_fixture_id=plan.fixture_instance.fixture_id,
1411
+ task_category=fixture.category.value,
1412
+ trial_id=plan.trial_id,
1413
+ trial_index=plan.trial_index,
1414
+ arm=plan.arm_order[0],
1415
+ arm_order_index=0,
1416
+ arm_order=plan.arm_order,
1417
+ seed_marker=str(plan.paired_seed),
1418
+ trial_plan_hash=plan.plan_hash,
1419
+ campaign_manifest_hash=plan.campaign_manifest.campaign_manifest_hash,
1420
+ model_manifest_hash=plan.campaign_manifest.model_manifest_hash,
1421
+ workflow_graph_hash=plan.campaign_manifest.workflow_graph_hash,
1422
+ fixture_pack_hash=plan.fixture_instance.fixture_pack_hash,
1423
+ fixture_instance_id=plan.fixture_instance.fixture_instance_id,
1424
+ fixture_id=plan.fixture_instance.fixture_id,
1425
+ fixture_hash=plan.fixture_instance.fixture_hash,
1426
+ fixture_snapshot_hash=plan.fixture_instance.fixture_snapshot_hash,
1427
+ runner_summaries=_trial_runner_summaries(plan),
1428
+ compiled_harness_hashes=_compiled_harness_hashes_by_stage(
1429
+ plan.spec_07_readiness
1430
+ ),
1431
+ arm_results=ordered_results,
1432
+ final_outcomes={
1433
+ result.arm_id: result.scorer_result.final_outcome
1434
+ for result in ordered_results
1435
+ },
1436
+ scorer_result_hashes={
1437
+ result.arm_id: result.scorer_result.result_hash
1438
+ for result in ordered_results
1439
+ },
1440
+ scorer_public_summaries=_trial_scorer_public_summaries(ordered_results),
1441
+ artifact_manifest_hashes={
1442
+ bundle.arm_id: calculate_eval_artifact_manifest_sha256(
1443
+ bundle.artifact_manifest
1444
+ )
1445
+ for bundle in ordered_bundles
1446
+ },
1447
+ artifact_roots={
1448
+ bundle.arm_id: bundle.artifact_root for bundle in ordered_bundles
1449
+ },
1450
+ resource_summary=_trial_resource_summary(ordered_bundles),
1451
+ model_usage_summary=EvalTrialModelUsageSummary(),
1452
+ invalid_trial_explanations={
1453
+ result.arm_id: result.scorer_result.invalid_trial_explanation
1454
+ for result in ordered_results
1455
+ if result.scorer_result.invalid_trial_explanation
1456
+ },
1457
+ started_at=plan.created_at,
1458
+ ended_at=plan.created_at,
1459
+ created_at=EVAL_TRIAL_DEFAULT_CREATED_AT,
1460
+ record_hash_kind=EVAL_TRIAL_RECORD_HASH_KIND,
1461
+ record_hash="0" * 64,
1462
+ )
1463
+ record = EvalTrialRecord.model_validate(
1464
+ record.model_copy(
1465
+ update={"record_hash": calculate_eval_trial_record_hash(record)}
1466
+ )
1467
+ )
1468
+ return EvalOfflineFakeTrialRun(
1469
+ trial_id=plan.trial_id,
1470
+ trial_plan_hash=plan.plan_hash,
1471
+ artifact_bundles=ordered_bundles,
1472
+ execution_results=ordered_results,
1473
+ trial_record=record,
1474
+ live_execution_admitted=False,
1475
+ diagnostics=(),
1476
+ )
1477
+
1478
+
1479
+ def append_eval_trial_record_to_campaign_store(
1480
+ store_root: str | Path,
1481
+ *,
1482
+ plan: EvalTrialPlan,
1483
+ record: EvalTrialRecord,
1484
+ plans: tuple[EvalTrialPlan, ...] | None = None,
1485
+ ) -> EvalTrialCampaignStoreAppendResult:
1486
+ """Append one complete newline-terminated record to a caller-selected store."""
1487
+ _validate_store_record_matches_plan(plan=plan, record=record)
1488
+ plan_set = _campaign_store_plan_set(plan=plan, plans=plans)
1489
+ campaign_dir = _campaign_store_dir(store_root, plan)
1490
+ manifest_path = campaign_dir / "manifest.json"
1491
+ plan_path = campaign_dir / "plan.json"
1492
+ trials_path = campaign_dir / "trials.jsonl"
1493
+
1494
+ manifest = _campaign_store_manifest(plan=plan, record_hashes=())
1495
+ manifest_bytes = canonical_eval_trial_store_manifest_bytes(manifest)
1496
+ plan_bytes = _canonical_campaign_store_plan_set_bytes(plan_set)
1497
+
1498
+ if manifest_path.exists() and manifest_path.read_bytes() != manifest_bytes:
1499
+ raise ValueError("campaign manifest mismatch; refusing to overwrite")
1500
+ if plan_path.exists() and plan_path.read_bytes() != plan_bytes:
1501
+ raise ValueError("campaign plan mismatch; refusing to overwrite")
1502
+
1503
+ existing_records, diagnostics = _read_campaign_trial_log(trials_path)
1504
+ if diagnostics:
1505
+ raise ValueError(diagnostics[0].summary)
1506
+ if record.trial_id in {existing.trial_id for existing in existing_records}:
1507
+ raise ValueError("duplicate trial IDs are rejected by append-only stores")
1508
+
1509
+ campaign_dir.mkdir(parents=True, exist_ok=True)
1510
+ (campaign_dir / "artifacts").mkdir(exist_ok=True)
1511
+ if not manifest_path.exists():
1512
+ manifest_path.write_bytes(manifest_bytes)
1513
+ if not plan_path.exists():
1514
+ plan_path.write_bytes(plan_bytes)
1515
+ with trials_path.open("ab") as handle:
1516
+ handle.write(canonical_eval_trial_record_bytes(record))
1517
+
1518
+ records = existing_records + (_record_summary(record),)
1519
+ index = _campaign_resume_index(plan=plan, records=records, plans=plan_set)
1520
+ return EvalTrialCampaignStoreAppendResult(
1521
+ campaign_store_root=plan.campaign_store_root,
1522
+ manifest=manifest,
1523
+ plan=plan,
1524
+ appended_trial_id=record.trial_id,
1525
+ appended_record_hash=record.record_hash,
1526
+ completed_trial_ids=tuple(item.trial_id for item in records),
1527
+ resume_index=index,
1528
+ )
1529
+
1530
+
1531
+ def resume_eval_trial_campaign_store(
1532
+ store_root: str | Path,
1533
+ *,
1534
+ plan: EvalTrialPlan,
1535
+ plans: tuple[EvalTrialPlan, ...] | None = None,
1536
+ ) -> EvalTrialCampaignStoreResumeResult:
1537
+ """Reconstruct append-only campaign progress without mutating trial records."""
1538
+ plan_set = _campaign_store_plan_set(plan=plan, plans=plans)
1539
+ campaign_dir = _campaign_store_dir(store_root, plan)
1540
+ manifest_path = campaign_dir / "manifest.json"
1541
+ plan_path = campaign_dir / "plan.json"
1542
+ trials_path = campaign_dir / "trials.jsonl"
1543
+ index_path = campaign_dir / "index.json"
1544
+
1545
+ diagnostics: list[EvalTrialInvalidDiagnostic] = []
1546
+ manifest = _campaign_store_manifest(plan=plan, record_hashes=())
1547
+ if manifest_path.exists() and manifest_path.read_bytes() != (
1548
+ canonical_eval_trial_store_manifest_bytes(manifest)
1549
+ ):
1550
+ diagnostics.append(
1551
+ _store_diagnostic(
1552
+ EvalTrialInvalidDiagnosticCode.STORE_MANIFEST_HASH_MISMATCH,
1553
+ "eval_trials.store.manifest_mismatch",
1554
+ "campaign manifest does not match the expected immutable manifest",
1555
+ )
1556
+ )
1557
+ if plan_path.exists() and plan_path.read_bytes() != (
1558
+ _canonical_campaign_store_plan_set_bytes(plan_set)
1559
+ ):
1560
+ diagnostics.append(
1561
+ _store_diagnostic(
1562
+ EvalTrialInvalidDiagnosticCode.PLAN_HASH_MISMATCH,
1563
+ "eval_trials.store.plan_mismatch",
1564
+ "campaign plan does not match the expected immutable plan",
1565
+ )
1566
+ )
1567
+ records, log_diagnostics = _read_campaign_trial_log(trials_path)
1568
+ diagnostics.extend(log_diagnostics)
1569
+ duplicate_ids = _duplicate_trial_ids(records)
1570
+ diagnostics.extend(
1571
+ _store_diagnostic(
1572
+ EvalTrialInvalidDiagnosticCode.TRIAL_LOG_DUPLICATE_TRIAL_ID,
1573
+ "eval_trials.store.duplicate_trial_id",
1574
+ f"duplicate trial ID in append-only log: {trial_id}",
1575
+ )
1576
+ for trial_id in duplicate_ids
1577
+ )
1578
+
1579
+ index: EvalTrialResumeIndex | None = None
1580
+ if not diagnostics:
1581
+ campaign_dir.mkdir(parents=True, exist_ok=True)
1582
+ (campaign_dir / "artifacts").mkdir(exist_ok=True)
1583
+ if not manifest_path.exists():
1584
+ manifest_path.write_bytes(
1585
+ canonical_eval_trial_store_manifest_bytes(manifest)
1586
+ )
1587
+ if not plan_path.exists():
1588
+ plan_path.write_bytes(_canonical_campaign_store_plan_set_bytes(plan_set))
1589
+ index = _campaign_resume_index(plan=plan, records=records, plans=plan_set)
1590
+ index_bytes = canonical_eval_trial_resume_index_bytes(index)
1591
+ if index_path.exists():
1592
+ if index_path.read_bytes() != index_bytes:
1593
+ diagnostics.append(
1594
+ _store_diagnostic(
1595
+ EvalTrialInvalidDiagnosticCode.RESUME_INDEX_HASH_MISMATCH,
1596
+ "eval_trials.store.resume_index_mismatch",
1597
+ "campaign resume index does not match reconstructed records",
1598
+ )
1599
+ )
1600
+ index = None
1601
+ else:
1602
+ index_path.write_bytes(index_bytes)
1603
+
1604
+ completed = tuple(record.trial_id for record in records)
1605
+ pending = tuple(
1606
+ item.trial_id for item in plan_set if item.trial_id not in completed
1607
+ )
1608
+ return EvalTrialCampaignStoreResumeResult(
1609
+ campaign_store_root=plan.campaign_store_root,
1610
+ plan=plan,
1611
+ completed_trial_ids=completed,
1612
+ pending_trial_ids=pending,
1613
+ records=records,
1614
+ resume_index=index,
1615
+ diagnostics=tuple(diagnostics),
1616
+ )
1617
+
1618
+
1619
+ def canonical_eval_trial_plan_bytes(plan: EvalTrialPlan) -> bytes:
1620
+ """Return canonical ASCII JSON bytes for a trial plan."""
1621
+ return _canonical_eval_trial_bytes(plan.model_dump(mode="json"))
1622
+
1623
+
1624
+ def canonical_eval_fake_runner_artifact_bundle_bytes(
1625
+ bundle: EvalFakeRunnerArtifactBundle,
1626
+ ) -> bytes:
1627
+ """Return canonical ASCII JSON bytes for an artifact bundle."""
1628
+ return _canonical_eval_trial_bytes(bundle.model_dump(mode="json"))
1629
+
1630
+
1631
+ def canonical_eval_trial_record_bytes(record: EvalTrialRecord) -> bytes:
1632
+ """Return canonical ASCII JSON bytes for a trial record."""
1633
+ return _canonical_eval_trial_bytes(record.model_dump(mode="json"))
1634
+
1635
+
1636
+ def canonical_eval_trial_store_manifest_bytes(
1637
+ manifest: EvalTrialStoreManifest,
1638
+ ) -> bytes:
1639
+ """Return canonical ASCII JSON bytes for a store manifest."""
1640
+ return _canonical_eval_trial_bytes(manifest.model_dump(mode="json"))
1641
+
1642
+
1643
+ def canonical_eval_trial_resume_index_bytes(index: EvalTrialResumeIndex) -> bytes:
1644
+ """Return canonical ASCII JSON bytes for a resume index."""
1645
+ return _canonical_eval_trial_bytes(index.model_dump(mode="json"))
1646
+
1647
+
1648
+ def calculate_eval_trial_plan_hash(plan: EvalTrialPlan) -> str:
1649
+ payload = plan.model_dump(mode="json")
1650
+ payload.pop("plan_hash", None)
1651
+ return hashlib.sha256(_canonical_eval_trial_bytes(payload)).hexdigest()
1652
+
1653
+
1654
+ def calculate_eval_fake_runner_artifact_bundle_hash(
1655
+ bundle: EvalFakeRunnerArtifactBundle,
1656
+ ) -> str:
1657
+ payload = bundle.model_dump(mode="json")
1658
+ payload.pop("artifact_bundle_hash", None)
1659
+ return hashlib.sha256(_canonical_eval_trial_bytes(payload)).hexdigest()
1660
+
1661
+
1662
+ def calculate_eval_trial_record_hash(record: EvalTrialRecord) -> str:
1663
+ payload = record.model_dump(mode="json")
1664
+ payload.pop("record_hash", None)
1665
+ return hashlib.sha256(_canonical_eval_trial_bytes(payload)).hexdigest()
1666
+
1667
+
1668
+ def calculate_eval_trial_store_manifest_hash(
1669
+ manifest: EvalTrialStoreManifest,
1670
+ ) -> str:
1671
+ payload = manifest.model_dump(mode="json")
1672
+ payload.pop("store_manifest_hash", None)
1673
+ return hashlib.sha256(_canonical_eval_trial_bytes(payload)).hexdigest()
1674
+
1675
+
1676
+ def calculate_eval_trial_resume_index_hash(index: EvalTrialResumeIndex) -> str:
1677
+ payload = index.model_dump(mode="json")
1678
+ payload.pop("resume_index_hash", None)
1679
+ return hashlib.sha256(_canonical_eval_trial_bytes(payload)).hexdigest()
1680
+
1681
+
1682
+ def _campaign_store_plan_set(
1683
+ *,
1684
+ plan: EvalTrialPlan,
1685
+ plans: tuple[EvalTrialPlan, ...] | None,
1686
+ ) -> tuple[EvalTrialPlan, ...]:
1687
+ plan_set = plans or (plan,)
1688
+ if not plan_set:
1689
+ raise ValueError("campaign plan set must include at least one trial plan")
1690
+ _validate_campaign_store_root(plan)
1691
+ if not any(
1692
+ item.trial_id == plan.trial_id and item.plan_hash == plan.plan_hash
1693
+ for item in plan_set
1694
+ ):
1695
+ raise ValueError("campaign plan set must include the active trial plan")
1696
+ campaign_manifest_hash = plan.campaign_manifest.campaign_manifest_hash
1697
+ campaign_store_root = plan.campaign_store_root
1698
+ trial_ids = tuple(item.trial_id for item in plan_set)
1699
+ plan_hashes = tuple(item.plan_hash for item in plan_set)
1700
+ if len(set(trial_ids)) != len(trial_ids):
1701
+ raise ValueError("campaign plan set rejects duplicate trial IDs")
1702
+ if len(set(plan_hashes)) != len(plan_hashes):
1703
+ raise ValueError("campaign plan set rejects duplicate plan hashes")
1704
+ for item in plan_set:
1705
+ _validate_campaign_store_root(item)
1706
+ if item.campaign_store_root != campaign_store_root:
1707
+ raise ValueError("campaign plan set must share one campaign store root")
1708
+ if item.campaign_manifest.campaign_manifest_hash != campaign_manifest_hash:
1709
+ raise ValueError("campaign plan set must share one campaign manifest")
1710
+ return tuple(
1711
+ sorted(
1712
+ plan_set,
1713
+ key=lambda item: (item.trial_index, item.trial_id, item.plan_hash),
1714
+ )
1715
+ )
1716
+
1717
+
1718
+ def _canonical_campaign_store_plan_set_bytes(
1719
+ plans: tuple[EvalTrialPlan, ...],
1720
+ ) -> bytes:
1721
+ first = plans[0]
1722
+ return _canonical_eval_trial_bytes(
1723
+ {
1724
+ "schema_version": EVAL_TRIAL_SCHEMA_VERSION,
1725
+ "campaign_manifest_hash": (first.campaign_manifest.campaign_manifest_hash),
1726
+ "campaign_store_root": first.campaign_store_root,
1727
+ "plan_hashes": tuple(item.plan_hash for item in plans),
1728
+ "plans": tuple(item.model_dump(mode="json") for item in plans),
1729
+ }
1730
+ )
1731
+
1732
+
1733
+ def _campaign_store_dir(store_root: str | Path, plan: EvalTrialPlan) -> Path:
1734
+ _validate_campaign_store_root(plan)
1735
+ return Path(store_root) / PurePosixPath(plan.campaign_store_root)
1736
+
1737
+
1738
+ def _validate_campaign_store_root(plan: EvalTrialPlan) -> None:
1739
+ _validate_relative_artifact_root(plan.campaign_store_root)
1740
+ parts = PurePosixPath(plan.campaign_store_root).parts
1741
+ if parts != ("eval", "campaigns", plan.campaign_manifest.campaign_id):
1742
+ raise ValueError("campaign store root must be eval/campaigns/<campaign-id>")
1743
+
1744
+
1745
+ def _validate_store_record_matches_plan(
1746
+ *,
1747
+ plan: EvalTrialPlan,
1748
+ record: EvalTrialRecord,
1749
+ ) -> None:
1750
+ _validate_campaign_store_root(plan)
1751
+ if record.campaign_id != plan.campaign_manifest.campaign_id:
1752
+ raise ValueError("trial record campaign_id must match campaign plan")
1753
+ if record.trial_id != plan.trial_id:
1754
+ raise ValueError("trial record trial_id must match campaign plan")
1755
+ if record.trial_plan_hash != plan.plan_hash:
1756
+ raise ValueError("trial record plan hash must match campaign plan")
1757
+ if record.campaign_manifest_hash != plan.campaign_manifest.campaign_manifest_hash:
1758
+ raise ValueError("trial record campaign manifest hash must match plan")
1759
+ expected_record_values = {
1760
+ "model_manifest_hash": plan.campaign_manifest.model_manifest_hash,
1761
+ "workflow_graph_hash": plan.campaign_manifest.workflow_graph_hash,
1762
+ "fixture_pack_hash": plan.fixture_instance.fixture_pack_hash,
1763
+ "fixture_instance_id": plan.fixture_instance.fixture_instance_id,
1764
+ "fixture_id": plan.fixture_instance.fixture_id,
1765
+ "fixture_hash": plan.fixture_instance.fixture_hash,
1766
+ "fixture_snapshot_hash": plan.fixture_instance.fixture_snapshot_hash,
1767
+ "started_at": plan.created_at,
1768
+ "ended_at": plan.created_at,
1769
+ }
1770
+ for field_name, expected_value in expected_record_values.items():
1771
+ if getattr(record, field_name) != expected_value:
1772
+ raise ValueError(f"trial record {field_name} must match campaign plan")
1773
+ for artifact_root in record.artifact_roots.values():
1774
+ _validate_artifact_root_under_campaign(
1775
+ artifact_root,
1776
+ plan.campaign_store_root,
1777
+ )
1778
+
1779
+
1780
+ def _campaign_store_manifest(
1781
+ *,
1782
+ plan: EvalTrialPlan,
1783
+ record_hashes: tuple[str, ...],
1784
+ ) -> EvalTrialStoreManifest:
1785
+ manifest = EvalTrialStoreManifest.model_construct(
1786
+ schema_version=EVAL_TRIAL_SCHEMA_VERSION,
1787
+ store_manifest_id=f"{plan.campaign_manifest.campaign_id}.store.v1",
1788
+ campaign_manifest_hash=plan.campaign_manifest.campaign_manifest_hash,
1789
+ record_hashes=record_hashes,
1790
+ append_only=True,
1791
+ store_manifest_hash_kind=EVAL_TRIAL_STORE_MANIFEST_HASH_KIND,
1792
+ store_manifest_hash="0" * 64,
1793
+ )
1794
+ return EvalTrialStoreManifest.model_validate(
1795
+ manifest.model_copy(
1796
+ update={
1797
+ "store_manifest_hash": calculate_eval_trial_store_manifest_hash(
1798
+ manifest
1799
+ )
1800
+ }
1801
+ )
1802
+ )
1803
+
1804
+
1805
+ def _campaign_resume_index(
1806
+ *,
1807
+ plan: EvalTrialPlan,
1808
+ records: tuple[EvalTrialCampaignStoreRecordSummary, ...],
1809
+ plans: tuple[EvalTrialPlan, ...],
1810
+ ) -> EvalTrialResumeIndex:
1811
+ completed_plan_hashes = {record.trial_plan_hash for record in records}
1812
+ index = EvalTrialResumeIndex.model_construct(
1813
+ schema_version=EVAL_TRIAL_SCHEMA_VERSION,
1814
+ resume_index_id=f"{plan.campaign_manifest.campaign_id}.resume.v1",
1815
+ campaign_manifest_hash=plan.campaign_manifest.campaign_manifest_hash,
1816
+ completed_trial_record_hashes=tuple(record.record_hash for record in records),
1817
+ pending_trial_plan_hashes=tuple(
1818
+ item.plan_hash
1819
+ for item in plans
1820
+ if item.plan_hash not in completed_plan_hashes
1821
+ ),
1822
+ invalid_trial_diagnostics=(),
1823
+ resume_index_hash_kind=EVAL_TRIAL_RESUME_INDEX_HASH_KIND,
1824
+ resume_index_hash="0" * 64,
1825
+ )
1826
+ return EvalTrialResumeIndex.model_validate(
1827
+ index.model_copy(
1828
+ update={"resume_index_hash": calculate_eval_trial_resume_index_hash(index)}
1829
+ )
1830
+ )
1831
+
1832
+
1833
+ def _read_campaign_trial_log(
1834
+ trials_path: Path,
1835
+ ) -> tuple[
1836
+ tuple[EvalTrialCampaignStoreRecordSummary, ...],
1837
+ tuple[EvalTrialInvalidDiagnostic, ...],
1838
+ ]:
1839
+ if not trials_path.exists():
1840
+ return (), ()
1841
+ data = trials_path.read_bytes()
1842
+ if data == b"":
1843
+ return (), ()
1844
+
1845
+ records: list[EvalTrialCampaignStoreRecordSummary] = []
1846
+ diagnostics: list[EvalTrialInvalidDiagnostic] = []
1847
+ lines = data.splitlines(keepends=True)
1848
+ for index, line in enumerate(lines, start=1):
1849
+ final_line = index == len(lines)
1850
+ has_newline = line.endswith(b"\n")
1851
+ payload = line[:-1] if has_newline else line
1852
+ if not payload:
1853
+ continue
1854
+ if not has_newline and final_line:
1855
+ diagnostics.append(
1856
+ _store_diagnostic(
1857
+ EvalTrialInvalidDiagnosticCode.TRIAL_LOG_MISSING_FINAL_NEWLINE,
1858
+ "eval_trials.store.missing_final_newline",
1859
+ "trials.jsonl is missing the required final newline",
1860
+ )
1861
+ )
1862
+ try:
1863
+ parsed = json.loads(payload.decode("utf-8"))
1864
+ except (UnicodeDecodeError, json.JSONDecodeError):
1865
+ diagnostics.append(
1866
+ _store_diagnostic(
1867
+ EvalTrialInvalidDiagnosticCode.TRIAL_LOG_PARTIAL_TRAILING_RECORD
1868
+ if final_line and not has_newline
1869
+ else EvalTrialInvalidDiagnosticCode.TRIAL_LOG_MALFORMED_TRAILING_LINE,
1870
+ "eval_trials.store.partial_trailing_record"
1871
+ if final_line and not has_newline
1872
+ else "eval_trials.store.malformed_trailing_line",
1873
+ "trials.jsonl contains a partial trailing record"
1874
+ if final_line and not has_newline
1875
+ else "trials.jsonl contains malformed JSON",
1876
+ )
1877
+ )
1878
+ continue
1879
+ if not has_newline and final_line:
1880
+ continue
1881
+ try:
1882
+ records.append(_public_trial_record_from_json(parsed))
1883
+ except ValueError:
1884
+ diagnostics.append(
1885
+ _store_diagnostic(
1886
+ EvalTrialInvalidDiagnosticCode.TRIAL_LOG_INVALID_RECORD,
1887
+ "eval_trials.store.invalid_record",
1888
+ "trials.jsonl contains an invalid trial record",
1889
+ )
1890
+ )
1891
+ return tuple(records), tuple(diagnostics)
1892
+
1893
+
1894
+ def _public_trial_record_from_json(value: Any) -> EvalTrialCampaignStoreRecordSummary:
1895
+ if not isinstance(value, dict):
1896
+ raise ValueError("trial log record must be a JSON object")
1897
+ for field_name in (
1898
+ "campaign_id",
1899
+ "campaign_manifest_hash",
1900
+ "record_hash",
1901
+ "schema_version",
1902
+ "trial_id",
1903
+ "trial_plan_hash",
1904
+ ):
1905
+ if field_name not in value:
1906
+ raise ValueError("trial log record is missing required public fields")
1907
+ if value["schema_version"] != EVAL_TRIAL_SCHEMA_VERSION:
1908
+ raise ValueError("trial log record has unsupported schema version")
1909
+ for field_name in ("campaign_manifest_hash", "record_hash", "trial_plan_hash"):
1910
+ if not isinstance(value[field_name], str):
1911
+ raise ValueError("trial log record hash fields must be strings")
1912
+ _validate_sha256(value[field_name])
1913
+ payload = dict(value)
1914
+ record_hash = payload.pop("record_hash")
1915
+ if hashlib.sha256(_canonical_eval_trial_bytes(payload)).hexdigest() != record_hash:
1916
+ raise ValueError("trial log record hash does not match public payload")
1917
+ if not isinstance(value["trial_id"], str):
1918
+ raise ValueError("trial log trial_id must be a string")
1919
+ return EvalTrialCampaignStoreRecordSummary(
1920
+ trial_id=value["trial_id"],
1921
+ trial_plan_hash=value["trial_plan_hash"],
1922
+ record_hash=record_hash,
1923
+ )
1924
+
1925
+
1926
+ def _record_summary(record: EvalTrialRecord) -> EvalTrialCampaignStoreRecordSummary:
1927
+ return EvalTrialCampaignStoreRecordSummary(
1928
+ trial_id=record.trial_id,
1929
+ trial_plan_hash=record.trial_plan_hash,
1930
+ record_hash=record.record_hash,
1931
+ )
1932
+
1933
+
1934
+ def _duplicate_trial_ids(
1935
+ records: tuple[EvalTrialCampaignStoreRecordSummary, ...],
1936
+ ) -> tuple[str, ...]:
1937
+ seen: set[str] = set()
1938
+ duplicates: list[str] = []
1939
+ for record in records:
1940
+ if record.trial_id in seen and record.trial_id not in duplicates:
1941
+ duplicates.append(record.trial_id)
1942
+ seen.add(record.trial_id)
1943
+ return tuple(duplicates)
1944
+
1945
+
1946
+ def _store_diagnostic(
1947
+ code: EvalTrialInvalidDiagnosticCode,
1948
+ rule_id: str,
1949
+ summary: str,
1950
+ ) -> EvalTrialInvalidDiagnostic:
1951
+ return EvalTrialInvalidDiagnostic(
1952
+ diagnostic_code=code,
1953
+ rule_id=rule_id,
1954
+ summary=summary,
1955
+ )
1956
+
1957
+
1958
+ def _offline_fake_runner_live_denials(
1959
+ *,
1960
+ execution_mode: EvalSuiteExecutionMode,
1961
+ live_execution_admitted: bool,
1962
+ allow_live_model_call: bool,
1963
+ allow_pi_execution: bool,
1964
+ allow_millforge_harness_execution: bool,
1965
+ ) -> tuple[EvalTrialInvalidDiagnostic, ...]:
1966
+ diagnostics: list[EvalTrialInvalidDiagnostic] = []
1967
+ if execution_mode != EvalSuiteExecutionMode.OFFLINE_FAKE:
1968
+ diagnostics.append(
1969
+ deny_eval_trial_live_execution(
1970
+ "Offline fake trial runner rejects live runner execution mode."
1971
+ )
1972
+ )
1973
+ if live_execution_admitted:
1974
+ diagnostics.append(
1975
+ deny_eval_trial_live_execution(
1976
+ "Offline fake trial runner rejects live execution admission."
1977
+ )
1978
+ )
1979
+ if allow_live_model_call:
1980
+ diagnostics.append(
1981
+ deny_eval_trial_live_execution(
1982
+ "Offline fake trial runner rejects model call execution."
1983
+ )
1984
+ )
1985
+ if allow_pi_execution:
1986
+ diagnostics.append(
1987
+ deny_eval_trial_live_execution(
1988
+ "Offline fake trial runner rejects Pi execution."
1989
+ )
1990
+ )
1991
+ if allow_millforge_harness_execution:
1992
+ diagnostics.append(
1993
+ deny_eval_trial_live_execution(
1994
+ "Offline fake trial runner rejects Millforge harness execution."
1995
+ )
1996
+ )
1997
+ return tuple(diagnostics)
1998
+
1999
+
2000
+ def _fake_runner_artifact_bundle(
2001
+ *,
2002
+ plan: EvalTrialPlan,
2003
+ arm_plan: EvalTrialArmPlan,
2004
+ ) -> EvalFakeRunnerArtifactBundle:
2005
+ public_artifact_ids = _fake_runner_public_artifact_ids()
2006
+ entries = tuple(
2007
+ _fake_runner_artifact_manifest_entry(
2008
+ trial_id=plan.trial_id,
2009
+ arm_id=arm_plan.arm_id,
2010
+ artifact_id=artifact_id,
2011
+ script_kind=plan.fake_runner_script.script_kind,
2012
+ )
2013
+ for artifact_id in public_artifact_ids
2014
+ )
2015
+ artifact_manifest = EvalArtifactManifestArtifact(
2016
+ trial_id=plan.trial_id,
2017
+ created_by="offline_fake_runner",
2018
+ summary="offline fake runner artifact manifest",
2019
+ entries=entries,
2020
+ )
2021
+ manifest_hash = calculate_eval_artifact_manifest_sha256(artifact_manifest)
2022
+ artifact_hashes = tuple(
2023
+ EvalHashRecord(hash_kind="eval_artifact_sha256_v1", sha256=entry.sha256)
2024
+ for entry in entries
2025
+ ) + (
2026
+ EvalHashRecord(
2027
+ hash_kind="eval_artifact_manifest_sha256_v1",
2028
+ sha256=manifest_hash,
2029
+ ),
2030
+ )
2031
+ bundle = EvalFakeRunnerArtifactBundle.model_construct(
2032
+ schema_version=EVAL_TRIAL_SCHEMA_VERSION,
2033
+ artifact_bundle_id=f"{plan.trial_id}.{arm_plan.arm_id.value}.bundle.v1",
2034
+ trial_plan_hash=plan.plan_hash,
2035
+ arm_id=arm_plan.arm_id,
2036
+ artifact_root=arm_plan.artifact_root,
2037
+ artifact_manifest=artifact_manifest,
2038
+ artifact_hashes=artifact_hashes,
2039
+ zero_model_usage=True,
2040
+ zero_external_usage=True,
2041
+ artifact_bundle_hash_kind=EVAL_TRIAL_ARTIFACT_BUNDLE_HASH_KIND,
2042
+ artifact_bundle_hash="0" * 64,
2043
+ )
2044
+ return EvalFakeRunnerArtifactBundle.model_validate(
2045
+ bundle.model_copy(
2046
+ update={
2047
+ "artifact_bundle_hash": (
2048
+ calculate_eval_fake_runner_artifact_bundle_hash(bundle)
2049
+ )
2050
+ }
2051
+ )
2052
+ )
2053
+
2054
+
2055
+ def _fake_runner_artifact_manifest_entry(
2056
+ *,
2057
+ trial_id: str,
2058
+ arm_id: EvalTrialArmId,
2059
+ artifact_id: EvalArtifactId,
2060
+ script_kind: EvalFakeOutcomeScriptKind,
2061
+ ) -> EvalArtifactManifestEntry:
2062
+ layout = eval_artifact_layout_entry(artifact_id)
2063
+ payload = {
2064
+ "artifact_id": artifact_id.value,
2065
+ "arm_id": arm_id.value,
2066
+ "script_kind": script_kind.value,
2067
+ "trial_id": trial_id,
2068
+ }
2069
+ artifact_bytes = _canonical_eval_trial_bytes(payload)
2070
+ return EvalArtifactManifestEntry(
2071
+ artifact_id=artifact_id,
2072
+ layout_path=layout.layout_path,
2073
+ media_type=layout.media_type,
2074
+ schema_id=layout.schema_id,
2075
+ byte_size=len(artifact_bytes),
2076
+ sha256=hashlib.sha256(artifact_bytes).hexdigest(),
2077
+ producer="offline_fake_runner",
2078
+ model_visible=layout.model_visible,
2079
+ )
2080
+
2081
+
2082
+ def _fake_runner_scorer_input(
2083
+ *,
2084
+ plan: EvalTrialPlan,
2085
+ fixture: EvalTaskFixture,
2086
+ arm_id: EvalTrialArmId,
2087
+ artifact_hashes: tuple[EvalHashRecord, ...],
2088
+ ) -> EvalScorerInput:
2089
+ script_kind = plan.fake_runner_script.script_kind
2090
+ artifact_ids = tuple(
2091
+ artifact_id.value for artifact_id in _fake_runner_public_artifact_ids()
2092
+ )
2093
+ terminal_results = tuple(
2094
+ terminal.value for terminal in plan.fake_runner_script.terminal_results
2095
+ )
2096
+ visible_passed = script_kind not in {
2097
+ EvalFakeOutcomeScriptKind.FALSE_CLOSURE,
2098
+ EvalFakeOutcomeScriptKind.FALSE_SUCCESS_WITHOUT_CLOSURE,
2099
+ }
2100
+ hidden_passed = visible_passed
2101
+ scorer_input = EvalScorerInput.model_construct(
2102
+ trial_id=plan.trial_id,
2103
+ fixture_id=fixture.fixture_id,
2104
+ fixture_hash=fixture.fixture_hash
2105
+ if script_kind is not EvalFakeOutcomeScriptKind.INVALID_TRIAL
2106
+ else "f" * 64,
2107
+ final_workspace_hash=_fake_workspace_hash(
2108
+ trial_id=plan.trial_id,
2109
+ arm_id=arm_id,
2110
+ script_kind=script_kind,
2111
+ ),
2112
+ path_limited_workspace_hashes=(),
2113
+ public_artifact_hashes=artifact_hashes,
2114
+ required_public_artifact_ids=artifact_ids,
2115
+ provided_public_artifact_ids=artifact_ids,
2116
+ malformed_artifact_ids=(),
2117
+ stage_terminal_results=terminal_results,
2118
+ capability_audit=EvalCapabilityAuditSummary(
2119
+ capability_violation=False,
2120
+ denied_capability_ids=(),
2121
+ summary="offline fake runner used no denied capabilities",
2122
+ ),
2123
+ visible_check_results=tuple(
2124
+ EvalCheckResult(check_id=check.check_id, passed=visible_passed)
2125
+ for check in fixture.visible_checks
2126
+ ),
2127
+ hidden_check_results=tuple(
2128
+ EvalCheckResult(check_id=check.check_id, passed=hidden_passed)
2129
+ for check in fixture.hidden_checks
2130
+ ),
2131
+ claimed_mutation_present=True,
2132
+ unauthorized_mutation=False,
2133
+ checker_public_evidence_valid=True,
2134
+ runtime_failure=script_kind is EvalFakeOutcomeScriptKind.RUNTIME_FAILURE,
2135
+ provider_failure=script_kind is EvalFakeOutcomeScriptKind.PROVIDER_FAILURE,
2136
+ invalid_trial_explanation=(
2137
+ "offline fake runner scripted invalid trial"
2138
+ if script_kind is EvalFakeOutcomeScriptKind.INVALID_TRIAL
2139
+ else None
2140
+ ),
2141
+ scorer_input_hash_kind="eval_suite_scorer_input_sha256_v1",
2142
+ scorer_input_hash="0" * 64,
2143
+ )
2144
+ return EvalScorerInput.model_validate(
2145
+ scorer_input.model_copy(
2146
+ update={"scorer_input_hash": calculate_eval_scorer_input_hash(scorer_input)}
2147
+ )
2148
+ )
2149
+
2150
+
2151
+ def _validate_fake_script_outcome(
2152
+ script: EvalTrialFakeRunnerScript,
2153
+ scorer_result: EvalScorerResult,
2154
+ ) -> None:
2155
+ if scorer_result.final_outcome != script.expected_outcome:
2156
+ raise ValueError(
2157
+ "offline fake script expected_outcome does not match scorer result"
2158
+ )
2159
+ if (
2160
+ script.script_kind is EvalFakeOutcomeScriptKind.FALSE_SUCCESS_WITHOUT_CLOSURE
2161
+ and not scorer_result.false_success
2162
+ ):
2163
+ raise ValueError("offline fake script did not produce false success")
2164
+
2165
+
2166
+ def _trial_runner_summaries(
2167
+ plan: EvalTrialPlan,
2168
+ ) -> Mapping[EvalTrialArmId, EvalTrialRunnerRecordSummary]:
2169
+ return _freeze_trial_mapping(
2170
+ {
2171
+ arm_plan.arm_id: EvalTrialRunnerRecordSummary(
2172
+ arm_id=arm_plan.arm_id,
2173
+ runner_kind=arm_plan.runner_kind,
2174
+ runner_descriptor_id=arm_plan.runner_descriptor_id,
2175
+ runner_descriptor_version="eval-trial.runner-descriptor.v1",
2176
+ baseline_pi_runtime_hash=arm_plan.baseline_pi_runtime_hash,
2177
+ baseline_runtime_diagnostic=arm_plan.baseline_runtime_diagnostic,
2178
+ )
2179
+ for arm_plan in plan.arm_plans
2180
+ }
2181
+ )
2182
+
2183
+
2184
+ def _trial_scorer_public_summaries(
2185
+ results: tuple[EvalTrialExecutionResult, EvalTrialExecutionResult],
2186
+ ) -> Mapping[EvalTrialArmId, EvalTrialScorerPublicSummary]:
2187
+ return _freeze_trial_mapping(
2188
+ {
2189
+ result.arm_id: EvalTrialScorerPublicSummary(
2190
+ arm_id=result.arm_id,
2191
+ scorer_version=result.scorer_result.scorer_version,
2192
+ final_outcome=result.scorer_result.final_outcome,
2193
+ primary_success=result.scorer_result.primary_success,
2194
+ false_closure=result.scorer_result.false_closure,
2195
+ false_success=result.scorer_result.false_success,
2196
+ correctly_blocked=result.scorer_result.correctly_blocked,
2197
+ capability_violation=result.scorer_result.capability_violation,
2198
+ artifact_complete=result.scorer_result.artifact_complete,
2199
+ missing_artifact_ids=result.scorer_result.missing_artifact_ids,
2200
+ malformed_artifact_ids=result.scorer_result.malformed_artifact_ids,
2201
+ failure_labels=tuple(
2202
+ label.value for label in result.scorer_result.failure_labels
2203
+ ),
2204
+ public_diagnostics=result.scorer_result.public_diagnostics,
2205
+ invalid_trial_explanation=(
2206
+ result.scorer_result.invalid_trial_explanation
2207
+ ),
2208
+ )
2209
+ for result in results
2210
+ }
2211
+ )
2212
+
2213
+
2214
+ def _trial_resource_summary(
2215
+ bundles: tuple[EvalFakeRunnerArtifactBundle, EvalFakeRunnerArtifactBundle],
2216
+ ) -> EvalTrialResourceSummary:
2217
+ hashes = tuple(record for bundle in bundles for record in bundle.artifact_hashes)
2218
+ return EvalTrialResourceSummary(
2219
+ artifact_count=sum(len(bundle.artifact_hashes) for bundle in bundles),
2220
+ artifact_bytes=sum(
2221
+ entry.byte_size
2222
+ for bundle in bundles
2223
+ for entry in bundle.artifact_manifest.entries
2224
+ ),
2225
+ resource_artifact_hashes=hashes,
2226
+ zero_external_usage=all(bundle.zero_external_usage for bundle in bundles),
2227
+ )
2228
+
2229
+
2230
+ def _fake_runner_public_artifact_ids() -> tuple[EvalArtifactId, ...]:
2231
+ return (
2232
+ EvalArtifactId.TASK,
2233
+ EvalArtifactId.ACCEPTANCE_CHECKS,
2234
+ EvalArtifactId.PLAN,
2235
+ EvalArtifactId.WORKSPACE_DIFF,
2236
+ EvalArtifactId.PATCH_SUMMARY,
2237
+ EvalArtifactId.TEST_RESULTS,
2238
+ EvalArtifactId.CHECKER_VERDICT,
2239
+ EvalArtifactId.ARBITER_VERDICT,
2240
+ EvalArtifactId.STAGE_RESULT,
2241
+ EvalArtifactId.EVENT_LOG,
2242
+ EvalArtifactId.RESOURCE_USAGE,
2243
+ EvalArtifactId.MODEL_USAGE,
2244
+ EvalArtifactId.VALIDATOR_RESULT,
2245
+ )
2246
+
2247
+
2248
+ def _fake_workspace_hash(
2249
+ *,
2250
+ trial_id: str,
2251
+ arm_id: EvalTrialArmId,
2252
+ script_kind: EvalFakeOutcomeScriptKind,
2253
+ ) -> str:
2254
+ return hashlib.sha256(
2255
+ _canonical_eval_trial_bytes(
2256
+ {
2257
+ "arm_id": arm_id.value,
2258
+ "script_kind": script_kind.value,
2259
+ "trial_id": trial_id,
2260
+ }
2261
+ )
2262
+ ).hexdigest()
2263
+
2264
+
2265
+ def _arm_definition(
2266
+ descriptor: EvalModeDescriptor,
2267
+ arm_id: EvalTrialArmId,
2268
+ ) -> EvalTrialArmDefinition:
2269
+ return EvalTrialArmDefinition(
2270
+ arm_id=arm_id,
2271
+ mode_id=descriptor.mode_id,
2272
+ descriptor_fingerprint=descriptor.descriptor_fingerprint,
2273
+ fairness_fingerprint=descriptor.fairness_fingerprint,
2274
+ runner_kind=descriptor.runner_bindings[0].runner_kind.value,
2275
+ descriptor=descriptor,
2276
+ )
2277
+
2278
+
2279
+ def _default_eval_trial_arm_plans(
2280
+ *,
2281
+ trial_id: str,
2282
+ trial_index: int,
2283
+ paired_seed: int,
2284
+ campaign_store_root: str,
2285
+ campaign_manifest: EvalCampaignManifest,
2286
+ arms: tuple[EvalTrialArmDefinition, EvalTrialArmDefinition],
2287
+ arm_order: tuple[EvalTrialArmId, EvalTrialArmId],
2288
+ fixture: EvalTaskFixture,
2289
+ readiness: EvalPresetReadinessReport,
2290
+ artifact_roots_by_arm: Mapping[EvalTrialArmId | str, str] | None = None,
2291
+ ) -> tuple[EvalTrialArmPlan, EvalTrialArmPlan]:
2292
+ roots = artifact_roots_by_arm or {}
2293
+ compiled_hashes = _compiled_harness_hashes_by_stage(readiness)
2294
+ hidden_check_set_ids = tuple(check.check_id for check in fixture.hidden_checks)
2295
+ hidden_check_set_hash = _hidden_check_set_hash(hidden_check_set_ids)
2296
+ visible_acceptance_hash = _visible_acceptance_criteria_hash(
2297
+ fixture.visible_acceptance_criteria
2298
+ )
2299
+ plans: list[EvalTrialArmPlan] = []
2300
+ for arm in arms:
2301
+ root = roots.get(arm.arm_id) or roots.get(arm.arm_id.value)
2302
+ root = root or _default_artifact_root(
2303
+ campaign_store_root=campaign_store_root,
2304
+ trial_id=trial_id,
2305
+ arm_id=arm.arm_id,
2306
+ )
2307
+ plans.append(
2308
+ EvalTrialArmPlan(
2309
+ trial_id=trial_id,
2310
+ trial_index=trial_index,
2311
+ arm_id=arm.arm_id,
2312
+ arm_order_index=arm_order.index(arm.arm_id),
2313
+ paired_seed=paired_seed,
2314
+ artifact_root=root,
2315
+ runner_kind=arm.runner_kind,
2316
+ runner_descriptor_id=arm.descriptor_fingerprint,
2317
+ campaign_manifest_hash=campaign_manifest.campaign_manifest_hash,
2318
+ model_manifest_hash=campaign_manifest.model_manifest_hash,
2319
+ workflow_graph_hash=campaign_manifest.workflow_graph_hash,
2320
+ fixture_pack_hash=campaign_manifest.fixture_pack_hash,
2321
+ fixture_id=fixture.fixture_id,
2322
+ fixture_hash=fixture.fixture_hash,
2323
+ visible_acceptance_criteria_hash=visible_acceptance_hash,
2324
+ hidden_check_set_ids=hidden_check_set_ids,
2325
+ hidden_check_set_hash=hidden_check_set_hash,
2326
+ scorer_version=campaign_manifest.scorer_version,
2327
+ treatment_compiled_harness_hashes=compiled_hashes
2328
+ if arm.arm_id is EvalTrialArmId.EVAL_SMALL_MILLFORGE
2329
+ else {},
2330
+ baseline_pi_runtime_hash=None,
2331
+ baseline_runtime_diagnostic=EvalTrialInvalidDiagnostic(
2332
+ diagnostic_code=(
2333
+ EvalTrialInvalidDiagnosticCode.LIVE_EXECUTION_UNAVAILABLE
2334
+ ),
2335
+ rule_id="eval_trials.pi_runtime.deferred",
2336
+ summary="Pi runtime hash is deferred for offline planning.",
2337
+ )
2338
+ if arm.arm_id is EvalTrialArmId.EVAL_SMALL_PI
2339
+ else None,
2340
+ )
2341
+ )
2342
+ return (plans[0], plans[1])
2343
+
2344
+
2345
+ def _validate_admitted_planning_arms(
2346
+ arms: tuple[EvalTrialArmDefinition, EvalTrialArmDefinition],
2347
+ ) -> None:
2348
+ if tuple(arm.arm_id.value for arm in arms) != EVAL_TRIAL_ADMITTED_ARM_IDS:
2349
+ raise ValueError("paired trial planning rejects unsupported arms")
2350
+
2351
+
2352
+ def _default_campaign_store_root(campaign_manifest: EvalCampaignManifest) -> str:
2353
+ return f"eval/campaigns/{campaign_manifest.campaign_id}"
2354
+
2355
+
2356
+ def _default_artifact_root(
2357
+ *,
2358
+ campaign_store_root: str,
2359
+ trial_id: str,
2360
+ arm_id: EvalTrialArmId,
2361
+ ) -> str:
2362
+ _validate_relative_artifact_root(campaign_store_root)
2363
+ return f"artifacts/{trial_id}/{arm_id.value}"
2364
+
2365
+
2366
+ def _trial_id(
2367
+ *,
2368
+ campaign_id: str,
2369
+ fixture_id: str,
2370
+ trial_index: int,
2371
+ seed: int,
2372
+ ) -> str:
2373
+ digest = hashlib.sha256(
2374
+ _canonical_eval_trial_bytes(
2375
+ {
2376
+ "campaign_id": campaign_id,
2377
+ "fixture_id": fixture_id,
2378
+ "seed": seed,
2379
+ "trial_index": trial_index,
2380
+ }
2381
+ )
2382
+ ).hexdigest()[:12]
2383
+ safe_fixture = re.sub(r"[^a-zA-Z0-9_.-]+", "_", fixture_id)
2384
+ return f"{campaign_id}.trial.{trial_index:04d}.{safe_fixture}.{digest}"
2385
+
2386
+
2387
+ def _paired_seed(*, seed: int, fixture_id: str, trial_index: int) -> int:
2388
+ digest = hashlib.sha256(
2389
+ _canonical_eval_trial_bytes(
2390
+ {"fixture_id": fixture_id, "seed": seed, "trial_index": trial_index}
2391
+ )
2392
+ ).hexdigest()
2393
+ return int(digest[:16], 16)
2394
+
2395
+
2396
+ def _randomized_arm_order(
2397
+ paired_seed: int,
2398
+ ) -> tuple[EvalTrialArmId, EvalTrialArmId]:
2399
+ arms = [EvalTrialArmId.EVAL_SMALL_PI, EvalTrialArmId.EVAL_SMALL_MILLFORGE]
2400
+ random.Random(paired_seed).shuffle(arms)
2401
+ return (arms[0], arms[1])
2402
+
2403
+
2404
+ def _fixture_instance_id(
2405
+ *,
2406
+ trial_id: str,
2407
+ trial_index: int,
2408
+ fixture_id: str,
2409
+ paired_seed: int,
2410
+ ) -> str:
2411
+ digest = hashlib.sha256(
2412
+ _canonical_eval_trial_bytes(
2413
+ {
2414
+ "fixture_id": fixture_id,
2415
+ "paired_seed": paired_seed,
2416
+ "trial_id": trial_id,
2417
+ "trial_index": trial_index,
2418
+ }
2419
+ )
2420
+ ).hexdigest()[:16]
2421
+ return f"{trial_id}.fixture_instance.{digest}"
2422
+
2423
+
2424
+ def _fixture_snapshot_hash(
2425
+ *,
2426
+ fixture: EvalTaskFixture,
2427
+ fixture_instance_id: str,
2428
+ paired_seed: int,
2429
+ ) -> str:
2430
+ return hashlib.sha256(
2431
+ _canonical_eval_trial_bytes(
2432
+ {
2433
+ "fixture_hash": fixture.fixture_hash,
2434
+ "fixture_id": fixture.fixture_id,
2435
+ "fixture_instance_id": fixture_instance_id,
2436
+ "paired_seed": paired_seed,
2437
+ }
2438
+ )
2439
+ ).hexdigest()
2440
+
2441
+
2442
+ def _visible_acceptance_criteria_hash(criteria: tuple[str, ...]) -> str:
2443
+ return hashlib.sha256(
2444
+ _canonical_eval_trial_bytes({"visible_acceptance_criteria": criteria})
2445
+ ).hexdigest()
2446
+
2447
+
2448
+ def _hidden_check_set_hash(check_ids: tuple[str, ...]) -> str:
2449
+ return hashlib.sha256(
2450
+ _canonical_eval_trial_bytes({"hidden_check_set_ids": check_ids})
2451
+ ).hexdigest()
2452
+
2453
+
2454
+ def _compiled_harness_hashes_by_stage(
2455
+ readiness: EvalPresetReadinessReport,
2456
+ ) -> Mapping[str, str]:
2457
+ required_stage_ids = tuple(stage_id.value for stage_id in EvalStageId)
2458
+ compiled = {
2459
+ plan.stage_id: plan.compiled_sha256 for plan in readiness.compiled_plans
2460
+ }
2461
+ if tuple(compiled) != required_stage_ids:
2462
+ raise ValueError(
2463
+ "Spec 07E readiness must include Planner, Builder, Checker, Arbiter"
2464
+ )
2465
+ return _freeze_trial_mapping(compiled)
2466
+
2467
+
2468
+ def _reject_fake_pi_runtime_hash(value: str) -> None:
2469
+ if value in {"0" * 64, "f" * 64} or len(set(value)) == 1:
2470
+ raise ValueError("Pi baseline runtime hash must not be a fake concrete hash")
2471
+
2472
+
2473
+ def _validate_relative_artifact_root(path: str) -> None:
2474
+ if (
2475
+ not path
2476
+ or _WINDOWS_ABSOLUTE_PATH.search(path)
2477
+ or _POSIX_ABSOLUTE_PATH.search(path)
2478
+ or _USER_HOME_PATH.search(path)
2479
+ ):
2480
+ raise ValueError("artifact roots must be stable relative paths")
2481
+ pure = PurePosixPath(path)
2482
+ if pure.is_absolute() or any(part in {"", ".", ".."} for part in pure.parts):
2483
+ raise ValueError("artifact roots must be stable relative paths")
2484
+ if "\\" in path:
2485
+ raise ValueError("artifact roots must use relative POSIX paths")
2486
+
2487
+
2488
+ def _validate_artifact_root_under_campaign(
2489
+ artifact_root: str,
2490
+ campaign_store_root: str,
2491
+ ) -> None:
2492
+ _validate_relative_artifact_root(artifact_root)
2493
+ _validate_relative_artifact_root(campaign_store_root)
2494
+ artifact_parts = PurePosixPath(artifact_root).parts
2495
+ if len(artifact_parts) < 2 or artifact_parts[0] != "artifacts":
2496
+ raise ValueError("artifact roots must stay inside the campaign store")
2497
+
2498
+
2499
+ def _canonical_eval_trial_bytes(value: Mapping[str, Any]) -> bytes:
2500
+ return (
2501
+ json.dumps(
2502
+ value,
2503
+ sort_keys=True,
2504
+ ensure_ascii=True,
2505
+ allow_nan=False,
2506
+ separators=(",", ":"),
2507
+ ).replace("\r\n", "\n")
2508
+ + "\n"
2509
+ ).encode("ascii")
2510
+
2511
+
2512
+ def _validate_sha256(value: str) -> None:
2513
+ if not _SHA256_RE.fullmatch(value):
2514
+ raise ValueError("expected lowercase sha256 hex digest")
2515
+
2516
+
2517
+ def _reject_forbidden_material(value: Any) -> None:
2518
+ if isinstance(value, Mapping):
2519
+ for key, child in value.items():
2520
+ key_text = str(key)
2521
+ _reject_secret_like_field_name(key_text)
2522
+ _reject_forbidden_material(key_text)
2523
+ _reject_forbidden_material(child)
2524
+ return
2525
+ if isinstance(value, (tuple, list, set, frozenset)):
2526
+ for child in value:
2527
+ _reject_forbidden_material(child)
2528
+ return
2529
+ if isinstance(value, BaseModel):
2530
+ _reject_forbidden_material(value.model_dump(mode="json"))
2531
+ return
2532
+ if isinstance(value, Enum):
2533
+ _reject_forbidden_material(value.value)
2534
+ return
2535
+ if isinstance(value, str):
2536
+ lowered = value.lower()
2537
+ if any(token in lowered for token in _DENIED_TEXT_TOKENS):
2538
+ raise ValueError("eval-trial payload contains forbidden private material")
2539
+ if any(pattern.search(value) for pattern in _CREDENTIAL_VALUE_PATTERNS):
2540
+ raise ValueError("eval-trial payload contains credential-shaped API key")
2541
+ if _ENDPOINT_URL.search(value):
2542
+ raise ValueError("eval-trial payloads must not contain endpoint URLs")
2543
+ if (
2544
+ _WINDOWS_ABSOLUTE_PATH.search(value)
2545
+ or _POSIX_ABSOLUTE_PATH.search(value)
2546
+ or _USER_HOME_PATH.search(value)
2547
+ ):
2548
+ raise ValueError("eval-trial payloads must not contain host paths")
2549
+
2550
+
2551
+ def _reject_secret_like_field_name(field_name: str) -> None:
2552
+ normalized = field_name.lower().replace("-", "_")
2553
+ if normalized.endswith("_free"):
2554
+ return
2555
+ if any(marker in normalized for marker in _SECRET_FIELD_MARKERS):
2556
+ raise ValueError("eval-trial payload contains secret-like field name")
2557
+
2558
+
2559
+ class _FrozenTrialDict(dict[Any, Any]):
2560
+ """Dict-shaped immutable mapping that remains serializable by Pydantic."""
2561
+
2562
+ def __readonly(self, *args: Any, **kwargs: Any) -> None:
2563
+ raise TypeError("eval-trial mappings are immutable")
2564
+
2565
+ __setitem__ = __readonly
2566
+ __delitem__ = __readonly
2567
+ clear = __readonly
2568
+ pop = __readonly
2569
+ popitem = __readonly # type: ignore[assignment]
2570
+ setdefault = __readonly
2571
+ update = __readonly
2572
+ __ior__ = __readonly # type: ignore[assignment]
2573
+
2574
+
2575
+ def _freeze_trial_mapping(value: Mapping[Any, Any]) -> Mapping[Any, Any]:
2576
+ return _FrozenTrialDict({key: child for key, child in value.items()})
2577
+
2578
+
2579
+ __all__ = [
2580
+ "EVAL_TRIAL_ADMITTED_ARM_IDS",
2581
+ "EVAL_TRIAL_ARTIFACT_BUNDLE_HASH_KIND",
2582
+ "EVAL_TRIAL_DEFAULT_CREATED_AT",
2583
+ "EVAL_TRIAL_PLAN_HASH_KIND",
2584
+ "EVAL_TRIAL_RECORD_HASH_KIND",
2585
+ "EVAL_TRIAL_RESUME_INDEX_HASH_KIND",
2586
+ "EVAL_TRIAL_SCHEMA_VERSION",
2587
+ "EVAL_TRIAL_STORE_MANIFEST_HASH_KIND",
2588
+ "EvalFakeOutcomeScriptKind",
2589
+ "EvalFakeRunnerArtifactBundle",
2590
+ "EvalOfflineFakeTrialRun",
2591
+ "EvalTrialCampaignStoreAppendResult",
2592
+ "EvalTrialCampaignStoreRecordSummary",
2593
+ "EvalTrialCampaignStoreResumeResult",
2594
+ "EvalTrialArmDefinition",
2595
+ "EvalTrialArmId",
2596
+ "EvalTrialArmParityEvidence",
2597
+ "EvalTrialArmPlan",
2598
+ "EvalTrialContractModel",
2599
+ "EvalTrialExecutionResult",
2600
+ "EvalTrialFakeRunnerScript",
2601
+ "EvalTrialFixtureInstance",
2602
+ "EvalTrialInvalidDiagnostic",
2603
+ "EvalTrialInvalidDiagnosticCode",
2604
+ "EvalTrialPlan",
2605
+ "EvalTrialRecord",
2606
+ "EvalTrialModelUsageSummary",
2607
+ "EvalTrialResumeIndex",
2608
+ "EvalTrialResourceSummary",
2609
+ "EvalTrialRunnerRecordSummary",
2610
+ "EvalTrialScorerPublicSummary",
2611
+ "EvalTrialStoreManifest",
2612
+ "calculate_eval_fake_runner_artifact_bundle_hash",
2613
+ "calculate_eval_trial_plan_hash",
2614
+ "calculate_eval_trial_record_hash",
2615
+ "calculate_eval_trial_resume_index_hash",
2616
+ "calculate_eval_trial_store_manifest_hash",
2617
+ "canonical_eval_fake_runner_artifact_bundle_bytes",
2618
+ "canonical_eval_trial_plan_bytes",
2619
+ "canonical_eval_trial_record_bytes",
2620
+ "canonical_eval_trial_resume_index_bytes",
2621
+ "canonical_eval_trial_store_manifest_bytes",
2622
+ "append_eval_trial_record_to_campaign_store",
2623
+ "default_eval_trial_arm_definitions",
2624
+ "default_eval_trial_fixture_instance",
2625
+ "default_eval_trial_parity_evidence",
2626
+ "default_eval_trial_plan",
2627
+ "deny_eval_trial_live_execution",
2628
+ "fixture_copy_unavailable_diagnostic",
2629
+ "plan_paired_eval_trials",
2630
+ "resume_eval_trial_campaign_store",
2631
+ "run_offline_fake_eval_trial",
2632
+ ]