simjecture 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- conjecture_solver/__init__.py +10 -0
- conjecture_solver/__main__.py +4 -0
- conjecture_solver/action_handlers.py +459 -0
- conjecture_solver/adapters/__init__.py +7 -0
- conjecture_solver/adapters/base.py +96 -0
- conjecture_solver/adapters/fake.py +155 -0
- conjecture_solver/adapters/pic.py +196 -0
- conjecture_solver/adapters/warpx.py +1375 -0
- conjecture_solver/autonomous_research.py +1071 -0
- conjecture_solver/benchmarks/__init__.py +6 -0
- conjecture_solver/benchmarks/electrostatic_pic.py +547 -0
- conjecture_solver/benchmarks/kinetic_sufficiency.py +318 -0
- conjecture_solver/builtin_skills/warpx/SKILL.md +151 -0
- conjecture_solver/builtin_skills/warpx/agents/openai.yaml +4 -0
- conjecture_solver/builtin_skills/warpx/examples/implicit_em_smoke.py +74 -0
- conjecture_solver/builtin_skills/warpx/examples/minimal_smoke.py +54 -0
- conjecture_solver/builtin_skills/warpx/examples/openpmd_field_smoke.py +87 -0
- conjecture_solver/builtin_skills/warpx/manifest.json +11 -0
- conjecture_solver/builtin_skills/warpx/references/2d-xz-commissioning.md +145 -0
- conjecture_solver/builtin_skills/warpx/references/cpu-launch-tuning.md +54 -0
- conjecture_solver/builtin_skills/warpx/references/diagnostics.md +139 -0
- conjecture_solver/builtin_skills/warpx/references/gpu-launch-tuning.md +79 -0
- conjecture_solver/builtin_skills/warpx/references/local-cuda-deployment.md +87 -0
- conjecture_solver/builtin_skills/warpx/references/numerical-risks.md +32 -0
- conjecture_solver/builtin_skills/warpx/references/picmi-interface.md +118 -0
- conjecture_solver/builtin_skills/warpx/references/resource-scaling.md +24 -0
- conjecture_solver/builtin_skills/warpx/references/time-integration.md +58 -0
- conjecture_solver/builtin_skills/warpx/scripts/benchmark_cpu_threads.py +207 -0
- conjecture_solver/builtin_skills/warpx/scripts/benchmark_gpu.py +242 -0
- conjecture_solver/builtin_skills/warpx/scripts/bootstrap_local_cuda.sh +82 -0
- conjecture_solver/builtin_skills/warpx/scripts/probe_local_cuda.py +110 -0
- conjecture_solver/builtin_skills/warpx/scripts/reduced_energy_budget.py +195 -0
- conjecture_solver/builtin_skills/warpx/scripts/run_local_cuda.sh +39 -0
- conjecture_solver/campaign.py +406 -0
- conjecture_solver/cli.py +963 -0
- conjecture_solver/confirmation.py +287 -0
- conjecture_solver/control.py +234 -0
- conjecture_solver/deployment.py +874 -0
- conjecture_solver/discovery.py +68 -0
- conjecture_solver/domains/__init__.py +40 -0
- conjecture_solver/domains/base.py +157 -0
- conjecture_solver/domains/kinetic_sufficiency.py +521 -0
- conjecture_solver/ledger.py +238 -0
- conjecture_solver/lifecycle.py +131 -0
- conjecture_solver/literature.py +494 -0
- conjecture_solver/llm.py +228 -0
- conjecture_solver/models.py +279 -0
- conjecture_solver/mvp_agent.py +3212 -0
- conjecture_solver/mvp_claims.py +1279 -0
- conjecture_solver/mvp_control.py +322 -0
- conjecture_solver/mvp_guidance.py +172 -0
- conjecture_solver/mvp_launch.py +1296 -0
- conjecture_solver/mvp_monitor.py +1487 -0
- conjecture_solver/mvp_skills.py +538 -0
- conjecture_solver/orchestration.py +578 -0
- conjecture_solver/outbox.py +526 -0
- conjecture_solver/parameters.py +394 -0
- conjecture_solver/proposals.py +427 -0
- conjecture_solver/research_tools.py +278 -0
- conjecture_solver/schema_export.py +246 -0
- conjecture_solver/search.py +1047 -0
- conjecture_solver/semantics.py +146 -0
- conjecture_solver/tui/__init__.py +7 -0
- conjecture_solver/tui/app.py +202 -0
- conjecture_solver/tui/claim_views.py +213 -0
- conjecture_solver/tui/screens.py +1014 -0
- conjecture_solver/warpx_analysis.py +184 -0
- conjecture_solver/warpx_campaign.py +148 -0
- conjecture_solver/warpx_confirmation.py +617 -0
- simjecture-0.1.0.dist-info/METADATA +400 -0
- simjecture-0.1.0.dist-info/RECORD +75 -0
- simjecture-0.1.0.dist-info/WHEEL +4 -0
- simjecture-0.1.0.dist-info/entry_points.txt +4 -0
- simjecture-0.1.0.dist-info/licenses/LICENSE +202 -0
- simjecture-0.1.0.dist-info/licenses/NOTICE +6 -0
|
@@ -0,0 +1,1047 @@
|
|
|
1
|
+
"""Blinded, budget-matched search over a constrained distribution grammar."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import itertools
|
|
7
|
+
import json
|
|
8
|
+
from datetime import UTC, datetime
|
|
9
|
+
from enum import StrEnum
|
|
10
|
+
from typing import Any, Literal, Protocol
|
|
11
|
+
|
|
12
|
+
import numpy as np
|
|
13
|
+
from pydantic import Field, ValidationError, model_validator
|
|
14
|
+
from scipy.stats import qmc
|
|
15
|
+
|
|
16
|
+
from .benchmarks.kinetic_sufficiency import (
|
|
17
|
+
DistributionMoments,
|
|
18
|
+
GaussianComponent,
|
|
19
|
+
GaussianMixture,
|
|
20
|
+
LinearMode,
|
|
21
|
+
moments,
|
|
22
|
+
solve_modes,
|
|
23
|
+
)
|
|
24
|
+
from .ledger import SQLiteEventLedger, StoredEvent
|
|
25
|
+
from .llm import CompletionResult, ModelRoute
|
|
26
|
+
from .models import StrictModel, utc_now
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class SearchMethod(StrEnum):
|
|
30
|
+
AI = "ai"
|
|
31
|
+
GRID = "grid"
|
|
32
|
+
RANDOM = "random"
|
|
33
|
+
LATIN_HYPERCUBE = "latin_hypercube"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class CandidateOutcome(StrEnum):
|
|
37
|
+
VALID = "valid"
|
|
38
|
+
NUMERICAL_FAILURE = "numerical_failure"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class ParameterInterval(StrictModel):
|
|
42
|
+
minimum: float
|
|
43
|
+
maximum: float
|
|
44
|
+
|
|
45
|
+
@model_validator(mode="after")
|
|
46
|
+
def ordered(self) -> ParameterInterval:
|
|
47
|
+
if self.minimum >= self.maximum:
|
|
48
|
+
raise ValueError("parameter interval must have positive width")
|
|
49
|
+
return self
|
|
50
|
+
|
|
51
|
+
def map_unit(self, value: float) -> float:
|
|
52
|
+
if not 0.0 <= value <= 1.0:
|
|
53
|
+
raise ValueError("normalized parameter must lie in [0, 1]")
|
|
54
|
+
return self.minimum + value * (self.maximum - self.minimum)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class SymmetricMixtureSearchSpace(StrictModel):
|
|
58
|
+
"""Two symmetric drift pairs with a shared derived thermal width."""
|
|
59
|
+
|
|
60
|
+
grammar: Literal["symmetric_unit_variance_gaussian_mixture_v1"] = (
|
|
61
|
+
"symmetric_unit_variance_gaussian_mixture_v1"
|
|
62
|
+
)
|
|
63
|
+
inner_pair_weight: ParameterInterval = ParameterInterval(minimum=0.1, maximum=0.9)
|
|
64
|
+
inner_drift: ParameterInterval = ParameterInterval(minimum=0.0, maximum=0.95)
|
|
65
|
+
outer_drift: ParameterInterval = ParameterInterval(minimum=0.0, maximum=0.95)
|
|
66
|
+
required_order: Literal["inner_drift <= outer_drift"] = "inner_drift <= outer_drift"
|
|
67
|
+
derived_width: Literal[
|
|
68
|
+
"sqrt(1-inner_pair_weight*inner_drift^2-(1-inner_pair_weight)*outer_drift^2)"
|
|
69
|
+
] = "sqrt(1-inner_pair_weight*inner_drift^2-(1-inner_pair_weight)*outer_drift^2)"
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class SymmetricMixtureCandidate(StrictModel):
|
|
73
|
+
inner_pair_weight: float = Field(gt=0, lt=1)
|
|
74
|
+
inner_drift: float = Field(ge=0, lt=1)
|
|
75
|
+
outer_drift: float = Field(ge=0, lt=1)
|
|
76
|
+
|
|
77
|
+
@model_validator(mode="after")
|
|
78
|
+
def valid_unit_variance_parameterization(self) -> SymmetricMixtureCandidate:
|
|
79
|
+
if self.inner_drift > self.outer_drift:
|
|
80
|
+
raise ValueError("inner_drift must not exceed outer_drift")
|
|
81
|
+
if self.thermal_variance <= 0:
|
|
82
|
+
raise ValueError("drifts leave no positive thermal variance")
|
|
83
|
+
return self
|
|
84
|
+
|
|
85
|
+
@property
|
|
86
|
+
def thermal_variance(self) -> float:
|
|
87
|
+
return 1.0 - (
|
|
88
|
+
self.inner_pair_weight * self.inner_drift**2
|
|
89
|
+
+ (1.0 - self.inner_pair_weight) * self.outer_drift**2
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
@property
|
|
93
|
+
def thermal_sigma(self) -> float:
|
|
94
|
+
return float(np.sqrt(self.thermal_variance))
|
|
95
|
+
|
|
96
|
+
def distribution(self) -> GaussianMixture:
|
|
97
|
+
inner_side_weight = self.inner_pair_weight / 2.0
|
|
98
|
+
outer_side_weight = (1.0 - self.inner_pair_weight) / 2.0
|
|
99
|
+
return GaussianMixture(
|
|
100
|
+
components=(
|
|
101
|
+
GaussianComponent(
|
|
102
|
+
weight=inner_side_weight,
|
|
103
|
+
drift=-self.inner_drift,
|
|
104
|
+
sigma=self.thermal_sigma,
|
|
105
|
+
),
|
|
106
|
+
GaussianComponent(
|
|
107
|
+
weight=inner_side_weight,
|
|
108
|
+
drift=self.inner_drift,
|
|
109
|
+
sigma=self.thermal_sigma,
|
|
110
|
+
),
|
|
111
|
+
GaussianComponent(
|
|
112
|
+
weight=outer_side_weight,
|
|
113
|
+
drift=-self.outer_drift,
|
|
114
|
+
sigma=self.thermal_sigma,
|
|
115
|
+
),
|
|
116
|
+
GaussianComponent(
|
|
117
|
+
weight=outer_side_weight,
|
|
118
|
+
drift=self.outer_drift,
|
|
119
|
+
sigma=self.thermal_sigma,
|
|
120
|
+
),
|
|
121
|
+
)
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
def distribution_hash(self) -> str:
|
|
125
|
+
aggregated: dict[tuple[float, float], float] = {}
|
|
126
|
+
for component in self.distribution().components:
|
|
127
|
+
key = (round(component.drift, 12), round(component.sigma, 12))
|
|
128
|
+
aggregated[key] = aggregated.get(key, 0.0) + component.weight
|
|
129
|
+
payload = json.dumps(
|
|
130
|
+
[
|
|
131
|
+
{
|
|
132
|
+
"drift": drift,
|
|
133
|
+
"sigma": sigma,
|
|
134
|
+
"weight": round(weight, 12),
|
|
135
|
+
}
|
|
136
|
+
for (drift, sigma), weight in sorted(aggregated.items())
|
|
137
|
+
],
|
|
138
|
+
sort_keys=True,
|
|
139
|
+
separators=(",", ":"),
|
|
140
|
+
)
|
|
141
|
+
return hashlib.sha256(payload.encode()).hexdigest()
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
class BlindedSearchRequest(StrictModel):
|
|
145
|
+
schema_version: Literal["0.1.0"] = "0.1.0"
|
|
146
|
+
id: str = "search_request_blinded_moment_sufficiency_v1"
|
|
147
|
+
research_goal: str = (
|
|
148
|
+
"Find a distribution with density=1, mean_velocity=0, and variance=1 whose "
|
|
149
|
+
"dominant linear growth rate differs from the unit Maxwellian reference."
|
|
150
|
+
)
|
|
151
|
+
model_family: Literal["linearized_1d_electrostatic_vlasov_poisson"] = (
|
|
152
|
+
"linearized_1d_electrostatic_vlasov_poisson"
|
|
153
|
+
)
|
|
154
|
+
wavenumber: float = Field(default=0.5, gt=0)
|
|
155
|
+
reference_distribution: GaussianMixture = GaussianMixture(
|
|
156
|
+
components=(GaussianComponent(weight=1.0, drift=0.0, sigma=1.0),)
|
|
157
|
+
)
|
|
158
|
+
target_moments: DistributionMoments = DistributionMoments(
|
|
159
|
+
density=1.0,
|
|
160
|
+
mean_velocity=0.0,
|
|
161
|
+
variance=1.0,
|
|
162
|
+
)
|
|
163
|
+
moment_tolerance: float = Field(default=1e-10, gt=0)
|
|
164
|
+
outcome_tolerance: float = Field(default=0.02, gt=0)
|
|
165
|
+
candidate_space: SymmetricMixtureSearchSpace = SymmetricMixtureSearchSpace()
|
|
166
|
+
evaluations_per_method: int = Field(default=8, ge=2, le=64)
|
|
167
|
+
baseline_seed: int = Field(default=20260809, ge=0)
|
|
168
|
+
comparison_methods: tuple[SearchMethod, ...] = (
|
|
169
|
+
SearchMethod.AI,
|
|
170
|
+
SearchMethod.GRID,
|
|
171
|
+
SearchMethod.RANDOM,
|
|
172
|
+
SearchMethod.LATIN_HYPERCUBE,
|
|
173
|
+
)
|
|
174
|
+
disclosure: Literal[
|
|
175
|
+
"candidate grammar and bounds only; no prior candidate evaluations or planted parameters"
|
|
176
|
+
] = "candidate grammar and bounds only; no prior candidate evaluations or planted parameters"
|
|
177
|
+
|
|
178
|
+
@model_validator(mode="after")
|
|
179
|
+
def unique_comparison_methods(self) -> BlindedSearchRequest:
|
|
180
|
+
if len(self.comparison_methods) != len(set(self.comparison_methods)):
|
|
181
|
+
raise ValueError("comparison methods must be unique")
|
|
182
|
+
if SearchMethod.AI not in self.comparison_methods:
|
|
183
|
+
raise ValueError("the autonomy checkpoint requires an AI strategy")
|
|
184
|
+
if abs(self.wavenumber - 0.5) > 1e-12:
|
|
185
|
+
raise ValueError("the first blinded challenge is preregistered at k=0.5")
|
|
186
|
+
return self
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
class CandidateSuggestion(StrictModel):
|
|
190
|
+
parameters: SymmetricMixtureCandidate
|
|
191
|
+
rationale: str = Field(min_length=1)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
class AIStrategyDraft(StrictModel):
|
|
195
|
+
schema_version: Literal["0.1.0"] = "0.1.0"
|
|
196
|
+
candidates: tuple[CandidateSuggestion, ...] = Field(min_length=1)
|
|
197
|
+
search_rationale: str = Field(min_length=1)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
class SearchModelCallProvenance(StrictModel):
|
|
201
|
+
request_id: str
|
|
202
|
+
model: str
|
|
203
|
+
route: ModelRoute
|
|
204
|
+
route_reason: str
|
|
205
|
+
finish_reason: str
|
|
206
|
+
prompt_template_version: Literal["blinded_search_v1"] = "blinded_search_v1"
|
|
207
|
+
prompt_hash: str
|
|
208
|
+
output_schema_hash: str
|
|
209
|
+
response_hash: str
|
|
210
|
+
usage: dict[str, Any] = Field(default_factory=dict)
|
|
211
|
+
admission_errors: tuple[str, ...] = ()
|
|
212
|
+
attempt_number: int = Field(ge=1)
|
|
213
|
+
created_at: datetime = Field(default_factory=utc_now)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
class CandidateProposal(StrictModel):
|
|
217
|
+
id: str
|
|
218
|
+
ordinal: int = Field(ge=1)
|
|
219
|
+
parameters: SymmetricMixtureCandidate
|
|
220
|
+
rationale: str = Field(min_length=1)
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
class SearchStrategy(StrictModel):
|
|
224
|
+
schema_version: Literal["0.1.0"] = "0.1.0"
|
|
225
|
+
request_id: str
|
|
226
|
+
method: SearchMethod
|
|
227
|
+
generator: str
|
|
228
|
+
proposals: tuple[CandidateProposal, ...] = Field(min_length=1)
|
|
229
|
+
model_calls: tuple[SearchModelCallProvenance, ...] = ()
|
|
230
|
+
|
|
231
|
+
@model_validator(mode="after")
|
|
232
|
+
def coherent_strategy(self) -> SearchStrategy:
|
|
233
|
+
if tuple(proposal.ordinal for proposal in self.proposals) != tuple(
|
|
234
|
+
range(1, len(self.proposals) + 1)
|
|
235
|
+
):
|
|
236
|
+
raise ValueError("proposal ordinals must be contiguous and one-based")
|
|
237
|
+
if len({proposal.id for proposal in self.proposals}) != len(self.proposals):
|
|
238
|
+
raise ValueError("candidate IDs must be unique")
|
|
239
|
+
if self.method is SearchMethod.AI and not self.model_calls:
|
|
240
|
+
raise ValueError("AI strategies require model-call provenance")
|
|
241
|
+
if self.method is not SearchMethod.AI and self.model_calls:
|
|
242
|
+
raise ValueError("baseline strategies cannot claim model-call provenance")
|
|
243
|
+
return self
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
class ReferenceQualification(StrictModel):
|
|
247
|
+
request_id: str
|
|
248
|
+
moments: DistributionMoments
|
|
249
|
+
mode: LinearMode
|
|
250
|
+
qualified: bool
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
class CandidateEvaluation(StrictModel):
|
|
254
|
+
request_id: str
|
|
255
|
+
method: SearchMethod
|
|
256
|
+
proposal: CandidateProposal
|
|
257
|
+
outcome: CandidateOutcome
|
|
258
|
+
moments: DistributionMoments | None = None
|
|
259
|
+
mode: LinearMode | None = None
|
|
260
|
+
reference_growth_rate: float | None = None
|
|
261
|
+
outcome_separation: float | None = Field(default=None, ge=0)
|
|
262
|
+
falsifies: bool = False
|
|
263
|
+
failure_detail: str | None = None
|
|
264
|
+
|
|
265
|
+
@model_validator(mode="after")
|
|
266
|
+
def result_fields_match_outcome(self) -> CandidateEvaluation:
|
|
267
|
+
successful_fields = (
|
|
268
|
+
self.moments,
|
|
269
|
+
self.mode,
|
|
270
|
+
self.reference_growth_rate,
|
|
271
|
+
self.outcome_separation,
|
|
272
|
+
)
|
|
273
|
+
if self.outcome is CandidateOutcome.VALID:
|
|
274
|
+
if any(value is None for value in successful_fields):
|
|
275
|
+
raise ValueError("valid evaluations require all scientific result fields")
|
|
276
|
+
if self.failure_detail is not None:
|
|
277
|
+
raise ValueError("valid evaluations cannot include a failure detail")
|
|
278
|
+
elif not self.failure_detail:
|
|
279
|
+
raise ValueError("failed evaluations require a failure detail")
|
|
280
|
+
return self
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
class MethodSearchResult(StrictModel):
|
|
284
|
+
method: SearchMethod
|
|
285
|
+
evaluations: tuple[CandidateEvaluation, ...] = Field(min_length=1)
|
|
286
|
+
first_falsifying_ordinal: int | None = Field(default=None, ge=1)
|
|
287
|
+
best_candidate_id: str | None = None
|
|
288
|
+
best_outcome_separation: float | None = Field(default=None, ge=0)
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
class BlindedSearchReport(StrictModel):
|
|
292
|
+
schema_version: Literal["0.1.0"] = "0.1.0"
|
|
293
|
+
request: BlindedSearchRequest
|
|
294
|
+
reference: ReferenceQualification
|
|
295
|
+
method_results: tuple[MethodSearchResult, ...]
|
|
296
|
+
equal_evaluation_budget: bool
|
|
297
|
+
confirmation_candidate_id: str | None = None
|
|
298
|
+
confirmation_candidate: SymmetricMixtureCandidate | None = None
|
|
299
|
+
selection_reason: str
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
class SearchStrategyRejected(ValueError):
|
|
303
|
+
def __init__(
|
|
304
|
+
self,
|
|
305
|
+
errors: tuple[str, ...],
|
|
306
|
+
model_calls: tuple[SearchModelCallProvenance, ...],
|
|
307
|
+
) -> None:
|
|
308
|
+
super().__init__("search strategy rejected: " + "; ".join(errors))
|
|
309
|
+
self.errors = errors
|
|
310
|
+
self.model_calls = model_calls
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
class SearchCompletionClient(Protocol):
|
|
314
|
+
def complete(
|
|
315
|
+
self,
|
|
316
|
+
messages: list[dict[str, str]],
|
|
317
|
+
*,
|
|
318
|
+
route: ModelRoute = ModelRoute.DEFAULT,
|
|
319
|
+
escalation_reason: str | None = None,
|
|
320
|
+
max_tokens: int = 1024,
|
|
321
|
+
temperature: float = 0.0,
|
|
322
|
+
) -> CompletionResult: ...
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def _canonical_candidate(
|
|
326
|
+
inner_pair_weight: float,
|
|
327
|
+
first_drift: float,
|
|
328
|
+
second_drift: float,
|
|
329
|
+
) -> SymmetricMixtureCandidate:
|
|
330
|
+
if first_drift <= second_drift:
|
|
331
|
+
return SymmetricMixtureCandidate(
|
|
332
|
+
inner_pair_weight=inner_pair_weight,
|
|
333
|
+
inner_drift=first_drift,
|
|
334
|
+
outer_drift=second_drift,
|
|
335
|
+
)
|
|
336
|
+
return SymmetricMixtureCandidate(
|
|
337
|
+
inner_pair_weight=1.0 - inner_pair_weight,
|
|
338
|
+
inner_drift=second_drift,
|
|
339
|
+
outer_drift=first_drift,
|
|
340
|
+
)
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def validate_candidate(
|
|
344
|
+
request: BlindedSearchRequest,
|
|
345
|
+
candidate: SymmetricMixtureCandidate,
|
|
346
|
+
) -> tuple[str, ...]:
|
|
347
|
+
space = request.candidate_space
|
|
348
|
+
errors: list[str] = []
|
|
349
|
+
checks = (
|
|
350
|
+
("inner_pair_weight", candidate.inner_pair_weight, space.inner_pair_weight),
|
|
351
|
+
("inner_drift", candidate.inner_drift, space.inner_drift),
|
|
352
|
+
("outer_drift", candidate.outer_drift, space.outer_drift),
|
|
353
|
+
)
|
|
354
|
+
for name, value, interval in checks:
|
|
355
|
+
if value < interval.minimum - 1e-12 or value > interval.maximum + 1e-12:
|
|
356
|
+
errors.append(f"{name} is outside the approved interval")
|
|
357
|
+
candidate_moments = moments(candidate.distribution())
|
|
358
|
+
for name, target in request.target_moments.model_dump().items():
|
|
359
|
+
if abs(getattr(candidate_moments, name) - target) > request.moment_tolerance:
|
|
360
|
+
errors.append(f"candidate does not satisfy target {name}")
|
|
361
|
+
return tuple(errors)
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
def _proposal(
|
|
365
|
+
request: BlindedSearchRequest,
|
|
366
|
+
method: SearchMethod,
|
|
367
|
+
ordinal: int,
|
|
368
|
+
candidate: SymmetricMixtureCandidate,
|
|
369
|
+
rationale: str,
|
|
370
|
+
) -> CandidateProposal:
|
|
371
|
+
errors = validate_candidate(request, candidate)
|
|
372
|
+
if errors:
|
|
373
|
+
raise ValueError("; ".join(errors))
|
|
374
|
+
identity = hashlib.sha256(
|
|
375
|
+
f"{request.id}:{method.value}:{ordinal}:{candidate.distribution_hash()}".encode()
|
|
376
|
+
).hexdigest()[:16]
|
|
377
|
+
return CandidateProposal(
|
|
378
|
+
id=f"candidate_{method.value}_{identity}",
|
|
379
|
+
ordinal=ordinal,
|
|
380
|
+
parameters=candidate,
|
|
381
|
+
rationale=rationale,
|
|
382
|
+
)
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def validate_strategy(
|
|
386
|
+
request: BlindedSearchRequest,
|
|
387
|
+
strategy: SearchStrategy,
|
|
388
|
+
) -> tuple[str, ...]:
|
|
389
|
+
errors: list[str] = []
|
|
390
|
+
if strategy.request_id != request.id:
|
|
391
|
+
errors.append("strategy targets a different request")
|
|
392
|
+
if strategy.method not in request.comparison_methods:
|
|
393
|
+
errors.append("strategy method is outside the comparison contract")
|
|
394
|
+
if len(strategy.proposals) != request.evaluations_per_method:
|
|
395
|
+
errors.append("strategy does not use the exact evaluation budget")
|
|
396
|
+
distribution_hashes: set[str] = set()
|
|
397
|
+
for proposal in strategy.proposals:
|
|
398
|
+
errors.extend(validate_candidate(request, proposal.parameters))
|
|
399
|
+
fingerprint = proposal.parameters.distribution_hash()
|
|
400
|
+
if fingerprint in distribution_hashes:
|
|
401
|
+
errors.append("strategy contains a duplicate physical distribution")
|
|
402
|
+
distribution_hashes.add(fingerprint)
|
|
403
|
+
return tuple(dict.fromkeys(errors))
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def admit_ai_strategy_draft(
|
|
407
|
+
request: BlindedSearchRequest,
|
|
408
|
+
draft: AIStrategyDraft,
|
|
409
|
+
model_calls: tuple[SearchModelCallProvenance, ...],
|
|
410
|
+
*,
|
|
411
|
+
generator: str = "openai_compatible_model",
|
|
412
|
+
) -> SearchStrategy:
|
|
413
|
+
proposals = tuple(
|
|
414
|
+
_proposal(
|
|
415
|
+
request,
|
|
416
|
+
SearchMethod.AI,
|
|
417
|
+
ordinal,
|
|
418
|
+
suggestion.parameters,
|
|
419
|
+
suggestion.rationale,
|
|
420
|
+
)
|
|
421
|
+
for ordinal, suggestion in enumerate(draft.candidates, start=1)
|
|
422
|
+
)
|
|
423
|
+
strategy = SearchStrategy(
|
|
424
|
+
request_id=request.id,
|
|
425
|
+
method=SearchMethod.AI,
|
|
426
|
+
generator=generator,
|
|
427
|
+
proposals=proposals,
|
|
428
|
+
model_calls=model_calls,
|
|
429
|
+
)
|
|
430
|
+
errors = validate_strategy(request, strategy)
|
|
431
|
+
if errors:
|
|
432
|
+
raise ValueError("; ".join(errors))
|
|
433
|
+
return strategy
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def offline_ai_fixture_strategy(request: BlindedSearchRequest) -> SearchStrategy:
|
|
437
|
+
"""Deterministic CI fixture; this is not evidence of a live model call."""
|
|
438
|
+
|
|
439
|
+
values = (
|
|
440
|
+
(0.50, 0.85, 0.95),
|
|
441
|
+
(0.50, 0.70, 0.90),
|
|
442
|
+
(0.20, 0.10, 0.92),
|
|
443
|
+
(0.80, 0.60, 0.94),
|
|
444
|
+
(0.40, 0.30, 0.80),
|
|
445
|
+
(0.60, 0.50, 0.85),
|
|
446
|
+
(0.30, 0.20, 0.70),
|
|
447
|
+
(0.70, 0.10, 0.60),
|
|
448
|
+
)
|
|
449
|
+
if request.evaluations_per_method != len(values):
|
|
450
|
+
raise ValueError("offline AI fixture is defined only for an eight-evaluation budget")
|
|
451
|
+
draft = AIStrategyDraft(
|
|
452
|
+
candidates=tuple(
|
|
453
|
+
CandidateSuggestion(
|
|
454
|
+
parameters=SymmetricMixtureCandidate(
|
|
455
|
+
inner_pair_weight=weight,
|
|
456
|
+
inner_drift=inner,
|
|
457
|
+
outer_drift=outer,
|
|
458
|
+
),
|
|
459
|
+
rationale="offline physics-guided CI fixture",
|
|
460
|
+
)
|
|
461
|
+
for weight, inner, outer in values
|
|
462
|
+
),
|
|
463
|
+
search_rationale="exercise the blinded-search boundary without a network dependency",
|
|
464
|
+
)
|
|
465
|
+
provenance = SearchModelCallProvenance(
|
|
466
|
+
request_id="offline_fixture_no_provider_request",
|
|
467
|
+
model="offline-physics-heuristic-fixture",
|
|
468
|
+
route=ModelRoute.DEFAULT,
|
|
469
|
+
route_reason="deterministic CI fixture; no model was called",
|
|
470
|
+
finish_reason="stop",
|
|
471
|
+
prompt_hash="0" * 64,
|
|
472
|
+
output_schema_hash="0" * 64,
|
|
473
|
+
response_hash=hashlib.sha256(draft.model_dump_json().encode()).hexdigest(),
|
|
474
|
+
usage={"total_tokens": 0},
|
|
475
|
+
attempt_number=1,
|
|
476
|
+
created_at=datetime(2026, 8, 9, tzinfo=UTC),
|
|
477
|
+
)
|
|
478
|
+
return admit_ai_strategy_draft(
|
|
479
|
+
request,
|
|
480
|
+
draft,
|
|
481
|
+
(provenance,),
|
|
482
|
+
generator="offline_fixture_not_a_live_model",
|
|
483
|
+
)
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
def offline_qualified_warpx_fixture_strategy(
|
|
487
|
+
request: BlindedSearchRequest,
|
|
488
|
+
) -> SearchStrategy:
|
|
489
|
+
"""CI-only batch whose expected winner exactly matches the qualified WarpX scope."""
|
|
490
|
+
|
|
491
|
+
values = (
|
|
492
|
+
(0.50, 0.95, 0.95),
|
|
493
|
+
(0.10, 0.00, 0.95),
|
|
494
|
+
(0.10, 0.45, 0.95),
|
|
495
|
+
(0.50, 0.80, 0.80),
|
|
496
|
+
(0.50, 0.50, 0.95),
|
|
497
|
+
(0.50, 0.60, 0.80),
|
|
498
|
+
(0.90, 0.00, 0.70),
|
|
499
|
+
(0.90, 0.20, 0.60),
|
|
500
|
+
)
|
|
501
|
+
if request.evaluations_per_method != len(values):
|
|
502
|
+
raise ValueError("qualified WarpX fixture requires an eight-evaluation budget")
|
|
503
|
+
draft = AIStrategyDraft(
|
|
504
|
+
candidates=tuple(
|
|
505
|
+
CandidateSuggestion(
|
|
506
|
+
parameters=SymmetricMixtureCandidate(
|
|
507
|
+
inner_pair_weight=weight,
|
|
508
|
+
inner_drift=inner,
|
|
509
|
+
outer_drift=outer,
|
|
510
|
+
),
|
|
511
|
+
rationale="offline qualified-WarpX integration fixture",
|
|
512
|
+
)
|
|
513
|
+
for weight, inner, outer in values
|
|
514
|
+
),
|
|
515
|
+
search_rationale=(
|
|
516
|
+
"exercise the qualified campaign boundary without claiming a live model call"
|
|
517
|
+
),
|
|
518
|
+
)
|
|
519
|
+
provenance = SearchModelCallProvenance(
|
|
520
|
+
request_id="offline_qualified_warpx_fixture_no_provider_request",
|
|
521
|
+
model="offline-qualified-warpx-fixture",
|
|
522
|
+
route=ModelRoute.DEFAULT,
|
|
523
|
+
route_reason="deterministic CI fixture; no model was called",
|
|
524
|
+
finish_reason="stop",
|
|
525
|
+
prompt_hash="0" * 64,
|
|
526
|
+
output_schema_hash="0" * 64,
|
|
527
|
+
response_hash=hashlib.sha256(draft.model_dump_json().encode()).hexdigest(),
|
|
528
|
+
usage={"total_tokens": 0},
|
|
529
|
+
attempt_number=1,
|
|
530
|
+
created_at=datetime(2026, 8, 9, tzinfo=UTC),
|
|
531
|
+
)
|
|
532
|
+
return admit_ai_strategy_draft(
|
|
533
|
+
request,
|
|
534
|
+
draft,
|
|
535
|
+
(provenance,),
|
|
536
|
+
generator="offline_qualified_warpx_fixture_not_a_live_model",
|
|
537
|
+
)
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
class AISearchStrategyGenerator:
|
|
541
|
+
def __init__(self, client: SearchCompletionClient, *, max_attempts: int = 2) -> None:
|
|
542
|
+
if max_attempts < 1:
|
|
543
|
+
raise ValueError("max_attempts must be positive")
|
|
544
|
+
self.client = client
|
|
545
|
+
self.max_attempts = max_attempts
|
|
546
|
+
|
|
547
|
+
@staticmethod
|
|
548
|
+
def _messages(request: BlindedSearchRequest) -> list[dict[str, str]]:
|
|
549
|
+
return [
|
|
550
|
+
{
|
|
551
|
+
"role": "system",
|
|
552
|
+
"content": (
|
|
553
|
+
"You are selecting a bounded batch of candidate distributions for a "
|
|
554
|
+
"computational-physics search. Return exactly one JSON object matching "
|
|
555
|
+
"the supplied schema, with no Markdown. Use exactly the stated candidate "
|
|
556
|
+
"budget and order candidates intentionally. You have no evaluation "
|
|
557
|
+
"results, hidden benchmark parameters, or permission to change the "
|
|
558
|
+
"grammar. You propose candidates only; deterministic tools evaluate them."
|
|
559
|
+
),
|
|
560
|
+
},
|
|
561
|
+
{
|
|
562
|
+
"role": "user",
|
|
563
|
+
"content": json.dumps(
|
|
564
|
+
{
|
|
565
|
+
"request": request.model_dump(mode="json"),
|
|
566
|
+
"output_schema": AIStrategyDraft.model_json_schema(),
|
|
567
|
+
},
|
|
568
|
+
sort_keys=True,
|
|
569
|
+
),
|
|
570
|
+
},
|
|
571
|
+
]
|
|
572
|
+
|
|
573
|
+
@staticmethod
|
|
574
|
+
def _provenance(
|
|
575
|
+
result: CompletionResult,
|
|
576
|
+
messages: list[dict[str, str]],
|
|
577
|
+
attempt_number: int,
|
|
578
|
+
) -> SearchModelCallProvenance:
|
|
579
|
+
prompt = json.dumps(messages, sort_keys=True, separators=(",", ":"))
|
|
580
|
+
schema = json.dumps(
|
|
581
|
+
AIStrategyDraft.model_json_schema(),
|
|
582
|
+
sort_keys=True,
|
|
583
|
+
separators=(",", ":"),
|
|
584
|
+
)
|
|
585
|
+
return SearchModelCallProvenance(
|
|
586
|
+
request_id=result.request_id,
|
|
587
|
+
model=result.model,
|
|
588
|
+
route=result.route,
|
|
589
|
+
route_reason=result.route_reason,
|
|
590
|
+
finish_reason=result.finish_reason,
|
|
591
|
+
prompt_hash=hashlib.sha256(prompt.encode()).hexdigest(),
|
|
592
|
+
output_schema_hash=hashlib.sha256(schema.encode()).hexdigest(),
|
|
593
|
+
response_hash=hashlib.sha256(result.content.encode()).hexdigest(),
|
|
594
|
+
usage=result.usage,
|
|
595
|
+
attempt_number=attempt_number,
|
|
596
|
+
)
|
|
597
|
+
|
|
598
|
+
def generate(
|
|
599
|
+
self,
|
|
600
|
+
request: BlindedSearchRequest,
|
|
601
|
+
*,
|
|
602
|
+
route: ModelRoute = ModelRoute.DEFAULT,
|
|
603
|
+
escalation_reason: str | None = None,
|
|
604
|
+
max_tokens: int = 16384,
|
|
605
|
+
) -> SearchStrategy:
|
|
606
|
+
messages = self._messages(request)
|
|
607
|
+
calls: list[SearchModelCallProvenance] = []
|
|
608
|
+
last_errors: tuple[str, ...] = ("model returned no strategy",)
|
|
609
|
+
attempt_limit = 1 if route is ModelRoute.ESCALATION else self.max_attempts
|
|
610
|
+
for attempt_number in range(1, attempt_limit + 1):
|
|
611
|
+
result = self.client.complete(
|
|
612
|
+
messages,
|
|
613
|
+
route=route,
|
|
614
|
+
escalation_reason=escalation_reason,
|
|
615
|
+
max_tokens=max_tokens,
|
|
616
|
+
temperature=0.0,
|
|
617
|
+
)
|
|
618
|
+
calls.append(self._provenance(result, messages, attempt_number))
|
|
619
|
+
draft: AIStrategyDraft | None = None
|
|
620
|
+
if result.finish_reason != "stop":
|
|
621
|
+
last_errors = (f"incomplete completion: {result.finish_reason}",)
|
|
622
|
+
else:
|
|
623
|
+
try:
|
|
624
|
+
draft = AIStrategyDraft.model_validate_json(result.content)
|
|
625
|
+
except ValidationError as error:
|
|
626
|
+
last_errors = (f"invalid strategy document: {error}",)
|
|
627
|
+
if draft is not None:
|
|
628
|
+
try:
|
|
629
|
+
strategy = admit_ai_strategy_draft(request, draft, tuple(calls))
|
|
630
|
+
except (ValueError, ValidationError) as error:
|
|
631
|
+
last_errors = (f"invalid admitted strategy: {error}",)
|
|
632
|
+
else:
|
|
633
|
+
return strategy
|
|
634
|
+
calls[-1] = calls[-1].model_copy(update={"admission_errors": last_errors})
|
|
635
|
+
messages.extend(
|
|
636
|
+
(
|
|
637
|
+
{"role": "assistant", "content": result.content},
|
|
638
|
+
{
|
|
639
|
+
"role": "user",
|
|
640
|
+
"content": (
|
|
641
|
+
"The deterministic admission gate rejected the strategy for: "
|
|
642
|
+
+ "; ".join(last_errors)
|
|
643
|
+
+ ". Return a complete corrected JSON object only."
|
|
644
|
+
),
|
|
645
|
+
},
|
|
646
|
+
)
|
|
647
|
+
)
|
|
648
|
+
raise SearchStrategyRejected(last_errors, tuple(calls))
|
|
649
|
+
|
|
650
|
+
|
|
651
|
+
def _strategy_from_candidates(
|
|
652
|
+
request: BlindedSearchRequest,
|
|
653
|
+
method: SearchMethod,
|
|
654
|
+
candidates: list[SymmetricMixtureCandidate],
|
|
655
|
+
generator: str,
|
|
656
|
+
) -> SearchStrategy:
|
|
657
|
+
proposals = tuple(
|
|
658
|
+
_proposal(
|
|
659
|
+
request,
|
|
660
|
+
method,
|
|
661
|
+
ordinal,
|
|
662
|
+
candidate,
|
|
663
|
+
f"preregistered {method.value} baseline candidate {ordinal}",
|
|
664
|
+
)
|
|
665
|
+
for ordinal, candidate in enumerate(candidates, start=1)
|
|
666
|
+
)
|
|
667
|
+
strategy = SearchStrategy(
|
|
668
|
+
request_id=request.id,
|
|
669
|
+
method=method,
|
|
670
|
+
generator=generator,
|
|
671
|
+
proposals=proposals,
|
|
672
|
+
)
|
|
673
|
+
errors = validate_strategy(request, strategy)
|
|
674
|
+
if errors:
|
|
675
|
+
raise RuntimeError("invalid deterministic baseline: " + "; ".join(errors))
|
|
676
|
+
return strategy
|
|
677
|
+
|
|
678
|
+
|
|
679
|
+
def grid_strategy(request: BlindedSearchRequest) -> SearchStrategy:
|
|
680
|
+
"""Traverse a growing Cartesian lattice in declared coordinate order."""
|
|
681
|
+
|
|
682
|
+
space = request.candidate_space
|
|
683
|
+
candidates: list[SymmetricMixtureCandidate] = []
|
|
684
|
+
seen: set[str] = set()
|
|
685
|
+
resolution = 2
|
|
686
|
+
while len(candidates) < request.evaluations_per_method:
|
|
687
|
+
levels = np.linspace(0.0, 1.0, resolution)
|
|
688
|
+
for weight_u, first_u, second_u in itertools.product(levels, repeat=3):
|
|
689
|
+
candidate = _canonical_candidate(
|
|
690
|
+
space.inner_pair_weight.map_unit(float(weight_u)),
|
|
691
|
+
space.inner_drift.map_unit(float(first_u)),
|
|
692
|
+
space.outer_drift.map_unit(float(second_u)),
|
|
693
|
+
)
|
|
694
|
+
fingerprint = candidate.distribution_hash()
|
|
695
|
+
if fingerprint in seen:
|
|
696
|
+
continue
|
|
697
|
+
seen.add(fingerprint)
|
|
698
|
+
candidates.append(candidate)
|
|
699
|
+
if len(candidates) == request.evaluations_per_method:
|
|
700
|
+
break
|
|
701
|
+
resolution += 1
|
|
702
|
+
return _strategy_from_candidates(
|
|
703
|
+
request,
|
|
704
|
+
SearchMethod.GRID,
|
|
705
|
+
candidates,
|
|
706
|
+
"deterministic_cartesian_lattice",
|
|
707
|
+
)
|
|
708
|
+
|
|
709
|
+
|
|
710
|
+
def random_strategy(request: BlindedSearchRequest) -> SearchStrategy:
|
|
711
|
+
rng = np.random.default_rng(request.baseline_seed)
|
|
712
|
+
space = request.candidate_space
|
|
713
|
+
candidates: list[SymmetricMixtureCandidate] = []
|
|
714
|
+
seen: set[str] = set()
|
|
715
|
+
while len(candidates) < request.evaluations_per_method:
|
|
716
|
+
unit = rng.random(3)
|
|
717
|
+
candidate = _canonical_candidate(
|
|
718
|
+
space.inner_pair_weight.map_unit(float(unit[0])),
|
|
719
|
+
space.inner_drift.map_unit(float(unit[1])),
|
|
720
|
+
space.outer_drift.map_unit(float(unit[2])),
|
|
721
|
+
)
|
|
722
|
+
fingerprint = candidate.distribution_hash()
|
|
723
|
+
if fingerprint not in seen:
|
|
724
|
+
seen.add(fingerprint)
|
|
725
|
+
candidates.append(candidate)
|
|
726
|
+
return _strategy_from_candidates(
|
|
727
|
+
request,
|
|
728
|
+
SearchMethod.RANDOM,
|
|
729
|
+
candidates,
|
|
730
|
+
f"numpy_pcg64_seed_{request.baseline_seed}",
|
|
731
|
+
)
|
|
732
|
+
|
|
733
|
+
|
|
734
|
+
def latin_hypercube_strategy(request: BlindedSearchRequest) -> SearchStrategy:
|
|
735
|
+
sampler = qmc.LatinHypercube(d=3, seed=request.baseline_seed)
|
|
736
|
+
samples = sampler.random(request.evaluations_per_method)
|
|
737
|
+
space = request.candidate_space
|
|
738
|
+
candidates = [
|
|
739
|
+
_canonical_candidate(
|
|
740
|
+
space.inner_pair_weight.map_unit(float(unit[0])),
|
|
741
|
+
space.inner_drift.map_unit(float(unit[1])),
|
|
742
|
+
space.outer_drift.map_unit(float(unit[2])),
|
|
743
|
+
)
|
|
744
|
+
for unit in samples
|
|
745
|
+
]
|
|
746
|
+
return _strategy_from_candidates(
|
|
747
|
+
request,
|
|
748
|
+
SearchMethod.LATIN_HYPERCUBE,
|
|
749
|
+
candidates,
|
|
750
|
+
f"scipy_latin_hypercube_seed_{request.baseline_seed}",
|
|
751
|
+
)
|
|
752
|
+
|
|
753
|
+
|
|
754
|
+
def baseline_strategies(request: BlindedSearchRequest) -> tuple[SearchStrategy, ...]:
|
|
755
|
+
factories = {
|
|
756
|
+
SearchMethod.GRID: grid_strategy,
|
|
757
|
+
SearchMethod.RANDOM: random_strategy,
|
|
758
|
+
SearchMethod.LATIN_HYPERCUBE: latin_hypercube_strategy,
|
|
759
|
+
}
|
|
760
|
+
return tuple(
|
|
761
|
+
factories[method](request)
|
|
762
|
+
for method in request.comparison_methods
|
|
763
|
+
if method is not SearchMethod.AI
|
|
764
|
+
)
|
|
765
|
+
|
|
766
|
+
|
|
767
|
+
def qualify_reference(request: BlindedSearchRequest) -> ReferenceQualification:
|
|
768
|
+
reference_moments = moments(request.reference_distribution)
|
|
769
|
+
target = request.target_moments
|
|
770
|
+
qualified = all(
|
|
771
|
+
abs(getattr(reference_moments, name) - getattr(target, name))
|
|
772
|
+
<= request.moment_tolerance
|
|
773
|
+
for name in DistributionMoments.model_fields
|
|
774
|
+
)
|
|
775
|
+
mode = solve_modes(request.reference_distribution, wavenumber=request.wavenumber)[0]
|
|
776
|
+
return ReferenceQualification(
|
|
777
|
+
request_id=request.id,
|
|
778
|
+
moments=reference_moments,
|
|
779
|
+
mode=mode,
|
|
780
|
+
qualified=qualified and mode.dielectric_residual <= 1e-8,
|
|
781
|
+
)
|
|
782
|
+
|
|
783
|
+
|
|
784
|
+
def evaluate_candidate(
|
|
785
|
+
request: BlindedSearchRequest,
|
|
786
|
+
reference: ReferenceQualification,
|
|
787
|
+
method: SearchMethod,
|
|
788
|
+
proposal: CandidateProposal,
|
|
789
|
+
) -> CandidateEvaluation:
|
|
790
|
+
try:
|
|
791
|
+
candidate_moments = moments(proposal.parameters.distribution())
|
|
792
|
+
candidate_mode = solve_modes(
|
|
793
|
+
proposal.parameters.distribution(),
|
|
794
|
+
wavenumber=request.wavenumber,
|
|
795
|
+
)[0]
|
|
796
|
+
except (RuntimeError, ValueError, FloatingPointError) as error:
|
|
797
|
+
return CandidateEvaluation(
|
|
798
|
+
request_id=request.id,
|
|
799
|
+
method=method,
|
|
800
|
+
proposal=proposal,
|
|
801
|
+
outcome=CandidateOutcome.NUMERICAL_FAILURE,
|
|
802
|
+
failure_detail=str(error),
|
|
803
|
+
)
|
|
804
|
+
moment_match = all(
|
|
805
|
+
abs(getattr(candidate_moments, name) - getattr(request.target_moments, name))
|
|
806
|
+
<= request.moment_tolerance
|
|
807
|
+
for name in DistributionMoments.model_fields
|
|
808
|
+
)
|
|
809
|
+
separation = abs(candidate_mode.growth_rate - reference.mode.growth_rate)
|
|
810
|
+
return CandidateEvaluation(
|
|
811
|
+
request_id=request.id,
|
|
812
|
+
method=method,
|
|
813
|
+
proposal=proposal,
|
|
814
|
+
outcome=CandidateOutcome.VALID,
|
|
815
|
+
moments=candidate_moments,
|
|
816
|
+
mode=candidate_mode,
|
|
817
|
+
reference_growth_rate=reference.mode.growth_rate,
|
|
818
|
+
outcome_separation=separation,
|
|
819
|
+
falsifies=moment_match and separation > request.outcome_tolerance,
|
|
820
|
+
)
|
|
821
|
+
|
|
822
|
+
|
|
823
|
+
def _method_result(
|
|
824
|
+
method: SearchMethod,
|
|
825
|
+
evaluations: tuple[CandidateEvaluation, ...],
|
|
826
|
+
) -> MethodSearchResult:
|
|
827
|
+
valid = [
|
|
828
|
+
evaluation
|
|
829
|
+
for evaluation in evaluations
|
|
830
|
+
if evaluation.outcome is CandidateOutcome.VALID
|
|
831
|
+
and evaluation.outcome_separation is not None
|
|
832
|
+
]
|
|
833
|
+
best = max(valid, key=lambda evaluation: evaluation.outcome_separation or -1.0, default=None)
|
|
834
|
+
first = next(
|
|
835
|
+
(evaluation.proposal.ordinal for evaluation in valid if evaluation.falsifies),
|
|
836
|
+
None,
|
|
837
|
+
)
|
|
838
|
+
return MethodSearchResult(
|
|
839
|
+
method=method,
|
|
840
|
+
evaluations=evaluations,
|
|
841
|
+
first_falsifying_ordinal=first,
|
|
842
|
+
best_candidate_id=best.proposal.id if best else None,
|
|
843
|
+
best_outcome_separation=best.outcome_separation if best else None,
|
|
844
|
+
)
|
|
845
|
+
|
|
846
|
+
|
|
847
|
+
def build_search_report(
|
|
848
|
+
request: BlindedSearchRequest,
|
|
849
|
+
reference: ReferenceQualification,
|
|
850
|
+
evaluations: dict[SearchMethod, tuple[CandidateEvaluation, ...]],
|
|
851
|
+
) -> BlindedSearchReport:
|
|
852
|
+
results = tuple(
|
|
853
|
+
_method_result(method, evaluations[method]) for method in request.comparison_methods
|
|
854
|
+
)
|
|
855
|
+
counts = {len(result.evaluations) for result in results}
|
|
856
|
+
equal_budget = counts == {request.evaluations_per_method}
|
|
857
|
+
ai_result = next(result for result in results if result.method is SearchMethod.AI)
|
|
858
|
+
selected_evaluation: CandidateEvaluation | None = None
|
|
859
|
+
if ai_result.first_falsifying_ordinal is not None:
|
|
860
|
+
ai_valid_witnesses = [
|
|
861
|
+
evaluation for evaluation in ai_result.evaluations if evaluation.falsifies
|
|
862
|
+
]
|
|
863
|
+
selected_evaluation = max(
|
|
864
|
+
ai_valid_witnesses,
|
|
865
|
+
key=lambda evaluation: evaluation.outcome_separation or -1.0,
|
|
866
|
+
)
|
|
867
|
+
reason = "best falsifying AI candidate, frozen before independent confirmation"
|
|
868
|
+
else:
|
|
869
|
+
reason = "AI strategy found no falsifying candidate within the declared budget"
|
|
870
|
+
return BlindedSearchReport(
|
|
871
|
+
request=request,
|
|
872
|
+
reference=reference,
|
|
873
|
+
method_results=results,
|
|
874
|
+
equal_evaluation_budget=equal_budget,
|
|
875
|
+
confirmation_candidate_id=(
|
|
876
|
+
selected_evaluation.proposal.id if selected_evaluation is not None else None
|
|
877
|
+
),
|
|
878
|
+
confirmation_candidate=(
|
|
879
|
+
selected_evaluation.proposal.parameters if selected_evaluation is not None else None
|
|
880
|
+
),
|
|
881
|
+
selection_reason=reason,
|
|
882
|
+
)
|
|
883
|
+
|
|
884
|
+
|
|
885
|
+
class SearchProjection:
|
|
886
|
+
def __init__(self) -> None:
|
|
887
|
+
self.request: BlindedSearchRequest | None = None
|
|
888
|
+
self.reference: ReferenceQualification | None = None
|
|
889
|
+
self.strategies: dict[SearchMethod, SearchStrategy] = {}
|
|
890
|
+
self.evaluations: dict[SearchMethod, dict[int, CandidateEvaluation]] = {}
|
|
891
|
+
self.report: BlindedSearchReport | None = None
|
|
892
|
+
|
|
893
|
+
@classmethod
|
|
894
|
+
def replay(cls, events: tuple[StoredEvent, ...]) -> SearchProjection:
|
|
895
|
+
state = cls()
|
|
896
|
+
for event in events:
|
|
897
|
+
if event.event_type == "search_created":
|
|
898
|
+
state.request = BlindedSearchRequest.model_validate(event.payload["request"])
|
|
899
|
+
elif event.event_type == "reference_qualified":
|
|
900
|
+
state.reference = ReferenceQualification.model_validate(event.payload["reference"])
|
|
901
|
+
elif event.event_type == "search_strategy_admitted":
|
|
902
|
+
strategy = SearchStrategy.model_validate(event.payload["strategy"])
|
|
903
|
+
state.strategies[strategy.method] = strategy
|
|
904
|
+
elif event.event_type == "candidate_evaluated":
|
|
905
|
+
evaluation = CandidateEvaluation.model_validate(event.payload["evaluation"])
|
|
906
|
+
state.evaluations.setdefault(evaluation.method, {})[
|
|
907
|
+
evaluation.proposal.ordinal
|
|
908
|
+
] = evaluation
|
|
909
|
+
elif event.event_type == "search_completed":
|
|
910
|
+
state.report = BlindedSearchReport.model_validate(event.payload["report"])
|
|
911
|
+
return state
|
|
912
|
+
|
|
913
|
+
|
|
914
|
+
class BlindedSearchRunner:
|
|
915
|
+
"""Replayable evaluator; model calls occur before this boundary, never inside it."""
|
|
916
|
+
|
|
917
|
+
def __init__(
|
|
918
|
+
self,
|
|
919
|
+
*,
|
|
920
|
+
campaign_id: str,
|
|
921
|
+
ledger: SQLiteEventLedger,
|
|
922
|
+
request: BlindedSearchRequest,
|
|
923
|
+
strategies: tuple[SearchStrategy, ...] = (),
|
|
924
|
+
) -> None:
|
|
925
|
+
self.campaign_id = campaign_id
|
|
926
|
+
self.ledger = ledger
|
|
927
|
+
self.request = request
|
|
928
|
+
self.supplied_strategies = {strategy.method: strategy for strategy in strategies}
|
|
929
|
+
if len(self.supplied_strategies) != len(strategies):
|
|
930
|
+
raise ValueError("only one strategy per method may be supplied")
|
|
931
|
+
|
|
932
|
+
def _state(self) -> SearchProjection:
|
|
933
|
+
return SearchProjection.replay(self.ledger.load(self.campaign_id))
|
|
934
|
+
|
|
935
|
+
def _append(
|
|
936
|
+
self,
|
|
937
|
+
event_type: str,
|
|
938
|
+
aggregate_type: str,
|
|
939
|
+
aggregate_id: str,
|
|
940
|
+
payload: dict[str, object],
|
|
941
|
+
idempotency_suffix: str,
|
|
942
|
+
) -> None:
|
|
943
|
+
self.ledger.append(
|
|
944
|
+
campaign_id=self.campaign_id,
|
|
945
|
+
event_type=event_type,
|
|
946
|
+
aggregate_type=aggregate_type,
|
|
947
|
+
aggregate_id=aggregate_id,
|
|
948
|
+
payload=payload,
|
|
949
|
+
idempotency_key=f"{self.campaign_id}:{idempotency_suffix}",
|
|
950
|
+
)
|
|
951
|
+
|
|
952
|
+
def run(self) -> BlindedSearchReport:
|
|
953
|
+
state = self._state()
|
|
954
|
+
if state.request is not None and state.request != self.request:
|
|
955
|
+
raise ValueError("search campaign already exists with a different request")
|
|
956
|
+
for method, recorded in state.strategies.items():
|
|
957
|
+
supplied = self.supplied_strategies.get(method)
|
|
958
|
+
if supplied is not None and supplied != recorded:
|
|
959
|
+
raise ValueError(f"recorded {method.value} strategy differs from supplied strategy")
|
|
960
|
+
if state.report is not None:
|
|
961
|
+
return state.report
|
|
962
|
+
|
|
963
|
+
if state.request is None:
|
|
964
|
+
self._append(
|
|
965
|
+
"search_created",
|
|
966
|
+
"search",
|
|
967
|
+
self.request.id,
|
|
968
|
+
{"request": self.request.model_dump(mode="json")},
|
|
969
|
+
"search-created",
|
|
970
|
+
)
|
|
971
|
+
state = self._state()
|
|
972
|
+
|
|
973
|
+
if state.reference is None:
|
|
974
|
+
reference = qualify_reference(self.request)
|
|
975
|
+
if not reference.qualified:
|
|
976
|
+
raise RuntimeError("reference distribution failed qualification")
|
|
977
|
+
self._append(
|
|
978
|
+
"reference_qualified",
|
|
979
|
+
"reference",
|
|
980
|
+
self.request.id,
|
|
981
|
+
{"reference": reference.model_dump(mode="json")},
|
|
982
|
+
"reference-qualified",
|
|
983
|
+
)
|
|
984
|
+
state = self._state()
|
|
985
|
+
assert state.reference is not None
|
|
986
|
+
|
|
987
|
+
for method in self.request.comparison_methods:
|
|
988
|
+
if method in state.strategies:
|
|
989
|
+
continue
|
|
990
|
+
strategy = self.supplied_strategies.get(method)
|
|
991
|
+
if strategy is None:
|
|
992
|
+
raise RuntimeError(f"missing preregistered {method.value} strategy")
|
|
993
|
+
errors = validate_strategy(self.request, strategy)
|
|
994
|
+
if errors:
|
|
995
|
+
raise ValueError("; ".join(errors))
|
|
996
|
+
self._append(
|
|
997
|
+
"search_strategy_admitted",
|
|
998
|
+
"search_strategy",
|
|
999
|
+
f"{self.request.id}:{method.value}",
|
|
1000
|
+
{"strategy": strategy.model_dump(mode="json")},
|
|
1001
|
+
f"strategy:{method.value}",
|
|
1002
|
+
)
|
|
1003
|
+
state = self._state()
|
|
1004
|
+
|
|
1005
|
+
for method in self.request.comparison_methods:
|
|
1006
|
+
strategy = state.strategies[method]
|
|
1007
|
+
existing = state.evaluations.get(method, {})
|
|
1008
|
+
for proposal in strategy.proposals:
|
|
1009
|
+
if proposal.ordinal in existing:
|
|
1010
|
+
continue
|
|
1011
|
+
evaluation = evaluate_candidate(
|
|
1012
|
+
self.request,
|
|
1013
|
+
state.reference,
|
|
1014
|
+
method,
|
|
1015
|
+
proposal,
|
|
1016
|
+
)
|
|
1017
|
+
self._append(
|
|
1018
|
+
"candidate_evaluated",
|
|
1019
|
+
"candidate",
|
|
1020
|
+
proposal.id,
|
|
1021
|
+
{"evaluation": evaluation.model_dump(mode="json")},
|
|
1022
|
+
f"evaluation:{method.value}:{proposal.ordinal}",
|
|
1023
|
+
)
|
|
1024
|
+
state = self._state()
|
|
1025
|
+
existing = state.evaluations.get(method, {})
|
|
1026
|
+
|
|
1027
|
+
state = self._state()
|
|
1028
|
+
ordered_evaluations = {
|
|
1029
|
+
method: tuple(
|
|
1030
|
+
state.evaluations[method][ordinal]
|
|
1031
|
+
for ordinal in range(1, self.request.evaluations_per_method + 1)
|
|
1032
|
+
)
|
|
1033
|
+
for method in self.request.comparison_methods
|
|
1034
|
+
}
|
|
1035
|
+
report = build_search_report(
|
|
1036
|
+
self.request,
|
|
1037
|
+
state.reference,
|
|
1038
|
+
ordered_evaluations,
|
|
1039
|
+
)
|
|
1040
|
+
self._append(
|
|
1041
|
+
"search_completed",
|
|
1042
|
+
"search",
|
|
1043
|
+
self.request.id,
|
|
1044
|
+
{"report": report.model_dump(mode="json")},
|
|
1045
|
+
"search-completed",
|
|
1046
|
+
)
|
|
1047
|
+
return self._state().report or report
|