eif-engine 4.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eif/__init__.py +23 -0
- eif/auto.py +199 -0
- eif/calibrate/__init__.py +15 -0
- eif/calibrate/bayesian.py +53 -0
- eif/calibrate/conformal.py +57 -0
- eif/calibrate/domain_constraints.py +169 -0
- eif/calibrate/ece.py +154 -0
- eif/calibrate/prior_strategy.py +38 -0
- eif/calibrate/trust.py +120 -0
- eif/causal_gate/__init__.py +31 -0
- eif/causal_gate/confound.py +196 -0
- eif/causal_gate/direction.py +28 -0
- eif/causal_gate/eif_causal_gate_v4_intent.yaml +265 -0
- eif/causal_gate/evidence_probe.py +280 -0
- eif/causal_gate/intervention.py +55 -0
- eif/causal_gate/iris_pipeline.py +334 -0
- eif/causal_gate/verdict.py +47 -0
- eif/challenge/__init__.py +17 -0
- eif/challenge/diversity.py +58 -0
- eif/challenge/eif_multi_critic_intent.yaml +198 -0
- eif/challenge/multi_critic.py +270 -0
- eif/challenge/protocol.py +86 -0
- eif/challenge/replicator.py +379 -0
- eif/cost_model/__init__.py +5 -0
- eif/cost_model/catch_rate_empirical.py +275 -0
- eif/cost_model/cost_ledger.py +107 -0
- eif/cost_model/eif_catchrate_intent.yaml +86 -0
- eif/declare/__init__.py +6 -0
- eif/declare/extractor.py +308 -0
- eif/declare/harking_guard.py +110 -0
- eif/declare/registry.py +146 -0
- eif/eif_engine_intent.yaml +296 -0
- eif/eif_v2_intent.yaml +779 -0
- eif/eif_v3_intent.yaml +360 -0
- eif/explain/__init__.py +7 -0
- eif/explain/artifact.py +78 -0
- eif/explain/eif_explain_coldstart_intent.yaml +216 -0
- eif/explain/hard_to_vary.py +168 -0
- eif/explain/reach.py +32 -0
- eif/falsify/__init__.py +7 -0
- eif/falsify/condition.py +58 -0
- eif/falsify/eif_evidence_collection_intent.yaml +310 -0
- eif/falsify/eif_native_tools_intent.yaml +128 -0
- eif/falsify/eif_v5_evidence_trust_weighting_intent.yaml +273 -0
- eif/falsify/evidence_collector.py +1304 -0
- eif/falsify/hard_to_vary.py +171 -0
- eif/falsify/native_tools.py +221 -0
- eif/falsify/sprt.py +84 -0
- eif/falsify/trivial_check.py +46 -0
- eif/hypothesis_agenda/__init__.py +27 -0
- eif/hypothesis_agenda/agenda.py +176 -0
- eif/hypothesis_agenda/eif_hypothesis_agenda_intent.yaml +168 -0
- eif/hypothesis_agenda/fdr.py +92 -0
- eif/hypothesis_agenda/scorer.py +132 -0
- eif/input_guard/__init__.py +13 -0
- eif/input_guard/detector.py +259 -0
- eif/input_guard/eif_input_guard_intent.yaml +181 -0
- eif/integration/__init__.py +34 -0
- eif/integration/eif_auto.py +228 -0
- eif/integration/interceptor.py +218 -0
- eif/mcp_server/__init__.py +1 -0
- eif/mcp_server/auth.py +81 -0
- eif/mcp_server/http_server.py +460 -0
- eif/mcp_server/resources.py +113 -0
- eif/mcp_server/server.py +3485 -0
- eif/memory/__init__.py +5 -0
- eif/memory/strategy.py +246 -0
- eif/programme/__init__.py +7 -0
- eif/programme/monitor.py +58 -0
- eif/programme/principle_memory.py +177 -0
- eif/programme/principle_revision.py +412 -0
- eif/programme/signals.py +131 -0
- eif/programme/status.py +56 -0
- eif/py.typed +0 -0
- eif/record/__init__.py +7 -0
- eif/record/chain.py +33 -0
- eif/record/compliance.py +62 -0
- eif/record/isc_disclosure.py +257 -0
- eif/record/outcome_store.py +177 -0
- eif/record/provenance.py +148 -0
- eif/record/research_object.py +412 -0
- eif/replicate/__init__.py +6 -0
- eif/replicate/divergence.py +51 -0
- eif/replicate/eif_independent_replication_intent.yaml +109 -0
- eif/replicate/independent.py +185 -0
- eif/replicate/protocol.py +68 -0
- eif/reporting/__init__.py +1 -0
- eif/reporting/pdf_renderer.py +169 -0
- eif/schemas.py +787 -0
- eif/sdk/__init__.py +36 -0
- eif/sdk/exceptions.py +87 -0
- eif/sdk/interceptor.py +316 -0
- eif/session.py +40 -0
- eif/sycophancy/__init__.py +24 -0
- eif/sycophancy/cost_model.py +215 -0
- eif/sycophancy/detector.py +270 -0
- eif/sycophancy/drift.py +306 -0
- eif/sycophancy/eif_sycophancy_intent.yaml +309 -0
- eif/sycophancy/faithfulness.py +106 -0
- eif/sycophancy/framing.py +191 -0
- eif/update/__init__.py +14 -0
- eif/update/eig.py +29 -0
- eif/update/paradigm.py +121 -0
- eif/update/posterior.py +21 -0
- eif/update/stopping.py +42 -0
- eif_engine-4.1.0.dist-info/METADATA +607 -0
- eif_engine-4.1.0.dist-info/RECORD +111 -0
- eif_engine-4.1.0.dist-info/WHEEL +4 -0
- eif_engine-4.1.0.dist-info/entry_points.txt +3 -0
- eif_engine-4.1.0.dist-info/licenses/LICENSE +21 -0
- eif_engine-4.1.0.dist-info/licenses/NOTICE +17 -0
|
@@ -0,0 +1,379 @@
|
|
|
1
|
+
"""DASES Adversarial Replicator Agent - V2 (F8).
|
|
2
|
+
|
|
3
|
+
DASES (arXiv:2603.29045 - "Let the Abyss Stare Back"): adaptive falsification
|
|
4
|
+
where the test strategy evolves based on what the artifact successfully defends.
|
|
5
|
+
|
|
6
|
+
The gap this closes vs. initial V2: V1 ran a single-shot adversarial probe.
|
|
7
|
+
The paper requires the falsifier to *adaptively evolve* its strategy: after
|
|
8
|
+
each round that the claim survives, a harder, targeted test is designed
|
|
9
|
+
specifically attacking the mechanism that allowed survival. This continues
|
|
10
|
+
until the claim fails or max_rounds is exhausted.
|
|
11
|
+
|
|
12
|
+
The full adaptive loop (DASES paper):
|
|
13
|
+
Round 1: Design initial indirect falsification test.
|
|
14
|
+
If survived:
|
|
15
|
+
Round 2: Design a NEW test targeting specifically what Round 1 exposed as
|
|
16
|
+
the claim's "defense mechanism" - not a re-test of Round 1.
|
|
17
|
+
If survived again:
|
|
18
|
+
Round 3 (optional): Attack the meta-assumption behind Round 2's defense.
|
|
19
|
+
Accept ONLY if the claim survives ALL rounds.
|
|
20
|
+
|
|
21
|
+
Constraints:
|
|
22
|
+
F8C1: system prompt must NOT contain 'evaluate'; must contain 'fail' or 'falsif'.
|
|
23
|
+
F8C2: test_designed must be non-empty, != claim verbatim, len >= 50 chars.
|
|
24
|
+
F8C3: eif_challenge with replication_mode='DASES' must include replication_result.
|
|
25
|
+
F8C4 [new]: adaptive evolution - round N+1 targets specifically what round N survived.
|
|
26
|
+
F8C5 [new]: max_rounds = 3; claim accepted only if it survives all completed rounds.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
import json
|
|
32
|
+
import logging
|
|
33
|
+
import re
|
|
34
|
+
from collections.abc import Callable
|
|
35
|
+
from dataclasses import dataclass
|
|
36
|
+
|
|
37
|
+
from eif.schemas import ReplicationResult
|
|
38
|
+
|
|
39
|
+
_log = logging.getLogger(__name__)
|
|
40
|
+
|
|
41
|
+
# F8C5: maximum falsification rounds
|
|
42
|
+
_MAX_ROUNDS: int = 3
|
|
43
|
+
|
|
44
|
+
# F8C1: adversarial system prompt - must contain 'fail'/'falsif', must NOT contain 'evaluate'
|
|
45
|
+
_DASES_SYSTEM_PROMPT = """\
|
|
46
|
+
Your ONLY goal is to find a way this claim fails.
|
|
47
|
+
|
|
48
|
+
You are an adversarial falsifier. Your job is NOT to assess or judge whether the claim is good or bad.
|
|
49
|
+
Your job is to design a specific indirect test that the claim must survive to remain credible.
|
|
50
|
+
|
|
51
|
+
Rules:
|
|
52
|
+
1. Do NOT test the claim directly - find an indirect path to failure.
|
|
53
|
+
2. Design a test that the claimant would not anticipate.
|
|
54
|
+
3. If the claim survives your falsification test, state that explicitly.
|
|
55
|
+
4. If the claim fails your test, explain precisely how and why.
|
|
56
|
+
5. Your test must be concrete and falsifiable - not vague.
|
|
57
|
+
|
|
58
|
+
Return JSON:
|
|
59
|
+
{
|
|
60
|
+
"test_designed": "A specific test: ...",
|
|
61
|
+
"test_result": "What the test revealed: ...",
|
|
62
|
+
"survived": true | false,
|
|
63
|
+
"confidence": 0.0–1.0,
|
|
64
|
+
"defense_mechanism": "If survived: what aspect of the claim allowed it to survive? (used to target the next round)"
|
|
65
|
+
}
|
|
66
|
+
Return ONLY the JSON.
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
_DASES_ROUND1_TEMPLATE = """\
|
|
70
|
+
Claim to falsify: {claim}
|
|
71
|
+
|
|
72
|
+
Supporting evidence (what the claimant would cite):
|
|
73
|
+
{evidence}
|
|
74
|
+
|
|
75
|
+
Session verdict: {verdict}
|
|
76
|
+
|
|
77
|
+
Design a non-obvious indirect falsification test for this claim.
|
|
78
|
+
"""
|
|
79
|
+
|
|
80
|
+
_DASES_ADAPTIVE_TEMPLATE = """\
|
|
81
|
+
Claim: {claim}
|
|
82
|
+
|
|
83
|
+
This claim survived round {prev_round} of adversarial falsification.
|
|
84
|
+
Round {prev_round} test: {prev_test}
|
|
85
|
+
Round {prev_round} result: {prev_result}
|
|
86
|
+
The defense mechanism that allowed survival: {defense_mechanism}
|
|
87
|
+
|
|
88
|
+
Design round {this_round}: a NEW falsification test that specifically targets
|
|
89
|
+
the defense mechanism identified above. This test must:
|
|
90
|
+
1. Be different from round {prev_round}'s test - do not repeat it.
|
|
91
|
+
2. Attack specifically: {defense_mechanism}
|
|
92
|
+
3. Be indirect - not a direct restatement of the claim.
|
|
93
|
+
4. Be harder to survive than round {prev_round}'s test.
|
|
94
|
+
|
|
95
|
+
Supporting evidence for context:
|
|
96
|
+
{evidence}
|
|
97
|
+
"""
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@dataclass
|
|
101
|
+
class _RoundResult:
|
|
102
|
+
"""Internal result for a single falsification round."""
|
|
103
|
+
round_num: int
|
|
104
|
+
test_designed: str
|
|
105
|
+
test_result: str
|
|
106
|
+
survived: bool
|
|
107
|
+
confidence: float
|
|
108
|
+
defense_mechanism: str = ""
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _heuristic_round1(claim_text: str) -> _RoundResult:
|
|
112
|
+
"""Heuristic round 1 test - mechanism test."""
|
|
113
|
+
return _RoundResult(
|
|
114
|
+
round_num=1,
|
|
115
|
+
test_designed=_enforce_test_quality(
|
|
116
|
+
"Mechanism test: Does the causal mechanism described survive removal of its assumed "
|
|
117
|
+
"intermediate steps? Remove each link in the causal chain and verify the claim holds "
|
|
118
|
+
"without any of its assumed mediators.",
|
|
119
|
+
claim_text,
|
|
120
|
+
),
|
|
121
|
+
test_result="Mechanism chain not independently verified - intermediate steps assumed.",
|
|
122
|
+
survived=False,
|
|
123
|
+
confidence=0.40,
|
|
124
|
+
defense_mechanism="",
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _heuristic_adaptive(claim_text: str, prev: _RoundResult) -> _RoundResult:
|
|
129
|
+
"""Heuristic adaptive test targeting what round N exposed."""
|
|
130
|
+
if prev.survived:
|
|
131
|
+
# Probe the boundary of what survived
|
|
132
|
+
test = (
|
|
133
|
+
f"Boundary condition test: The claim survived round {prev.round_num}'s mechanism test. "
|
|
134
|
+
f"Now probe the claim at its stated boundary values - does it hold at the extremes "
|
|
135
|
+
f"of its domain, or only under the specific conditions tested? "
|
|
136
|
+
f"Test: apply the claim to the minimum and maximum plausible values of its key variable."
|
|
137
|
+
)
|
|
138
|
+
survived = False
|
|
139
|
+
defense = "survived by relying on implicit domain assumptions not stated in the claim"
|
|
140
|
+
else:
|
|
141
|
+
test = (
|
|
142
|
+
"Temporal stability test: Does the claim hold over time? "
|
|
143
|
+
"Verify whether the same evidence collected 6 months earlier or later "
|
|
144
|
+
"would yield the same conclusion - or whether the claim is time-sensitive "
|
|
145
|
+
"in ways not disclosed."
|
|
146
|
+
)
|
|
147
|
+
survived = False
|
|
148
|
+
defense = ""
|
|
149
|
+
|
|
150
|
+
return _RoundResult(
|
|
151
|
+
round_num=prev.round_num + 1,
|
|
152
|
+
test_designed=_enforce_test_quality(test, claim_text),
|
|
153
|
+
test_result=(
|
|
154
|
+
f"Round {prev.round_num + 1} heuristic test applied. "
|
|
155
|
+
"Claim does not survive boundary/temporal probing without additional evidence."
|
|
156
|
+
),
|
|
157
|
+
survived=survived,
|
|
158
|
+
confidence=0.35,
|
|
159
|
+
defense_mechanism=defense,
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _heuristic_round3(claim_text: str, prev: _RoundResult) -> _RoundResult:
|
|
164
|
+
"""Meta-assumption attack for round 3."""
|
|
165
|
+
test = (
|
|
166
|
+
"Meta-assumption test: Both prior falsification rounds tested the claim's "
|
|
167
|
+
"direct mechanism and boundary conditions. This round targets the meta-assumption "
|
|
168
|
+
"underlying those defenses: is the measurement framework itself valid? "
|
|
169
|
+
"Test: could the claim be an artifact of how the measurement was constructed "
|
|
170
|
+
"rather than a real property of the phenomenon?"
|
|
171
|
+
)
|
|
172
|
+
return _RoundResult(
|
|
173
|
+
round_num=3,
|
|
174
|
+
test_designed=_enforce_test_quality(test, claim_text),
|
|
175
|
+
test_result=(
|
|
176
|
+
"Meta-assumption test reveals the claim's conclusion depends on the validity "
|
|
177
|
+
"of its measurement framework, which has not been independently verified."
|
|
178
|
+
),
|
|
179
|
+
survived=False,
|
|
180
|
+
confidence=0.30,
|
|
181
|
+
defense_mechanism="",
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _run_llm_round(
|
|
186
|
+
claim_text: str,
|
|
187
|
+
evidence_str: str,
|
|
188
|
+
session_decision: str,
|
|
189
|
+
round_num: int,
|
|
190
|
+
prev_round: _RoundResult | None,
|
|
191
|
+
llm_fn: Callable,
|
|
192
|
+
) -> _RoundResult | None:
|
|
193
|
+
"""Run a single LLM-driven falsification round. Returns None on failure."""
|
|
194
|
+
try:
|
|
195
|
+
if round_num == 1 or prev_round is None:
|
|
196
|
+
user_msg = _DASES_ROUND1_TEMPLATE.format(
|
|
197
|
+
claim=claim_text[:500],
|
|
198
|
+
evidence=evidence_str[:800],
|
|
199
|
+
verdict=session_decision,
|
|
200
|
+
)
|
|
201
|
+
else:
|
|
202
|
+
user_msg = _DASES_ADAPTIVE_TEMPLATE.format(
|
|
203
|
+
claim=claim_text[:400],
|
|
204
|
+
prev_round=prev_round.round_num,
|
|
205
|
+
prev_test=prev_round.test_designed[:300],
|
|
206
|
+
prev_result=prev_round.test_result[:200],
|
|
207
|
+
defense_mechanism=prev_round.defense_mechanism[:200] or "unspecified defense",
|
|
208
|
+
this_round=round_num,
|
|
209
|
+
evidence=evidence_str[:600],
|
|
210
|
+
)
|
|
211
|
+
|
|
212
|
+
raw = llm_fn(user_msg, f"dases_round_{round_num}", system_prompt=_DASES_SYSTEM_PROMPT)
|
|
213
|
+
text = raw if isinstance(raw, str) else json.dumps(raw)
|
|
214
|
+
text = re.sub(r"```(?:json)?\s*(.*?)\s*```", r"\1", text, flags=re.DOTALL).strip()
|
|
215
|
+
parsed = json.loads(text)
|
|
216
|
+
|
|
217
|
+
test_designed = _enforce_test_quality(
|
|
218
|
+
str(parsed.get("test_designed", "")).strip(), claim_text
|
|
219
|
+
)
|
|
220
|
+
test_result = str(parsed.get("test_result", "")).strip() or "Result not parseable"
|
|
221
|
+
survived = bool(parsed.get("survived", True))
|
|
222
|
+
confidence = max(0.0, min(1.0, float(parsed.get("confidence", 0.5))))
|
|
223
|
+
defense = str(parsed.get("defense_mechanism", "")).strip()
|
|
224
|
+
|
|
225
|
+
if round_num > 1 and test_designed == (prev_round.test_designed if prev_round else ""):
|
|
226
|
+
_log.warning("DASES: round %d repeated round %d test - using heuristic", round_num, round_num - 1)
|
|
227
|
+
return None
|
|
228
|
+
|
|
229
|
+
return _RoundResult(
|
|
230
|
+
round_num=round_num,
|
|
231
|
+
test_designed=test_designed,
|
|
232
|
+
test_result=test_result,
|
|
233
|
+
survived=survived,
|
|
234
|
+
confidence=confidence,
|
|
235
|
+
defense_mechanism=defense,
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
except Exception as exc: # noqa: BLE001
|
|
239
|
+
_log.warning("DASES: LLM call failed round %d (%s)", round_num, exc)
|
|
240
|
+
return None
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def run_adversarial_replication(
|
|
244
|
+
claim_text: str,
|
|
245
|
+
supporting_evidence: list[str] | None = None,
|
|
246
|
+
session_decision: str = "UNKNOWN",
|
|
247
|
+
llm_fn: Callable | None = None,
|
|
248
|
+
max_rounds: int = _MAX_ROUNDS,
|
|
249
|
+
) -> ReplicationResult:
|
|
250
|
+
"""Run DASES adaptive multi-round adversarial falsification on a claim.
|
|
251
|
+
|
|
252
|
+
The adaptive loop (F8C4/F8C5):
|
|
253
|
+
Round 1: Initial indirect falsification test.
|
|
254
|
+
Round 2 (if round 1 survived): target the specific defense mechanism from round 1.
|
|
255
|
+
Round 3 (if round 2 survived): attack the meta-assumption behind round 2's defense.
|
|
256
|
+
Claim is ACCEPTED only if it survives all completed rounds.
|
|
257
|
+
|
|
258
|
+
Args:
|
|
259
|
+
claim_text: The claim to falsify.
|
|
260
|
+
supporting_evidence: Evidence the claimant would cite.
|
|
261
|
+
session_decision: The current session verdict (ACT / HALT / REVISE).
|
|
262
|
+
llm_fn: Optional LLM function. When None, heuristic tests are used.
|
|
263
|
+
max_rounds: Maximum falsification rounds (default: 3, per F8C5).
|
|
264
|
+
|
|
265
|
+
Returns:
|
|
266
|
+
ReplicationResult with the full multi-round audit summary.
|
|
267
|
+
|
|
268
|
+
Guarantees:
|
|
269
|
+
F8C1: system_prompt does not contain 'evaluate'; contains 'fail'/'falsif'.
|
|
270
|
+
F8C2: test_designed != claim_text verbatim; len >= 50 chars.
|
|
271
|
+
F8C4: round N+1 targets specifically what round N survived.
|
|
272
|
+
F8C5: max_rounds respected; claim accepted only if all rounds survived.
|
|
273
|
+
"""
|
|
274
|
+
assert "evaluate" not in _DASES_SYSTEM_PROMPT.lower(), \
|
|
275
|
+
"F8C1 violated: system prompt contains 'evaluate'"
|
|
276
|
+
assert any(w in _DASES_SYSTEM_PROMPT.lower() for w in ("fail", "falsif")), \
|
|
277
|
+
"F8C1 violated: prompt missing 'fail'/'falsif'"
|
|
278
|
+
|
|
279
|
+
evidence_str = "\n".join(f"- {e}" for e in (supporting_evidence or [])) or "(none provided)"
|
|
280
|
+
max_rounds = max(1, min(max_rounds, _MAX_ROUNDS))
|
|
281
|
+
|
|
282
|
+
rounds: list[_RoundResult] = []
|
|
283
|
+
prev_round: _RoundResult | None = None
|
|
284
|
+
|
|
285
|
+
for round_num in range(1, max_rounds + 1):
|
|
286
|
+
# Only continue if the previous round was survived (adaptive progression)
|
|
287
|
+
if round_num > 1 and rounds and not rounds[-1].survived:
|
|
288
|
+
_log.info("DASES: claim failed round %d - stopping at round %d", round_num - 1, round_num - 1)
|
|
289
|
+
break
|
|
290
|
+
|
|
291
|
+
_log.info("DASES: starting round %d (max=%d)", round_num, max_rounds)
|
|
292
|
+
|
|
293
|
+
# Try LLM first, fall back to heuristic
|
|
294
|
+
result: _RoundResult | None = None
|
|
295
|
+
if llm_fn is not None:
|
|
296
|
+
result = _run_llm_round(
|
|
297
|
+
claim_text, evidence_str, session_decision,
|
|
298
|
+
round_num, prev_round, llm_fn,
|
|
299
|
+
)
|
|
300
|
+
|
|
301
|
+
if result is None:
|
|
302
|
+
# Heuristic path
|
|
303
|
+
if round_num == 1:
|
|
304
|
+
result = _heuristic_round1(claim_text)
|
|
305
|
+
elif round_num == 2:
|
|
306
|
+
result = _heuristic_adaptive(claim_text, prev_round) # type: ignore[arg-type]
|
|
307
|
+
else:
|
|
308
|
+
result = _heuristic_round3(claim_text, prev_round) # type: ignore[arg-type]
|
|
309
|
+
|
|
310
|
+
rounds.append(result)
|
|
311
|
+
prev_round = result
|
|
312
|
+
|
|
313
|
+
_log.info(
|
|
314
|
+
"DASES: round %d complete - survived=%s confidence=%.2f",
|
|
315
|
+
round_num, result.survived, result.confidence,
|
|
316
|
+
)
|
|
317
|
+
|
|
318
|
+
# F8C5: claim accepted only if it survived ALL completed rounds
|
|
319
|
+
final_survived = all(r.survived for r in rounds)
|
|
320
|
+
final_confidence = round(sum(r.confidence for r in rounds) / len(rounds), 4) if rounds else 0.4
|
|
321
|
+
|
|
322
|
+
# Consolidate test_designed (primary round) and test_result (multi-round summary)
|
|
323
|
+
primary = rounds[0] if rounds else None
|
|
324
|
+
test_designed = primary.test_designed if primary else _enforce_test_quality("", claim_text)
|
|
325
|
+
test_result = _build_multi_round_summary(rounds, claim_text)
|
|
326
|
+
|
|
327
|
+
_log.info(
|
|
328
|
+
"DASES: adaptive replication complete - rounds=%d final_survived=%s confidence=%.2f",
|
|
329
|
+
len(rounds), final_survived, final_confidence,
|
|
330
|
+
)
|
|
331
|
+
|
|
332
|
+
return ReplicationResult(
|
|
333
|
+
claim_text=claim_text,
|
|
334
|
+
survived=final_survived,
|
|
335
|
+
test_designed=test_designed,
|
|
336
|
+
test_result=test_result,
|
|
337
|
+
confidence=final_confidence,
|
|
338
|
+
)
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def _build_multi_round_summary(rounds: list[_RoundResult], claim_text: str) -> str:
|
|
342
|
+
"""Build a human-readable summary of all falsification rounds."""
|
|
343
|
+
if not rounds:
|
|
344
|
+
return "No rounds completed."
|
|
345
|
+
|
|
346
|
+
lines = [f"DASES adaptive falsification - {len(rounds)} round(s):"]
|
|
347
|
+
for r in rounds:
|
|
348
|
+
status = "SURVIVED" if r.survived else "FAILED"
|
|
349
|
+
lines.append(
|
|
350
|
+
f" Round {r.round_num} [{status}]: {r.test_designed[:120]}... "
|
|
351
|
+
f"→ {r.test_result[:100]}"
|
|
352
|
+
)
|
|
353
|
+
if r.survived and r.defense_mechanism:
|
|
354
|
+
lines.append(f" Defense: {r.defense_mechanism[:100]}")
|
|
355
|
+
|
|
356
|
+
all_survived = all(r.survived for r in rounds)
|
|
357
|
+
if all_survived:
|
|
358
|
+
lines.append(f"VERDICT: claim survived all {len(rounds)} round(s). ACCEPT.")
|
|
359
|
+
else:
|
|
360
|
+
failed_at = next((r.round_num for r in rounds if not r.survived), len(rounds))
|
|
361
|
+
lines.append(f"VERDICT: claim failed at round {failed_at}. REJECT.")
|
|
362
|
+
|
|
363
|
+
return " | ".join(lines)
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def _enforce_test_quality(test_designed: str, claim_text: str) -> str:
|
|
367
|
+
"""F8C2: Ensure test_designed is non-empty, != claim, and len >= 50 chars."""
|
|
368
|
+
if not test_designed or test_designed.strip() == claim_text.strip():
|
|
369
|
+
test_designed = (
|
|
370
|
+
f"Indirect boundary test: verify the claim '{claim_text[:80]}' holds "
|
|
371
|
+
"under conditions not stated in the original evidence (e.g. different population, "
|
|
372
|
+
"time period, or measurement method)."
|
|
373
|
+
)
|
|
374
|
+
if len(test_designed) < 50:
|
|
375
|
+
test_designed = test_designed + (
|
|
376
|
+
" This test targets the structural assumptions of the claim rather than its "
|
|
377
|
+
"direct evidential support."
|
|
378
|
+
)
|
|
379
|
+
return test_designed
|
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
"""Empirical catch rate measurement for the compound error model (F14).
|
|
2
|
+
|
|
3
|
+
The compound error model (Architecture §9.2) uses literature-derived rates:
|
|
4
|
+
f = 0.6 (FALSIFY catch rate) - analogous to POPPER mechanism
|
|
5
|
+
u = 0.5 (UPDATE routing delta) - analogous to LAPD mechanism
|
|
6
|
+
c = 0.4 (CHALLENGE hit rate) - multi-agent literature estimates
|
|
7
|
+
|
|
8
|
+
The cross-check audit §5.1 flags explicitly: "These have not been measured on
|
|
9
|
+
an actual EIF implementation." This module closes that gap by computing actual
|
|
10
|
+
rates from outcome-linked provenance records (OutcomeRecord rows in the outcome_store).
|
|
11
|
+
|
|
12
|
+
Definitions:
|
|
13
|
+
f_empirical - FALSIFY catch rate:
|
|
14
|
+
fraction of HIGH-consequence WRONG claims (outcome=False) that were
|
|
15
|
+
caught by FALSIFY (SPRT REJECT or trivial_flag=True).
|
|
16
|
+
A claim is "caught" when the engine raised a flag that a decision-maker
|
|
17
|
+
would notice. Denominator: total HIGH-consequence claims in labeled-wrong sessions.
|
|
18
|
+
|
|
19
|
+
c_empirical - CHALLENGE hit rate:
|
|
20
|
+
fraction of ProvenanceRecords where ChallengeResult.counter_evidence_found=True.
|
|
21
|
+
Denominator: all records that ran the CHALLENGE phase (challenge is not None).
|
|
22
|
+
|
|
23
|
+
u_empirical - UPDATE routing-change rate:
|
|
24
|
+
fraction of ProvenanceRecords where UPDATE changed routing across a threshold
|
|
25
|
+
boundary (ACT↔REVISE or REVISE↔HALT). Uses UpdateResult pairs inside a record.
|
|
26
|
+
Denominator: records that have >= 1 UpdateResult.
|
|
27
|
+
|
|
28
|
+
The store (~/.eif/catch_rates.json) accumulates statistics across sessions using
|
|
29
|
+
a running mean. When >= 5 outcome-linked provenance records are measured (F14C3),
|
|
30
|
+
empirical rates replace literature defaults in the compound error calculation.
|
|
31
|
+
|
|
32
|
+
Research:
|
|
33
|
+
EIF Architecture §9.2 - compound error model with f, u, c parameters.
|
|
34
|
+
Cross-check audit §5.1 - "not measured on actual EIF data; may be off 20-30%".
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
from __future__ import annotations
|
|
38
|
+
|
|
39
|
+
import json
|
|
40
|
+
import logging
|
|
41
|
+
import os
|
|
42
|
+
import tempfile
|
|
43
|
+
from datetime import datetime
|
|
44
|
+
from pathlib import Path
|
|
45
|
+
|
|
46
|
+
from eif.schemas import (
|
|
47
|
+
THRESHOLD_ACT,
|
|
48
|
+
THRESHOLD_HALT,
|
|
49
|
+
CatchRateReport,
|
|
50
|
+
OutcomeRecord,
|
|
51
|
+
ProvenanceRecord,
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
_log = logging.getLogger("eif.catch_rate_empirical")
|
|
55
|
+
|
|
56
|
+
# Literature defaults from Architecture §9.2 and v3 intent F14 problem text:
|
|
57
|
+
# f = 0.6 (FALSIFY catch rate) - POPPER-analogous mechanism
|
|
58
|
+
# c = 0.4 (CHALLENGE hit rate) - multi-agent literature estimates
|
|
59
|
+
# u = 0.5 (UPDATE routing-change) - LAPD-analogous mechanism
|
|
60
|
+
# NOTE: earlier code had c=0.5/u=0.4 (swapped vs intent). Fixed to match intent.
|
|
61
|
+
_F_LIT: float = 0.6
|
|
62
|
+
_C_LIT: float = 0.4 # CHALLENGE - was incorrectly 0.5
|
|
63
|
+
_U_LIT: float = 0.5 # UPDATE - was incorrectly 0.4
|
|
64
|
+
|
|
65
|
+
# How many sessions needed before empirical rates replace literature defaults
|
|
66
|
+
EMPIRICAL_MIN_SESSIONS: int = 5
|
|
67
|
+
PARTIAL_MIN_SESSIONS: int = 3
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _store_path() -> Path:
|
|
71
|
+
base = os.environ.get("EIF_STORE_DIR", os.path.expanduser("~/.eif"))
|
|
72
|
+
path = Path(base) / "catch_rates.json"
|
|
73
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
74
|
+
return path
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
78
|
+
# Measurement functions
|
|
79
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def compute_session_catch_rates(
|
|
83
|
+
provenance_chain: list[ProvenanceRecord],
|
|
84
|
+
outcome_records: list[OutcomeRecord],
|
|
85
|
+
) -> CatchRateReport:
|
|
86
|
+
"""Compute per-phase catch rates from a session's provenance chain and labeled outcomes.
|
|
87
|
+
|
|
88
|
+
outcome_records: labeled outcomes from the outcome_store for this session.
|
|
89
|
+
When outcome_records is empty, f_empirical is None (no ground truth).
|
|
90
|
+
|
|
91
|
+
Returns a CatchRateReport with empirical measurements and compound error
|
|
92
|
+
residuals using BOTH empirical and literature rates.
|
|
93
|
+
"""
|
|
94
|
+
# Build lookup: provenance_record_id → outcome (True/False)
|
|
95
|
+
outcome_map: dict[str, bool] = {
|
|
96
|
+
o.provenance_record_id: o.outcome for o in outcome_records
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
# ── f_empirical: FALSIFY catch rate ──────────────────────────────────────
|
|
100
|
+
# f_empirical = 0.0 when denominator is 0 (no labeled wrong claims - intent F14C1).
|
|
101
|
+
# This is distinct from None: 0.0 means "measured, nothing to catch";
|
|
102
|
+
# data_status INSUFFICIENT_DATA prevents it from replacing the literature default.
|
|
103
|
+
f_empirical: float = 0.0
|
|
104
|
+
wrong_claims_total = 0
|
|
105
|
+
wrong_claims_caught = 0
|
|
106
|
+
|
|
107
|
+
for record in provenance_chain:
|
|
108
|
+
rec_outcome = outcome_map.get(record.record_id)
|
|
109
|
+
if rec_outcome is None:
|
|
110
|
+
continue # unlabeled - skip for catch-rate denominator
|
|
111
|
+
if rec_outcome is True:
|
|
112
|
+
continue # agent was correct - not a "wrong" session to catch
|
|
113
|
+
|
|
114
|
+
# Session was wrong (outcome=False)
|
|
115
|
+
# F14C1: denominator = HIGH-consequence claims only (intent is explicit).
|
|
116
|
+
# CalibrationResult now carries consequence_of_wrong; for records assembled
|
|
117
|
+
# before that field was added, fall back to all ASSUMED+GUESSED as proxy.
|
|
118
|
+
high_claims = [
|
|
119
|
+
c for c in record.calibration if c.consequence_of_wrong == "HIGH"
|
|
120
|
+
] if record.calibration else []
|
|
121
|
+
if not high_claims:
|
|
122
|
+
# Fallback: use ASSUMED+GUESSED from registry (pre-field migration records)
|
|
123
|
+
n_at_risk = len(record.registry.guessed) + len(record.registry.assumed)
|
|
124
|
+
else:
|
|
125
|
+
n_at_risk = len(high_claims)
|
|
126
|
+
|
|
127
|
+
# Caught: any SPRT REJECT or trivial_flag=True in falsification conditions
|
|
128
|
+
has_reject = any(s.decision == "REJECT" for s in record.sprt_results)
|
|
129
|
+
has_trivial = any(fc.trivial_flag for fc in record.falsification_conditions)
|
|
130
|
+
caught = has_reject or has_trivial
|
|
131
|
+
|
|
132
|
+
wrong_claims_total += max(1, n_at_risk)
|
|
133
|
+
if caught:
|
|
134
|
+
wrong_claims_caught += max(1, n_at_risk)
|
|
135
|
+
|
|
136
|
+
if wrong_claims_total > 0:
|
|
137
|
+
f_empirical = wrong_claims_caught / wrong_claims_total
|
|
138
|
+
|
|
139
|
+
# ── c_empirical: CHALLENGE counter-evidence hit rate ─────────────────────
|
|
140
|
+
c_empirical: float | None = None
|
|
141
|
+
challenged_records = [r for r in provenance_chain if r.challenge is not None]
|
|
142
|
+
if challenged_records:
|
|
143
|
+
c_empirical = sum(
|
|
144
|
+
1 for r in challenged_records if r.challenge.counter_evidence_found
|
|
145
|
+
) / len(challenged_records)
|
|
146
|
+
|
|
147
|
+
# ── u_empirical: UPDATE routing-change rate ───────────────────────────────
|
|
148
|
+
u_empirical: float | None = None
|
|
149
|
+
records_with_updates = [r for r in provenance_chain if r.updates]
|
|
150
|
+
if records_with_updates:
|
|
151
|
+
routing_changes = 0
|
|
152
|
+
for record in records_with_updates:
|
|
153
|
+
if len(record.updates) < 1:
|
|
154
|
+
continue
|
|
155
|
+
first_update = record.updates[0]
|
|
156
|
+
last_update = record.updates[-1]
|
|
157
|
+
routing_changed = _routing_bucket(first_update.prior_posterior) != \
|
|
158
|
+
_routing_bucket(last_update.updated_posterior)
|
|
159
|
+
if routing_changed:
|
|
160
|
+
routing_changes += 1
|
|
161
|
+
u_empirical = routing_changes / len(records_with_updates)
|
|
162
|
+
|
|
163
|
+
# ── Determine data status ─────────────────────────────────────────────────
|
|
164
|
+
labeled_count = len([r for r in provenance_chain if r.record_id in outcome_map])
|
|
165
|
+
if labeled_count >= EMPIRICAL_MIN_SESSIONS:
|
|
166
|
+
data_status = "SUFFICIENT"
|
|
167
|
+
elif labeled_count >= PARTIAL_MIN_SESSIONS:
|
|
168
|
+
data_status = "PARTIAL"
|
|
169
|
+
else:
|
|
170
|
+
data_status = "INSUFFICIENT_DATA"
|
|
171
|
+
|
|
172
|
+
# ── Determine which rates to use for primary calculation ──────────────────
|
|
173
|
+
use_empirical = data_status == "SUFFICIENT"
|
|
174
|
+
f_used = f_empirical if (use_empirical and f_empirical is not None) else _F_LIT
|
|
175
|
+
c_used = c_empirical if (use_empirical and c_empirical is not None) else _C_LIT
|
|
176
|
+
u_used = u_empirical if (use_empirical and u_empirical is not None) else _U_LIT
|
|
177
|
+
|
|
178
|
+
# ── Compound error residuals ───────────────────────────────────────────────
|
|
179
|
+
# R6-06: when data_status reaches SUFFICIENT but c/u phases never ran (no
|
|
180
|
+
# CHALLENGE or UPDATE calls in this corpus), fill missing legs with literature
|
|
181
|
+
# defaults so compound_error_empirical is always populated once f is measured.
|
|
182
|
+
# The note field already flags INSUFFICIENT data for unavailable rates.
|
|
183
|
+
compound_empirical: float | None = None
|
|
184
|
+
if f_empirical is not None:
|
|
185
|
+
_c_for_compound = c_empirical if c_empirical is not None else _C_LIT
|
|
186
|
+
_u_for_compound = u_empirical if u_empirical is not None else _U_LIT
|
|
187
|
+
compound_empirical = (1 - f_empirical) * (1 - _c_for_compound) * (1 - _u_for_compound)
|
|
188
|
+
|
|
189
|
+
compound_literature = (1 - _F_LIT) * (1 - _C_LIT) * (1 - _U_LIT)
|
|
190
|
+
|
|
191
|
+
return CatchRateReport(
|
|
192
|
+
f_empirical=f_empirical,
|
|
193
|
+
c_empirical=c_empirical,
|
|
194
|
+
u_empirical=u_empirical,
|
|
195
|
+
sessions_measured=labeled_count, # count of outcome-linked provenance records (not unique session_ids)
|
|
196
|
+
data_status=data_status,
|
|
197
|
+
f_literature=_F_LIT,
|
|
198
|
+
c_literature=_C_LIT,
|
|
199
|
+
u_literature=_U_LIT,
|
|
200
|
+
compound_error_empirical=compound_empirical,
|
|
201
|
+
compound_error_literature=compound_literature,
|
|
202
|
+
f_used=f_used,
|
|
203
|
+
c_used=c_used,
|
|
204
|
+
u_used=u_used,
|
|
205
|
+
generated_at=datetime.utcnow(),
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
210
|
+
# Persistent store
|
|
211
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def load_catch_rate_store() -> CatchRateReport | None:
|
|
215
|
+
"""Load the latest CatchRateReport from disk, or None if not present."""
|
|
216
|
+
path = _store_path()
|
|
217
|
+
if not path.exists():
|
|
218
|
+
return None
|
|
219
|
+
try:
|
|
220
|
+
raw = json.loads(path.read_text(encoding="utf-8"))
|
|
221
|
+
return CatchRateReport.model_validate(raw)
|
|
222
|
+
except Exception as exc:
|
|
223
|
+
_log.warning("catch_rate_store: could not load %s (%s)", path, exc)
|
|
224
|
+
return None
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def update_catch_rate_store(report: CatchRateReport) -> None:
|
|
228
|
+
"""Persist a CatchRateReport to disk atomically."""
|
|
229
|
+
path = _store_path()
|
|
230
|
+
data = report.model_dump(mode="json")
|
|
231
|
+
_atomic_write(path, data)
|
|
232
|
+
_log.info(
|
|
233
|
+
"catch_rate_store: updated (outcome_linked_records=%d, status=%s, f=%.2f c=%.2f u=%.2f)",
|
|
234
|
+
report.sessions_measured, report.data_status,
|
|
235
|
+
report.f_used, report.c_used, report.u_used,
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def get_catch_rate_report() -> CatchRateReport:
|
|
240
|
+
"""Return the stored CatchRateReport, or a fresh default with literature values."""
|
|
241
|
+
stored = load_catch_rate_store()
|
|
242
|
+
if stored is not None:
|
|
243
|
+
return stored
|
|
244
|
+
return CatchRateReport(
|
|
245
|
+
data_status="INSUFFICIENT_DATA",
|
|
246
|
+
compound_error_literature=(1 - _F_LIT) * (1 - _C_LIT) * (1 - _U_LIT),
|
|
247
|
+
generated_at=datetime.utcnow(),
|
|
248
|
+
)
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
252
|
+
# Internal helpers
|
|
253
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def _routing_bucket(posterior: float) -> str:
|
|
257
|
+
if posterior >= THRESHOLD_ACT:
|
|
258
|
+
return "ACT"
|
|
259
|
+
elif posterior >= THRESHOLD_HALT:
|
|
260
|
+
return "REVISE"
|
|
261
|
+
return "HALT"
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _atomic_write(path: Path, data: dict) -> None:
|
|
265
|
+
tmp_fd, tmp_path = tempfile.mkstemp(dir=path.parent, suffix=".tmp")
|
|
266
|
+
try:
|
|
267
|
+
with os.fdopen(tmp_fd, "w", encoding="utf-8") as f:
|
|
268
|
+
json.dump(data, f, indent=2, default=str)
|
|
269
|
+
os.replace(tmp_path, path)
|
|
270
|
+
except Exception:
|
|
271
|
+
try:
|
|
272
|
+
os.unlink(tmp_path)
|
|
273
|
+
except OSError:
|
|
274
|
+
pass
|
|
275
|
+
raise
|