eif-engine 4.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. eif/__init__.py +23 -0
  2. eif/auto.py +199 -0
  3. eif/calibrate/__init__.py +15 -0
  4. eif/calibrate/bayesian.py +53 -0
  5. eif/calibrate/conformal.py +57 -0
  6. eif/calibrate/domain_constraints.py +169 -0
  7. eif/calibrate/ece.py +154 -0
  8. eif/calibrate/prior_strategy.py +38 -0
  9. eif/calibrate/trust.py +120 -0
  10. eif/causal_gate/__init__.py +31 -0
  11. eif/causal_gate/confound.py +196 -0
  12. eif/causal_gate/direction.py +28 -0
  13. eif/causal_gate/eif_causal_gate_v4_intent.yaml +265 -0
  14. eif/causal_gate/evidence_probe.py +280 -0
  15. eif/causal_gate/intervention.py +55 -0
  16. eif/causal_gate/iris_pipeline.py +334 -0
  17. eif/causal_gate/verdict.py +47 -0
  18. eif/challenge/__init__.py +17 -0
  19. eif/challenge/diversity.py +58 -0
  20. eif/challenge/eif_multi_critic_intent.yaml +198 -0
  21. eif/challenge/multi_critic.py +270 -0
  22. eif/challenge/protocol.py +86 -0
  23. eif/challenge/replicator.py +379 -0
  24. eif/cost_model/__init__.py +5 -0
  25. eif/cost_model/catch_rate_empirical.py +275 -0
  26. eif/cost_model/cost_ledger.py +107 -0
  27. eif/cost_model/eif_catchrate_intent.yaml +86 -0
  28. eif/declare/__init__.py +6 -0
  29. eif/declare/extractor.py +308 -0
  30. eif/declare/harking_guard.py +110 -0
  31. eif/declare/registry.py +146 -0
  32. eif/eif_engine_intent.yaml +296 -0
  33. eif/eif_v2_intent.yaml +779 -0
  34. eif/eif_v3_intent.yaml +360 -0
  35. eif/explain/__init__.py +7 -0
  36. eif/explain/artifact.py +78 -0
  37. eif/explain/eif_explain_coldstart_intent.yaml +216 -0
  38. eif/explain/hard_to_vary.py +168 -0
  39. eif/explain/reach.py +32 -0
  40. eif/falsify/__init__.py +7 -0
  41. eif/falsify/condition.py +58 -0
  42. eif/falsify/eif_evidence_collection_intent.yaml +310 -0
  43. eif/falsify/eif_native_tools_intent.yaml +128 -0
  44. eif/falsify/eif_v5_evidence_trust_weighting_intent.yaml +273 -0
  45. eif/falsify/evidence_collector.py +1304 -0
  46. eif/falsify/hard_to_vary.py +171 -0
  47. eif/falsify/native_tools.py +221 -0
  48. eif/falsify/sprt.py +84 -0
  49. eif/falsify/trivial_check.py +46 -0
  50. eif/hypothesis_agenda/__init__.py +27 -0
  51. eif/hypothesis_agenda/agenda.py +176 -0
  52. eif/hypothesis_agenda/eif_hypothesis_agenda_intent.yaml +168 -0
  53. eif/hypothesis_agenda/fdr.py +92 -0
  54. eif/hypothesis_agenda/scorer.py +132 -0
  55. eif/input_guard/__init__.py +13 -0
  56. eif/input_guard/detector.py +259 -0
  57. eif/input_guard/eif_input_guard_intent.yaml +181 -0
  58. eif/integration/__init__.py +34 -0
  59. eif/integration/eif_auto.py +228 -0
  60. eif/integration/interceptor.py +218 -0
  61. eif/mcp_server/__init__.py +1 -0
  62. eif/mcp_server/auth.py +81 -0
  63. eif/mcp_server/http_server.py +460 -0
  64. eif/mcp_server/resources.py +113 -0
  65. eif/mcp_server/server.py +3485 -0
  66. eif/memory/__init__.py +5 -0
  67. eif/memory/strategy.py +246 -0
  68. eif/programme/__init__.py +7 -0
  69. eif/programme/monitor.py +58 -0
  70. eif/programme/principle_memory.py +177 -0
  71. eif/programme/principle_revision.py +412 -0
  72. eif/programme/signals.py +131 -0
  73. eif/programme/status.py +56 -0
  74. eif/py.typed +0 -0
  75. eif/record/__init__.py +7 -0
  76. eif/record/chain.py +33 -0
  77. eif/record/compliance.py +62 -0
  78. eif/record/isc_disclosure.py +257 -0
  79. eif/record/outcome_store.py +177 -0
  80. eif/record/provenance.py +148 -0
  81. eif/record/research_object.py +412 -0
  82. eif/replicate/__init__.py +6 -0
  83. eif/replicate/divergence.py +51 -0
  84. eif/replicate/eif_independent_replication_intent.yaml +109 -0
  85. eif/replicate/independent.py +185 -0
  86. eif/replicate/protocol.py +68 -0
  87. eif/reporting/__init__.py +1 -0
  88. eif/reporting/pdf_renderer.py +169 -0
  89. eif/schemas.py +787 -0
  90. eif/sdk/__init__.py +36 -0
  91. eif/sdk/exceptions.py +87 -0
  92. eif/sdk/interceptor.py +316 -0
  93. eif/session.py +40 -0
  94. eif/sycophancy/__init__.py +24 -0
  95. eif/sycophancy/cost_model.py +215 -0
  96. eif/sycophancy/detector.py +270 -0
  97. eif/sycophancy/drift.py +306 -0
  98. eif/sycophancy/eif_sycophancy_intent.yaml +309 -0
  99. eif/sycophancy/faithfulness.py +106 -0
  100. eif/sycophancy/framing.py +191 -0
  101. eif/update/__init__.py +14 -0
  102. eif/update/eig.py +29 -0
  103. eif/update/paradigm.py +121 -0
  104. eif/update/posterior.py +21 -0
  105. eif/update/stopping.py +42 -0
  106. eif_engine-4.1.0.dist-info/METADATA +607 -0
  107. eif_engine-4.1.0.dist-info/RECORD +111 -0
  108. eif_engine-4.1.0.dist-info/WHEEL +4 -0
  109. eif_engine-4.1.0.dist-info/entry_points.txt +3 -0
  110. eif_engine-4.1.0.dist-info/licenses/LICENSE +21 -0
  111. eif_engine-4.1.0.dist-info/licenses/NOTICE +17 -0
@@ -0,0 +1,379 @@
1
+ """DASES Adversarial Replicator Agent - V2 (F8).
2
+
3
+ DASES (arXiv:2603.29045 - "Let the Abyss Stare Back"): adaptive falsification
4
+ where the test strategy evolves based on what the artifact successfully defends.
5
+
6
+ The gap this closes vs. initial V2: V1 ran a single-shot adversarial probe.
7
+ The paper requires the falsifier to *adaptively evolve* its strategy: after
8
+ each round that the claim survives, a harder, targeted test is designed
9
+ specifically attacking the mechanism that allowed survival. This continues
10
+ until the claim fails or max_rounds is exhausted.
11
+
12
+ The full adaptive loop (DASES paper):
13
+ Round 1: Design initial indirect falsification test.
14
+ If survived:
15
+ Round 2: Design a NEW test targeting specifically what Round 1 exposed as
16
+ the claim's "defense mechanism" - not a re-test of Round 1.
17
+ If survived again:
18
+ Round 3 (optional): Attack the meta-assumption behind Round 2's defense.
19
+ Accept ONLY if the claim survives ALL rounds.
20
+
21
+ Constraints:
22
+ F8C1: system prompt must NOT contain 'evaluate'; must contain 'fail' or 'falsif'.
23
+ F8C2: test_designed must be non-empty, != claim verbatim, len >= 50 chars.
24
+ F8C3: eif_challenge with replication_mode='DASES' must include replication_result.
25
+ F8C4 [new]: adaptive evolution - round N+1 targets specifically what round N survived.
26
+ F8C5 [new]: max_rounds = 3; claim accepted only if it survives all completed rounds.
27
+ """
28
+
29
+ from __future__ import annotations
30
+
31
+ import json
32
+ import logging
33
+ import re
34
+ from collections.abc import Callable
35
+ from dataclasses import dataclass
36
+
37
+ from eif.schemas import ReplicationResult
38
+
39
+ _log = logging.getLogger(__name__)
40
+
41
+ # F8C5: maximum falsification rounds
42
+ _MAX_ROUNDS: int = 3
43
+
44
+ # F8C1: adversarial system prompt - must contain 'fail'/'falsif', must NOT contain 'evaluate'
45
+ _DASES_SYSTEM_PROMPT = """\
46
+ Your ONLY goal is to find a way this claim fails.
47
+
48
+ You are an adversarial falsifier. Your job is NOT to assess or judge whether the claim is good or bad.
49
+ Your job is to design a specific indirect test that the claim must survive to remain credible.
50
+
51
+ Rules:
52
+ 1. Do NOT test the claim directly - find an indirect path to failure.
53
+ 2. Design a test that the claimant would not anticipate.
54
+ 3. If the claim survives your falsification test, state that explicitly.
55
+ 4. If the claim fails your test, explain precisely how and why.
56
+ 5. Your test must be concrete and falsifiable - not vague.
57
+
58
+ Return JSON:
59
+ {
60
+ "test_designed": "A specific test: ...",
61
+ "test_result": "What the test revealed: ...",
62
+ "survived": true | false,
63
+ "confidence": 0.0–1.0,
64
+ "defense_mechanism": "If survived: what aspect of the claim allowed it to survive? (used to target the next round)"
65
+ }
66
+ Return ONLY the JSON.
67
+ """
68
+
69
+ _DASES_ROUND1_TEMPLATE = """\
70
+ Claim to falsify: {claim}
71
+
72
+ Supporting evidence (what the claimant would cite):
73
+ {evidence}
74
+
75
+ Session verdict: {verdict}
76
+
77
+ Design a non-obvious indirect falsification test for this claim.
78
+ """
79
+
80
+ _DASES_ADAPTIVE_TEMPLATE = """\
81
+ Claim: {claim}
82
+
83
+ This claim survived round {prev_round} of adversarial falsification.
84
+ Round {prev_round} test: {prev_test}
85
+ Round {prev_round} result: {prev_result}
86
+ The defense mechanism that allowed survival: {defense_mechanism}
87
+
88
+ Design round {this_round}: a NEW falsification test that specifically targets
89
+ the defense mechanism identified above. This test must:
90
+ 1. Be different from round {prev_round}'s test - do not repeat it.
91
+ 2. Attack specifically: {defense_mechanism}
92
+ 3. Be indirect - not a direct restatement of the claim.
93
+ 4. Be harder to survive than round {prev_round}'s test.
94
+
95
+ Supporting evidence for context:
96
+ {evidence}
97
+ """
98
+
99
+
100
+ @dataclass
101
+ class _RoundResult:
102
+ """Internal result for a single falsification round."""
103
+ round_num: int
104
+ test_designed: str
105
+ test_result: str
106
+ survived: bool
107
+ confidence: float
108
+ defense_mechanism: str = ""
109
+
110
+
111
+ def _heuristic_round1(claim_text: str) -> _RoundResult:
112
+ """Heuristic round 1 test - mechanism test."""
113
+ return _RoundResult(
114
+ round_num=1,
115
+ test_designed=_enforce_test_quality(
116
+ "Mechanism test: Does the causal mechanism described survive removal of its assumed "
117
+ "intermediate steps? Remove each link in the causal chain and verify the claim holds "
118
+ "without any of its assumed mediators.",
119
+ claim_text,
120
+ ),
121
+ test_result="Mechanism chain not independently verified - intermediate steps assumed.",
122
+ survived=False,
123
+ confidence=0.40,
124
+ defense_mechanism="",
125
+ )
126
+
127
+
128
+ def _heuristic_adaptive(claim_text: str, prev: _RoundResult) -> _RoundResult:
129
+ """Heuristic adaptive test targeting what round N exposed."""
130
+ if prev.survived:
131
+ # Probe the boundary of what survived
132
+ test = (
133
+ f"Boundary condition test: The claim survived round {prev.round_num}'s mechanism test. "
134
+ f"Now probe the claim at its stated boundary values - does it hold at the extremes "
135
+ f"of its domain, or only under the specific conditions tested? "
136
+ f"Test: apply the claim to the minimum and maximum plausible values of its key variable."
137
+ )
138
+ survived = False
139
+ defense = "survived by relying on implicit domain assumptions not stated in the claim"
140
+ else:
141
+ test = (
142
+ "Temporal stability test: Does the claim hold over time? "
143
+ "Verify whether the same evidence collected 6 months earlier or later "
144
+ "would yield the same conclusion - or whether the claim is time-sensitive "
145
+ "in ways not disclosed."
146
+ )
147
+ survived = False
148
+ defense = ""
149
+
150
+ return _RoundResult(
151
+ round_num=prev.round_num + 1,
152
+ test_designed=_enforce_test_quality(test, claim_text),
153
+ test_result=(
154
+ f"Round {prev.round_num + 1} heuristic test applied. "
155
+ "Claim does not survive boundary/temporal probing without additional evidence."
156
+ ),
157
+ survived=survived,
158
+ confidence=0.35,
159
+ defense_mechanism=defense,
160
+ )
161
+
162
+
163
+ def _heuristic_round3(claim_text: str, prev: _RoundResult) -> _RoundResult:
164
+ """Meta-assumption attack for round 3."""
165
+ test = (
166
+ "Meta-assumption test: Both prior falsification rounds tested the claim's "
167
+ "direct mechanism and boundary conditions. This round targets the meta-assumption "
168
+ "underlying those defenses: is the measurement framework itself valid? "
169
+ "Test: could the claim be an artifact of how the measurement was constructed "
170
+ "rather than a real property of the phenomenon?"
171
+ )
172
+ return _RoundResult(
173
+ round_num=3,
174
+ test_designed=_enforce_test_quality(test, claim_text),
175
+ test_result=(
176
+ "Meta-assumption test reveals the claim's conclusion depends on the validity "
177
+ "of its measurement framework, which has not been independently verified."
178
+ ),
179
+ survived=False,
180
+ confidence=0.30,
181
+ defense_mechanism="",
182
+ )
183
+
184
+
185
+ def _run_llm_round(
186
+ claim_text: str,
187
+ evidence_str: str,
188
+ session_decision: str,
189
+ round_num: int,
190
+ prev_round: _RoundResult | None,
191
+ llm_fn: Callable,
192
+ ) -> _RoundResult | None:
193
+ """Run a single LLM-driven falsification round. Returns None on failure."""
194
+ try:
195
+ if round_num == 1 or prev_round is None:
196
+ user_msg = _DASES_ROUND1_TEMPLATE.format(
197
+ claim=claim_text[:500],
198
+ evidence=evidence_str[:800],
199
+ verdict=session_decision,
200
+ )
201
+ else:
202
+ user_msg = _DASES_ADAPTIVE_TEMPLATE.format(
203
+ claim=claim_text[:400],
204
+ prev_round=prev_round.round_num,
205
+ prev_test=prev_round.test_designed[:300],
206
+ prev_result=prev_round.test_result[:200],
207
+ defense_mechanism=prev_round.defense_mechanism[:200] or "unspecified defense",
208
+ this_round=round_num,
209
+ evidence=evidence_str[:600],
210
+ )
211
+
212
+ raw = llm_fn(user_msg, f"dases_round_{round_num}", system_prompt=_DASES_SYSTEM_PROMPT)
213
+ text = raw if isinstance(raw, str) else json.dumps(raw)
214
+ text = re.sub(r"```(?:json)?\s*(.*?)\s*```", r"\1", text, flags=re.DOTALL).strip()
215
+ parsed = json.loads(text)
216
+
217
+ test_designed = _enforce_test_quality(
218
+ str(parsed.get("test_designed", "")).strip(), claim_text
219
+ )
220
+ test_result = str(parsed.get("test_result", "")).strip() or "Result not parseable"
221
+ survived = bool(parsed.get("survived", True))
222
+ confidence = max(0.0, min(1.0, float(parsed.get("confidence", 0.5))))
223
+ defense = str(parsed.get("defense_mechanism", "")).strip()
224
+
225
+ if round_num > 1 and test_designed == (prev_round.test_designed if prev_round else ""):
226
+ _log.warning("DASES: round %d repeated round %d test - using heuristic", round_num, round_num - 1)
227
+ return None
228
+
229
+ return _RoundResult(
230
+ round_num=round_num,
231
+ test_designed=test_designed,
232
+ test_result=test_result,
233
+ survived=survived,
234
+ confidence=confidence,
235
+ defense_mechanism=defense,
236
+ )
237
+
238
+ except Exception as exc: # noqa: BLE001
239
+ _log.warning("DASES: LLM call failed round %d (%s)", round_num, exc)
240
+ return None
241
+
242
+
243
+ def run_adversarial_replication(
244
+ claim_text: str,
245
+ supporting_evidence: list[str] | None = None,
246
+ session_decision: str = "UNKNOWN",
247
+ llm_fn: Callable | None = None,
248
+ max_rounds: int = _MAX_ROUNDS,
249
+ ) -> ReplicationResult:
250
+ """Run DASES adaptive multi-round adversarial falsification on a claim.
251
+
252
+ The adaptive loop (F8C4/F8C5):
253
+ Round 1: Initial indirect falsification test.
254
+ Round 2 (if round 1 survived): target the specific defense mechanism from round 1.
255
+ Round 3 (if round 2 survived): attack the meta-assumption behind round 2's defense.
256
+ Claim is ACCEPTED only if it survives all completed rounds.
257
+
258
+ Args:
259
+ claim_text: The claim to falsify.
260
+ supporting_evidence: Evidence the claimant would cite.
261
+ session_decision: The current session verdict (ACT / HALT / REVISE).
262
+ llm_fn: Optional LLM function. When None, heuristic tests are used.
263
+ max_rounds: Maximum falsification rounds (default: 3, per F8C5).
264
+
265
+ Returns:
266
+ ReplicationResult with the full multi-round audit summary.
267
+
268
+ Guarantees:
269
+ F8C1: system_prompt does not contain 'evaluate'; contains 'fail'/'falsif'.
270
+ F8C2: test_designed != claim_text verbatim; len >= 50 chars.
271
+ F8C4: round N+1 targets specifically what round N survived.
272
+ F8C5: max_rounds respected; claim accepted only if all rounds survived.
273
+ """
274
+ assert "evaluate" not in _DASES_SYSTEM_PROMPT.lower(), \
275
+ "F8C1 violated: system prompt contains 'evaluate'"
276
+ assert any(w in _DASES_SYSTEM_PROMPT.lower() for w in ("fail", "falsif")), \
277
+ "F8C1 violated: prompt missing 'fail'/'falsif'"
278
+
279
+ evidence_str = "\n".join(f"- {e}" for e in (supporting_evidence or [])) or "(none provided)"
280
+ max_rounds = max(1, min(max_rounds, _MAX_ROUNDS))
281
+
282
+ rounds: list[_RoundResult] = []
283
+ prev_round: _RoundResult | None = None
284
+
285
+ for round_num in range(1, max_rounds + 1):
286
+ # Only continue if the previous round was survived (adaptive progression)
287
+ if round_num > 1 and rounds and not rounds[-1].survived:
288
+ _log.info("DASES: claim failed round %d - stopping at round %d", round_num - 1, round_num - 1)
289
+ break
290
+
291
+ _log.info("DASES: starting round %d (max=%d)", round_num, max_rounds)
292
+
293
+ # Try LLM first, fall back to heuristic
294
+ result: _RoundResult | None = None
295
+ if llm_fn is not None:
296
+ result = _run_llm_round(
297
+ claim_text, evidence_str, session_decision,
298
+ round_num, prev_round, llm_fn,
299
+ )
300
+
301
+ if result is None:
302
+ # Heuristic path
303
+ if round_num == 1:
304
+ result = _heuristic_round1(claim_text)
305
+ elif round_num == 2:
306
+ result = _heuristic_adaptive(claim_text, prev_round) # type: ignore[arg-type]
307
+ else:
308
+ result = _heuristic_round3(claim_text, prev_round) # type: ignore[arg-type]
309
+
310
+ rounds.append(result)
311
+ prev_round = result
312
+
313
+ _log.info(
314
+ "DASES: round %d complete - survived=%s confidence=%.2f",
315
+ round_num, result.survived, result.confidence,
316
+ )
317
+
318
+ # F8C5: claim accepted only if it survived ALL completed rounds
319
+ final_survived = all(r.survived for r in rounds)
320
+ final_confidence = round(sum(r.confidence for r in rounds) / len(rounds), 4) if rounds else 0.4
321
+
322
+ # Consolidate test_designed (primary round) and test_result (multi-round summary)
323
+ primary = rounds[0] if rounds else None
324
+ test_designed = primary.test_designed if primary else _enforce_test_quality("", claim_text)
325
+ test_result = _build_multi_round_summary(rounds, claim_text)
326
+
327
+ _log.info(
328
+ "DASES: adaptive replication complete - rounds=%d final_survived=%s confidence=%.2f",
329
+ len(rounds), final_survived, final_confidence,
330
+ )
331
+
332
+ return ReplicationResult(
333
+ claim_text=claim_text,
334
+ survived=final_survived,
335
+ test_designed=test_designed,
336
+ test_result=test_result,
337
+ confidence=final_confidence,
338
+ )
339
+
340
+
341
+ def _build_multi_round_summary(rounds: list[_RoundResult], claim_text: str) -> str:
342
+ """Build a human-readable summary of all falsification rounds."""
343
+ if not rounds:
344
+ return "No rounds completed."
345
+
346
+ lines = [f"DASES adaptive falsification - {len(rounds)} round(s):"]
347
+ for r in rounds:
348
+ status = "SURVIVED" if r.survived else "FAILED"
349
+ lines.append(
350
+ f" Round {r.round_num} [{status}]: {r.test_designed[:120]}... "
351
+ f"→ {r.test_result[:100]}"
352
+ )
353
+ if r.survived and r.defense_mechanism:
354
+ lines.append(f" Defense: {r.defense_mechanism[:100]}")
355
+
356
+ all_survived = all(r.survived for r in rounds)
357
+ if all_survived:
358
+ lines.append(f"VERDICT: claim survived all {len(rounds)} round(s). ACCEPT.")
359
+ else:
360
+ failed_at = next((r.round_num for r in rounds if not r.survived), len(rounds))
361
+ lines.append(f"VERDICT: claim failed at round {failed_at}. REJECT.")
362
+
363
+ return " | ".join(lines)
364
+
365
+
366
+ def _enforce_test_quality(test_designed: str, claim_text: str) -> str:
367
+ """F8C2: Ensure test_designed is non-empty, != claim, and len >= 50 chars."""
368
+ if not test_designed or test_designed.strip() == claim_text.strip():
369
+ test_designed = (
370
+ f"Indirect boundary test: verify the claim '{claim_text[:80]}' holds "
371
+ "under conditions not stated in the original evidence (e.g. different population, "
372
+ "time period, or measurement method)."
373
+ )
374
+ if len(test_designed) < 50:
375
+ test_designed = test_designed + (
376
+ " This test targets the structural assumptions of the claim rather than its "
377
+ "direct evidential support."
378
+ )
379
+ return test_designed
@@ -0,0 +1,5 @@
1
+ """EIF cost model - empirical catch-rate measurement."""
2
+
3
+ from eif.cost_model.catch_rate_empirical import get_catch_rate_report
4
+
5
+ __all__ = ["get_catch_rate_report"]
@@ -0,0 +1,275 @@
1
+ """Empirical catch rate measurement for the compound error model (F14).
2
+
3
+ The compound error model (Architecture §9.2) uses literature-derived rates:
4
+ f = 0.6 (FALSIFY catch rate) - analogous to POPPER mechanism
5
+ u = 0.5 (UPDATE routing delta) - analogous to LAPD mechanism
6
+ c = 0.4 (CHALLENGE hit rate) - multi-agent literature estimates
7
+
8
+ The cross-check audit §5.1 flags explicitly: "These have not been measured on
9
+ an actual EIF implementation." This module closes that gap by computing actual
10
+ rates from outcome-linked provenance records (OutcomeRecord rows in the outcome_store).
11
+
12
+ Definitions:
13
+ f_empirical - FALSIFY catch rate:
14
+ fraction of HIGH-consequence WRONG claims (outcome=False) that were
15
+ caught by FALSIFY (SPRT REJECT or trivial_flag=True).
16
+ A claim is "caught" when the engine raised a flag that a decision-maker
17
+ would notice. Denominator: total HIGH-consequence claims in labeled-wrong sessions.
18
+
19
+ c_empirical - CHALLENGE hit rate:
20
+ fraction of ProvenanceRecords where ChallengeResult.counter_evidence_found=True.
21
+ Denominator: all records that ran the CHALLENGE phase (challenge is not None).
22
+
23
+ u_empirical - UPDATE routing-change rate:
24
+ fraction of ProvenanceRecords where UPDATE changed routing across a threshold
25
+ boundary (ACT↔REVISE or REVISE↔HALT). Uses UpdateResult pairs inside a record.
26
+ Denominator: records that have >= 1 UpdateResult.
27
+
28
+ The store (~/.eif/catch_rates.json) accumulates statistics across sessions using
29
+ a running mean. When >= 5 outcome-linked provenance records are measured (F14C3),
30
+ empirical rates replace literature defaults in the compound error calculation.
31
+
32
+ Research:
33
+ EIF Architecture §9.2 - compound error model with f, u, c parameters.
34
+ Cross-check audit §5.1 - "not measured on actual EIF data; may be off 20-30%".
35
+ """
36
+
37
+ from __future__ import annotations
38
+
39
+ import json
40
+ import logging
41
+ import os
42
+ import tempfile
43
+ from datetime import datetime
44
+ from pathlib import Path
45
+
46
+ from eif.schemas import (
47
+ THRESHOLD_ACT,
48
+ THRESHOLD_HALT,
49
+ CatchRateReport,
50
+ OutcomeRecord,
51
+ ProvenanceRecord,
52
+ )
53
+
54
+ _log = logging.getLogger("eif.catch_rate_empirical")
55
+
56
+ # Literature defaults from Architecture §9.2 and v3 intent F14 problem text:
57
+ # f = 0.6 (FALSIFY catch rate) - POPPER-analogous mechanism
58
+ # c = 0.4 (CHALLENGE hit rate) - multi-agent literature estimates
59
+ # u = 0.5 (UPDATE routing-change) - LAPD-analogous mechanism
60
+ # NOTE: earlier code had c=0.5/u=0.4 (swapped vs intent). Fixed to match intent.
61
+ _F_LIT: float = 0.6
62
+ _C_LIT: float = 0.4 # CHALLENGE - was incorrectly 0.5
63
+ _U_LIT: float = 0.5 # UPDATE - was incorrectly 0.4
64
+
65
+ # How many sessions needed before empirical rates replace literature defaults
66
+ EMPIRICAL_MIN_SESSIONS: int = 5
67
+ PARTIAL_MIN_SESSIONS: int = 3
68
+
69
+
70
+ def _store_path() -> Path:
71
+ base = os.environ.get("EIF_STORE_DIR", os.path.expanduser("~/.eif"))
72
+ path = Path(base) / "catch_rates.json"
73
+ path.parent.mkdir(parents=True, exist_ok=True)
74
+ return path
75
+
76
+
77
+ # ─────────────────────────────────────────────────────────────────────────────
78
+ # Measurement functions
79
+ # ─────────────────────────────────────────────────────────────────────────────
80
+
81
+
82
+ def compute_session_catch_rates(
83
+ provenance_chain: list[ProvenanceRecord],
84
+ outcome_records: list[OutcomeRecord],
85
+ ) -> CatchRateReport:
86
+ """Compute per-phase catch rates from a session's provenance chain and labeled outcomes.
87
+
88
+ outcome_records: labeled outcomes from the outcome_store for this session.
89
+ When outcome_records is empty, f_empirical is None (no ground truth).
90
+
91
+ Returns a CatchRateReport with empirical measurements and compound error
92
+ residuals using BOTH empirical and literature rates.
93
+ """
94
+ # Build lookup: provenance_record_id → outcome (True/False)
95
+ outcome_map: dict[str, bool] = {
96
+ o.provenance_record_id: o.outcome for o in outcome_records
97
+ }
98
+
99
+ # ── f_empirical: FALSIFY catch rate ──────────────────────────────────────
100
+ # f_empirical = 0.0 when denominator is 0 (no labeled wrong claims - intent F14C1).
101
+ # This is distinct from None: 0.0 means "measured, nothing to catch";
102
+ # data_status INSUFFICIENT_DATA prevents it from replacing the literature default.
103
+ f_empirical: float = 0.0
104
+ wrong_claims_total = 0
105
+ wrong_claims_caught = 0
106
+
107
+ for record in provenance_chain:
108
+ rec_outcome = outcome_map.get(record.record_id)
109
+ if rec_outcome is None:
110
+ continue # unlabeled - skip for catch-rate denominator
111
+ if rec_outcome is True:
112
+ continue # agent was correct - not a "wrong" session to catch
113
+
114
+ # Session was wrong (outcome=False)
115
+ # F14C1: denominator = HIGH-consequence claims only (intent is explicit).
116
+ # CalibrationResult now carries consequence_of_wrong; for records assembled
117
+ # before that field was added, fall back to all ASSUMED+GUESSED as proxy.
118
+ high_claims = [
119
+ c for c in record.calibration if c.consequence_of_wrong == "HIGH"
120
+ ] if record.calibration else []
121
+ if not high_claims:
122
+ # Fallback: use ASSUMED+GUESSED from registry (pre-field migration records)
123
+ n_at_risk = len(record.registry.guessed) + len(record.registry.assumed)
124
+ else:
125
+ n_at_risk = len(high_claims)
126
+
127
+ # Caught: any SPRT REJECT or trivial_flag=True in falsification conditions
128
+ has_reject = any(s.decision == "REJECT" for s in record.sprt_results)
129
+ has_trivial = any(fc.trivial_flag for fc in record.falsification_conditions)
130
+ caught = has_reject or has_trivial
131
+
132
+ wrong_claims_total += max(1, n_at_risk)
133
+ if caught:
134
+ wrong_claims_caught += max(1, n_at_risk)
135
+
136
+ if wrong_claims_total > 0:
137
+ f_empirical = wrong_claims_caught / wrong_claims_total
138
+
139
+ # ── c_empirical: CHALLENGE counter-evidence hit rate ─────────────────────
140
+ c_empirical: float | None = None
141
+ challenged_records = [r for r in provenance_chain if r.challenge is not None]
142
+ if challenged_records:
143
+ c_empirical = sum(
144
+ 1 for r in challenged_records if r.challenge.counter_evidence_found
145
+ ) / len(challenged_records)
146
+
147
+ # ── u_empirical: UPDATE routing-change rate ───────────────────────────────
148
+ u_empirical: float | None = None
149
+ records_with_updates = [r for r in provenance_chain if r.updates]
150
+ if records_with_updates:
151
+ routing_changes = 0
152
+ for record in records_with_updates:
153
+ if len(record.updates) < 1:
154
+ continue
155
+ first_update = record.updates[0]
156
+ last_update = record.updates[-1]
157
+ routing_changed = _routing_bucket(first_update.prior_posterior) != \
158
+ _routing_bucket(last_update.updated_posterior)
159
+ if routing_changed:
160
+ routing_changes += 1
161
+ u_empirical = routing_changes / len(records_with_updates)
162
+
163
+ # ── Determine data status ─────────────────────────────────────────────────
164
+ labeled_count = len([r for r in provenance_chain if r.record_id in outcome_map])
165
+ if labeled_count >= EMPIRICAL_MIN_SESSIONS:
166
+ data_status = "SUFFICIENT"
167
+ elif labeled_count >= PARTIAL_MIN_SESSIONS:
168
+ data_status = "PARTIAL"
169
+ else:
170
+ data_status = "INSUFFICIENT_DATA"
171
+
172
+ # ── Determine which rates to use for primary calculation ──────────────────
173
+ use_empirical = data_status == "SUFFICIENT"
174
+ f_used = f_empirical if (use_empirical and f_empirical is not None) else _F_LIT
175
+ c_used = c_empirical if (use_empirical and c_empirical is not None) else _C_LIT
176
+ u_used = u_empirical if (use_empirical and u_empirical is not None) else _U_LIT
177
+
178
+ # ── Compound error residuals ───────────────────────────────────────────────
179
+ # R6-06: when data_status reaches SUFFICIENT but c/u phases never ran (no
180
+ # CHALLENGE or UPDATE calls in this corpus), fill missing legs with literature
181
+ # defaults so compound_error_empirical is always populated once f is measured.
182
+ # The note field already flags INSUFFICIENT data for unavailable rates.
183
+ compound_empirical: float | None = None
184
+ if f_empirical is not None:
185
+ _c_for_compound = c_empirical if c_empirical is not None else _C_LIT
186
+ _u_for_compound = u_empirical if u_empirical is not None else _U_LIT
187
+ compound_empirical = (1 - f_empirical) * (1 - _c_for_compound) * (1 - _u_for_compound)
188
+
189
+ compound_literature = (1 - _F_LIT) * (1 - _C_LIT) * (1 - _U_LIT)
190
+
191
+ return CatchRateReport(
192
+ f_empirical=f_empirical,
193
+ c_empirical=c_empirical,
194
+ u_empirical=u_empirical,
195
+ sessions_measured=labeled_count, # count of outcome-linked provenance records (not unique session_ids)
196
+ data_status=data_status,
197
+ f_literature=_F_LIT,
198
+ c_literature=_C_LIT,
199
+ u_literature=_U_LIT,
200
+ compound_error_empirical=compound_empirical,
201
+ compound_error_literature=compound_literature,
202
+ f_used=f_used,
203
+ c_used=c_used,
204
+ u_used=u_used,
205
+ generated_at=datetime.utcnow(),
206
+ )
207
+
208
+
209
+ # ─────────────────────────────────────────────────────────────────────────────
210
+ # Persistent store
211
+ # ─────────────────────────────────────────────────────────────────────────────
212
+
213
+
214
+ def load_catch_rate_store() -> CatchRateReport | None:
215
+ """Load the latest CatchRateReport from disk, or None if not present."""
216
+ path = _store_path()
217
+ if not path.exists():
218
+ return None
219
+ try:
220
+ raw = json.loads(path.read_text(encoding="utf-8"))
221
+ return CatchRateReport.model_validate(raw)
222
+ except Exception as exc:
223
+ _log.warning("catch_rate_store: could not load %s (%s)", path, exc)
224
+ return None
225
+
226
+
227
+ def update_catch_rate_store(report: CatchRateReport) -> None:
228
+ """Persist a CatchRateReport to disk atomically."""
229
+ path = _store_path()
230
+ data = report.model_dump(mode="json")
231
+ _atomic_write(path, data)
232
+ _log.info(
233
+ "catch_rate_store: updated (outcome_linked_records=%d, status=%s, f=%.2f c=%.2f u=%.2f)",
234
+ report.sessions_measured, report.data_status,
235
+ report.f_used, report.c_used, report.u_used,
236
+ )
237
+
238
+
239
+ def get_catch_rate_report() -> CatchRateReport:
240
+ """Return the stored CatchRateReport, or a fresh default with literature values."""
241
+ stored = load_catch_rate_store()
242
+ if stored is not None:
243
+ return stored
244
+ return CatchRateReport(
245
+ data_status="INSUFFICIENT_DATA",
246
+ compound_error_literature=(1 - _F_LIT) * (1 - _C_LIT) * (1 - _U_LIT),
247
+ generated_at=datetime.utcnow(),
248
+ )
249
+
250
+
251
+ # ─────────────────────────────────────────────────────────────────────────────
252
+ # Internal helpers
253
+ # ─────────────────────────────────────────────────────────────────────────────
254
+
255
+
256
+ def _routing_bucket(posterior: float) -> str:
257
+ if posterior >= THRESHOLD_ACT:
258
+ return "ACT"
259
+ elif posterior >= THRESHOLD_HALT:
260
+ return "REVISE"
261
+ return "HALT"
262
+
263
+
264
+ def _atomic_write(path: Path, data: dict) -> None:
265
+ tmp_fd, tmp_path = tempfile.mkstemp(dir=path.parent, suffix=".tmp")
266
+ try:
267
+ with os.fdopen(tmp_fd, "w", encoding="utf-8") as f:
268
+ json.dump(data, f, indent=2, default=str)
269
+ os.replace(tmp_path, path)
270
+ except Exception:
271
+ try:
272
+ os.unlink(tmp_path)
273
+ except OSError:
274
+ pass
275
+ raise