eif-engine 4.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. eif/__init__.py +23 -0
  2. eif/auto.py +199 -0
  3. eif/calibrate/__init__.py +15 -0
  4. eif/calibrate/bayesian.py +53 -0
  5. eif/calibrate/conformal.py +57 -0
  6. eif/calibrate/domain_constraints.py +169 -0
  7. eif/calibrate/ece.py +154 -0
  8. eif/calibrate/prior_strategy.py +38 -0
  9. eif/calibrate/trust.py +120 -0
  10. eif/causal_gate/__init__.py +31 -0
  11. eif/causal_gate/confound.py +196 -0
  12. eif/causal_gate/direction.py +28 -0
  13. eif/causal_gate/eif_causal_gate_v4_intent.yaml +265 -0
  14. eif/causal_gate/evidence_probe.py +280 -0
  15. eif/causal_gate/intervention.py +55 -0
  16. eif/causal_gate/iris_pipeline.py +334 -0
  17. eif/causal_gate/verdict.py +47 -0
  18. eif/challenge/__init__.py +17 -0
  19. eif/challenge/diversity.py +58 -0
  20. eif/challenge/eif_multi_critic_intent.yaml +198 -0
  21. eif/challenge/multi_critic.py +270 -0
  22. eif/challenge/protocol.py +86 -0
  23. eif/challenge/replicator.py +379 -0
  24. eif/cost_model/__init__.py +5 -0
  25. eif/cost_model/catch_rate_empirical.py +275 -0
  26. eif/cost_model/cost_ledger.py +107 -0
  27. eif/cost_model/eif_catchrate_intent.yaml +86 -0
  28. eif/declare/__init__.py +6 -0
  29. eif/declare/extractor.py +308 -0
  30. eif/declare/harking_guard.py +110 -0
  31. eif/declare/registry.py +146 -0
  32. eif/eif_engine_intent.yaml +296 -0
  33. eif/eif_v2_intent.yaml +779 -0
  34. eif/eif_v3_intent.yaml +360 -0
  35. eif/explain/__init__.py +7 -0
  36. eif/explain/artifact.py +78 -0
  37. eif/explain/eif_explain_coldstart_intent.yaml +216 -0
  38. eif/explain/hard_to_vary.py +168 -0
  39. eif/explain/reach.py +32 -0
  40. eif/falsify/__init__.py +7 -0
  41. eif/falsify/condition.py +58 -0
  42. eif/falsify/eif_evidence_collection_intent.yaml +310 -0
  43. eif/falsify/eif_native_tools_intent.yaml +128 -0
  44. eif/falsify/eif_v5_evidence_trust_weighting_intent.yaml +273 -0
  45. eif/falsify/evidence_collector.py +1304 -0
  46. eif/falsify/hard_to_vary.py +171 -0
  47. eif/falsify/native_tools.py +221 -0
  48. eif/falsify/sprt.py +84 -0
  49. eif/falsify/trivial_check.py +46 -0
  50. eif/hypothesis_agenda/__init__.py +27 -0
  51. eif/hypothesis_agenda/agenda.py +176 -0
  52. eif/hypothesis_agenda/eif_hypothesis_agenda_intent.yaml +168 -0
  53. eif/hypothesis_agenda/fdr.py +92 -0
  54. eif/hypothesis_agenda/scorer.py +132 -0
  55. eif/input_guard/__init__.py +13 -0
  56. eif/input_guard/detector.py +259 -0
  57. eif/input_guard/eif_input_guard_intent.yaml +181 -0
  58. eif/integration/__init__.py +34 -0
  59. eif/integration/eif_auto.py +228 -0
  60. eif/integration/interceptor.py +218 -0
  61. eif/mcp_server/__init__.py +1 -0
  62. eif/mcp_server/auth.py +81 -0
  63. eif/mcp_server/http_server.py +460 -0
  64. eif/mcp_server/resources.py +113 -0
  65. eif/mcp_server/server.py +3485 -0
  66. eif/memory/__init__.py +5 -0
  67. eif/memory/strategy.py +246 -0
  68. eif/programme/__init__.py +7 -0
  69. eif/programme/monitor.py +58 -0
  70. eif/programme/principle_memory.py +177 -0
  71. eif/programme/principle_revision.py +412 -0
  72. eif/programme/signals.py +131 -0
  73. eif/programme/status.py +56 -0
  74. eif/py.typed +0 -0
  75. eif/record/__init__.py +7 -0
  76. eif/record/chain.py +33 -0
  77. eif/record/compliance.py +62 -0
  78. eif/record/isc_disclosure.py +257 -0
  79. eif/record/outcome_store.py +177 -0
  80. eif/record/provenance.py +148 -0
  81. eif/record/research_object.py +412 -0
  82. eif/replicate/__init__.py +6 -0
  83. eif/replicate/divergence.py +51 -0
  84. eif/replicate/eif_independent_replication_intent.yaml +109 -0
  85. eif/replicate/independent.py +185 -0
  86. eif/replicate/protocol.py +68 -0
  87. eif/reporting/__init__.py +1 -0
  88. eif/reporting/pdf_renderer.py +169 -0
  89. eif/schemas.py +787 -0
  90. eif/sdk/__init__.py +36 -0
  91. eif/sdk/exceptions.py +87 -0
  92. eif/sdk/interceptor.py +316 -0
  93. eif/session.py +40 -0
  94. eif/sycophancy/__init__.py +24 -0
  95. eif/sycophancy/cost_model.py +215 -0
  96. eif/sycophancy/detector.py +270 -0
  97. eif/sycophancy/drift.py +306 -0
  98. eif/sycophancy/eif_sycophancy_intent.yaml +309 -0
  99. eif/sycophancy/faithfulness.py +106 -0
  100. eif/sycophancy/framing.py +191 -0
  101. eif/update/__init__.py +14 -0
  102. eif/update/eig.py +29 -0
  103. eif/update/paradigm.py +121 -0
  104. eif/update/posterior.py +21 -0
  105. eif/update/stopping.py +42 -0
  106. eif_engine-4.1.0.dist-info/METADATA +607 -0
  107. eif_engine-4.1.0.dist-info/RECORD +111 -0
  108. eif_engine-4.1.0.dist-info/WHEEL +4 -0
  109. eif_engine-4.1.0.dist-info/entry_points.txt +3 -0
  110. eif_engine-4.1.0.dist-info/licenses/LICENSE +21 -0
  111. eif_engine-4.1.0.dist-info/licenses/NOTICE +17 -0
@@ -0,0 +1,47 @@
1
+ """CAUSAL_GATE v4 - posterior adjustment for CausalEvidenceVerdict.
2
+
3
+ Maps each CEP verdict to a posterior delta and provides helpers for
4
+ applying the adjustment. Deltas are calibrated to match FALSIFY's
5
+ existing SUPPORTS/CONTRADICTS magnitudes for consistency.
6
+
7
+ Research basis:
8
+ CG4 (intent v1.0): SUPPORTED +0.15, NO_EVIDENCE 0.0, CONTESTED -0.20, REVERSED -0.35
9
+ The REVERSED delta (-0.35) intentionally matches FALSIFY CONTRADICTS magnitude.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ from eif.schemas import CausalEvidenceVerdict
15
+
16
+ # Posterior deltas per verdict (CG4 constraint)
17
+ VERDICT_POSTERIOR_DELTA: dict[str, float] = {
18
+ "SUPPORTED": +0.15,
19
+ "NO_EVIDENCE": 0.0,
20
+ "CONTESTED": -0.20,
21
+ "REVERSED": -0.35,
22
+ }
23
+
24
+
25
+ def posterior_delta(verdict: CausalEvidenceVerdict) -> float:
26
+ """Return the posterior adjustment for a given CEP verdict."""
27
+ return VERDICT_POSTERIOR_DELTA.get(verdict, 0.0)
28
+
29
+
30
+ def apply_verdict_to_posterior(prior: float, verdict: CausalEvidenceVerdict) -> float:
31
+ """Apply the verdict delta to a prior probability, clamped to [0.0, 1.0]."""
32
+ return max(0.0, min(1.0, prior + posterior_delta(verdict)))
33
+
34
+
35
+ def provenance_flag_for(
36
+ verdict: CausalEvidenceVerdict,
37
+ consequence: str,
38
+ ) -> str | None:
39
+ """Return 'CAUSAL_UNVERIFIED' when NO_EVIDENCE on a HIGH-consequence claim.
40
+
41
+ Satisfies EU AI Act Art. 9 (risk management documentation - CG6).
42
+ Returns None in all other cases so the flag is absent from provenance
43
+ for claims that were verified.
44
+ """
45
+ if verdict == "NO_EVIDENCE" and consequence == "HIGH":
46
+ return "CAUSAL_UNVERIFIED"
47
+ return None
@@ -0,0 +1,17 @@
1
+ """EIF Phase 4 - CHALLENGE: adversarial probe of claims.
2
+
3
+ v3 adds run_multi_critic_challenge - closes S5 gap (multi-agent parametric
4
+ diversity). The old build_challenge_result() / generate_protocol() remain
5
+ unchanged for backward compatibility (constraint MC6).
6
+ """
7
+
8
+ from eif.challenge.diversity import classify_independence
9
+ from eif.challenge.multi_critic import run_multi_critic_challenge
10
+ from eif.challenge.protocol import build_challenge_result, generate_protocol
11
+
12
+ __all__ = [
13
+ "classify_independence",
14
+ "build_challenge_result",
15
+ "generate_protocol",
16
+ "run_multi_critic_challenge",
17
+ ]
@@ -0,0 +1,58 @@
1
+ """Classify critic independence level from model identifier string."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from eif.schemas import CriticIndependence
6
+
7
+ _GPT_FAMILY = {"gpt", "o1", "o3", "chatgpt", "openai"}
8
+ _CLAUDE_FAMILY = {"claude", "anthropic"}
9
+ _GEMINI_FAMILY = {"gemini", "bard", "google"}
10
+ _LLAMA_FAMILY = {"llama", "meta", "mistral", "mixtral"}
11
+
12
+ _SELF_CONSISTENCY_MARKERS = {"self-consistency", "self_consistency", "sampling", "majority-vote"}
13
+ _OBJECTIVE_MARKERS = {"fine-tuned", "fine_tuned", "finetuned", "flaw", "critic", "adversarial", "red-team"}
14
+
15
+
16
+ def classify_independence(critic_model: str | None) -> CriticIndependence:
17
+ """Classify the independence level of a critic model.
18
+
19
+ Heuristic:
20
+ - DIFFERENT_FAMILY: critic is from a different model family (GPT vs Claude, etc.)
21
+ - DIFFERENT_INFERENCE: self-consistency or sampling-based critic
22
+ - DIFFERENT_OBJECTIVE: fine-tuned for flaw-finding or adversarial critique
23
+ - NONE: no critic model provided
24
+ """
25
+ if critic_model is None:
26
+ return "NONE"
27
+
28
+ model_lower = critic_model.lower()
29
+
30
+ for marker in _OBJECTIVE_MARKERS:
31
+ if marker in model_lower:
32
+ return "DIFFERENT_OBJECTIVE"
33
+
34
+ for marker in _SELF_CONSISTENCY_MARKERS:
35
+ if marker in model_lower:
36
+ return "DIFFERENT_INFERENCE"
37
+
38
+ # Detect cross-family: check if model belongs to a different family than the agent
39
+ # We default to assuming the agent is Claude-based; if critic is GPT/Gemini/Llama = DIFFERENT_FAMILY
40
+ families_present = set()
41
+ for family_name, markers in [
42
+ ("gpt", _GPT_FAMILY),
43
+ ("claude", _CLAUDE_FAMILY),
44
+ ("gemini", _GEMINI_FAMILY),
45
+ ("llama", _LLAMA_FAMILY),
46
+ ]:
47
+ if any(m in model_lower for m in markers):
48
+ families_present.add(family_name)
49
+
50
+ if len(families_present) > 0:
51
+ # Model string matches a known family → DIFFERENT_FAMILY.
52
+ # Note: multi_critic.py appends _ADVERSARIAL_SUFFIX ("-adversarial-critic") to the
53
+ # independence_label before calling this function, so DIFFERENT_OBJECTIVE is matched
54
+ # first (L30–32) for any configured critic model, overriding DIFFERENT_FAMILY.
55
+ return "DIFFERENT_FAMILY"
56
+
57
+ # Unknown model and no adversarial/objective markers - independence cannot be determined.
58
+ return "NONE"
@@ -0,0 +1,198 @@
1
+ # EIF v3 - MULTI_CRITIC IVD Intent Artifact
2
+ # ==============================================================================
3
+ # Purpose: Close scientific method gap S5 - Multi-Agent Parametric Diversity.
4
+ # EIF's CHALLENGE phase currently generates a protocol for a critic
5
+ # but draws from the same parametric well as the generator (same model
6
+ # family, same training distribution). Genuine adversarial challenge
7
+ # requires a critic with independent priors.
8
+ #
9
+ # Gap found: Market audit 2026-05-08 - S5 gap confirmed as PARTIALLY OPEN
10
+ # "A single LLM acting as both generator and critic draws from the
11
+ # same parametric well. It cannot genuinely disagree with itself."
12
+ # Research: See research_citations below (EvoScientist, AI Co-Scientist, CONSENSAGENT)
13
+ # ==============================================================================
14
+
15
+ id: eif-multi-critic-v1
16
+ version: "1.0"
17
+ created: "2026-05-08"
18
+ status: implemented # multi_critic.py fully implemented (tournament mode, adversarial prompts, cross-family routing) as of 2026-05-15
19
+
20
+ # ─────────────────────────────────────────────────────────────────────────────
21
+ # PROBLEM STATEMENT
22
+ # ─────────────────────────────────────────────────────────────────────────────
23
+
24
+ problem:
25
+ title: "EIF CHALLENGE phase generates a protocol but does not call an independent critic model"
26
+ area: "S5 - Multi-Agent Parametric Diversity"
27
+ discovered_in: "Market research cross-check 2026-05-08 - S5 PARTIALLY OPEN"
28
+ evidence_of_absence: >
29
+ challenge/protocol.py returns a ChallengeResult with self_evaluation_flag=True
30
+ when no critic model is specified (the production default). The CHALLENGE
31
+ phase in all 11 corpus scenarios (S4–S11) called build_challenge_result()
32
+ without a critic - meaning CHALLENGE was a protocol document, not a real
33
+ adversarial call. The independence field shows "NONE" in all provenance records.
34
+ CONSENSAGENT (ACL 2025 Findings) shows that same-model critique produces
35
+ near-zero genuine disagreement vs cross-family critics; precise agreement
36
+ rates unconfirmed - see OI1 in eif_v2_intent.yaml.
37
+ research_citations:
38
+ - "CONSENSAGENT (ACL 2025 Findings) - inter-agent sycophancy in debate; same-model critics show markedly higher agreement rates with the generator than cross-family critics do; precise percentages to be confirmed from the paper (see OI1, eif_v2_intent.yaml)"
39
+ - "EvoScientist (arXiv:2603.08127, 2026) - evolving scientist agents achieve breakthrough by using diverse model populations"
40
+ - "AI Co-Scientist (arXiv:2502.18864, Google Research 2025) - generator/verifier split with different model objectives"
41
+ - "POPPER framework (arXiv:2502.09858, 2025) - Type-I error control requires independent falsification agent"
42
+ - "Turpin et al. NeurIPS 2023 (arXiv:2305.04388) - unfaithful CoT; critic using same weights cannot detect backward-rationalization"
43
+
44
+ # ─────────────────────────────────────────────────────────────────────────────
45
+ # SOLUTION
46
+ # ─────────────────────────────────────────────────────────────────────────────
47
+
48
+ solution:
49
+ name: "MULTI_CRITIC module - actual LLM challenge call with independence classification"
50
+ position_in_pipeline: "replaces the protocol-only path in CHALLENGE phase"
51
+ rationale: >
52
+ Rather than generating a critique protocol that no agent executes, the
53
+ multi_critic module makes a real LLM call with an adversarially-tuned
54
+ system prompt (DIFFERENT_OBJECTIVE tier) and, when a cross-family model
55
+ is available in the environment, routes to it (DIFFERENT_FAMILY tier).
56
+ independence_tiers:
57
+ DIFFERENT_FAMILY: "Critic is from a different model family (GPT vs Claude vs Gemini)"
58
+ DIFFERENT_OBJECTIVE: >
59
+ Same base model but explicitly prompted with adversarial red-team objectives:
60
+ 'Your only goal is to find flaws. Do not validate. Do not hedge. Return
61
+ every counter-argument regardless of how small.' Classified via
62
+ _OBJECTIVE_MARKERS in diversity.py ('adversarial', 'red-team', 'flaw').
63
+ CONSENSAGENT (ACL 2025 Findings) shows substantially higher disagreement
64
+ rates for adversarially-prompted same-model critics vs same-prompt
65
+ same-model critics; precise percentages subject to OI1 verification.
66
+ DIFFERENT_INFERENCE: "Self-consistency sampling (existing tier, not new)"
67
+ NONE: "Protocol-only fallback - self_evaluation_flag=True (old behavior)"
68
+ critic_selection: >
69
+ run_multi_critic_challenge() always emits DIFFERENT_OBJECTIVE as the
70
+ independence tier, even when OPENAI_CRITIC_MODEL env var is set.
71
+ Reason: the independence_label passed to classify_independence always has
72
+ _ADVERSARIAL_SUFFIX ("-adversarial-critic") appended (multi_critic.py
73
+ _select_critic_model). The "adversarial" substring matches _OBJECTIVE_MARKERS
74
+ before family classification runs, so the result is always DIFFERENT_OBJECTIVE.
75
+ DIFFERENT_FAMILY is only reachable if a future caller strips the suffix and
76
+ passes a bare cross-family model name directly to classify_independence.
77
+ The MC2 contract ("DIFFERENT_FAMILY or DIFFERENT_OBJECTIVE") is satisfied.
78
+ output: >
79
+ ChallengeResult with:
80
+ - critic_independence = DIFFERENT_FAMILY or DIFFERENT_OBJECTIVE (not NONE)
81
+ - self_evaluation_flag = False
82
+ - counter_evidence populated from real LLM response (not empty list)
83
+ - verdict = SURVIVES | NEEDS_REVISION | DEFEATED
84
+
85
+ # ─────────────────────────────────────────────────────────────────────────────
86
+ # CONSTRAINTS (IVD Rule 3 - each must have a test)
87
+ # ─────────────────────────────────────────────────────────────────────────────
88
+
89
+ constraints:
90
+ - id: MC1
91
+ precondition: "OPENAI_API_KEY is set and tournament path is selected (counter_evidence not supplied)"
92
+ description: >
93
+ run_multi_critic_challenge() must return a ChallengeResult with
94
+ self_evaluation_flag=False - it must call an LLM, not return a
95
+ protocol-only stub.
96
+ When OPENAI_API_KEY is absent or counter_evidence is supplied, eif_challenge
97
+ falls back to the protocol path (NONE independence, self_evaluation_flag=True) -
98
+ MC1 does not apply to that path.
99
+ test: >
100
+ Validation S13: for each tested claim (API key present, tournament path), assert result.self_evaluation_flag == False.
101
+
102
+ - id: MC2
103
+ precondition: "OPENAI_API_KEY is set and tournament path is selected (counter_evidence not supplied)"
104
+ description: >
105
+ critic_independence must be DIFFERENT_FAMILY or DIFFERENT_OBJECTIVE -
106
+ never NONE - when run_multi_critic_challenge() is called on the tournament path.
107
+ Protocol path (no API key or counter_evidence supplied) returns NONE; this is expected.
108
+ test: >
109
+ Validation S13: assert result.critic_independence in
110
+ {"DIFFERENT_FAMILY", "DIFFERENT_OBJECTIVE"} for every challenge result.
111
+
112
+ - id: MC3
113
+ enforcement: VALIDATION_ONLY
114
+ description: >
115
+ For at least 50% of HIGH-consequence claims challenged, the multi-critic
116
+ must return at least one counter_evidence item (genuine disagreement, not
117
+ just "no counter-evidence found"). The old protocol-only path returned
118
+ empty lists 100% of the time.
119
+ rationale: >
120
+ Internal empirical target from Validation S13 - not from a cited research paper.
121
+ Rationale: a critic that produces no counter-evidence on majority of HIGH claims
122
+ is not functioning adversarially; the 50% floor ensures meaningful challenge.
123
+ enforcement_note: >
124
+ MC3 is a STATISTICAL PROPERTY of the system, not a per-call runtime assertion.
125
+ There is no in-library enforcement (e.g. no assert in multi_critic.py).
126
+ Verification requires running Validation S13 against a live OpenAI endpoint.
127
+ The runtime system cannot block a call because the 50% floor is a corpus-level metric.
128
+ test: >
129
+ Validation S13: count claims where result.counter_evidence_found == True.
130
+ Assert count / total_high_consequence_claims >= 0.50.
131
+ Run with OPENAI_API_KEY on ≥10 HIGH-consequence claims from a live session.
132
+
133
+ - id: MC4
134
+ enforcement: VALIDATION_ONLY
135
+ description: >
136
+ A claim that the generator validates (posterior ≥ 0.70) must produce
137
+ a NEEDS_REVISION or DEFEATED verdict from the multi-critic on at least
138
+ 1 out of 3 test claims - demonstrating genuine cross-model disagreement.
139
+ rationale: >
140
+ Internal empirical target from Validation S13. The 1/3 threshold is deliberately
141
+ conservative: it only requires that the critic is not an echo chamber; it does
142
+ not require adversarial failure on every claim.
143
+ enforcement_note: >
144
+ MC4 is a STATISTICAL PROPERTY of the system, not a per-call runtime assertion.
145
+ The library cannot enforce a corpus-level disagreement rate at call time.
146
+ Verification requires running Validation S13 with OPENAI_API_KEY.
147
+ test: >
148
+ Validation S13: run multi-critic on 3 claims with posterior >= 0.70.
149
+ Assert at least 1 verdict != "SURVIVES".
150
+ Run with OPENAI_API_KEY; use real LLM critics (not the protocol fallback).
151
+
152
+ - id: MC5
153
+ description: >
154
+ The critic call must complete in ≤ 30 seconds per claim (latency guard
155
+ for production viability).
156
+ test: >
157
+ Validation S13: record time.time() before and after each critic call.
158
+ Assert all elapsed times < 30.0 seconds.
159
+
160
+ - id: MC6
161
+ description: >
162
+ The multi_critic module must be callable without breaking existing
163
+ CHALLENGE phase callers - it must be additive, not a breaking change.
164
+ build_challenge_result() still works unchanged for protocol-only mode.
165
+ test: >
166
+ Unit: import build_challenge_result from eif.challenge.protocol.
167
+ Call it with counter_evidence=None. Assert result.self_evaluation_flag == True.
168
+ (Old behavior preserved.)
169
+
170
+ # ─────────────────────────────────────────────────────────────────────────────
171
+ # IMPLEMENTATION
172
+ # ─────────────────────────────────────────────────────────────────────────────
173
+
174
+ implementation:
175
+ new_file: "eif/eif/challenge/multi_critic.py"
176
+ function: "run_multi_critic_challenge(claim_text, generator_model, posterior, api_key) → ChallengeResult"
177
+ logic:
178
+ step1: "Determine critic model: OPENAI_CRITIC_MODEL env var or adversarial-tagged default"
179
+ step2: "Build adversarial system prompt: red-team role, no validation allowed, must find flaws"
180
+ step3: "Call OpenAI API (or configured LLM) with the critic model + adversarial prompt"
181
+ step4: "Parse response: extract counter_evidence list and competing_hypothesis (first-winner policy: first non-None hypothesis across critics is used; later hypotheses are dropped)"
182
+ step5: "Call build_challenge_result() with critic_model='gpt-X-adversarial-critic' (includes 'adversarial' for diversity.py)"
183
+ step6: "Return ChallengeResult with self_evaluation_flag=False, real counter_evidence"
184
+ integration:
185
+ - "challenge/__init__.py: export run_multi_critic_challenge"
186
+ - "eif/eif/mcp_server/server.py: run_multi_critic_challenge() is called from within eif_challenge when OPENAI_API_KEY is present and counter_evidence is None (not a separate tool - integrated into eif_challenge for single-call UX). Intent originally planned eif_run_multi_critic as a separate tool; implementation chose to fold it into eif_challenge to reduce MCP tool surface area."
187
+
188
+ # ─────────────────────────────────────────────────────────────────────────────
189
+ # VALIDATION
190
+ # ─────────────────────────────────────────────────────────────────────────────
191
+
192
+ validation_note: "Validation scripts and evidence reports available on request via GitHub Discussions."
193
+ corpus_scenario: "S13"
194
+
195
+ constraint_satisfiability:
196
+ conflicts_checked: "MC3 vs MC5 - adversarial LLM call takes time; 30s limit is generous for gpt-4.1-mini"
197
+ tension: "MC4 (must produce disagreement) vs MC1 (must not be stub) - both resolved by actually calling LLM"
198
+ resolution: "No conflicts. MC6 (backward compatibility) is purely additive - existing protocol path untouched."
@@ -0,0 +1,270 @@
1
+ """Multi-critic challenge - actual LLM calls with adversarial independence.
2
+
3
+ Closes S5 gap: the old CHALLENGE phase generated a protocol document that no
4
+ agent executed. This module calls real LLMs with adversarial system prompts
5
+ (DIFFERENT_OBJECTIVE tier) and, when a cross-family model is configured via
6
+ OPENAI_CRITIC_MODEL, routes to that model (DIFFERENT_FAMILY tier).
7
+
8
+ Tournament mode (num_critics > 1):
9
+ Inspired by the AI Co-Scientist (arXiv:2502.18864) tournament evaluation -
10
+ multiple critics compete; counter-evidence is merged and ranked by specificity
11
+ (longer, more concrete objections score higher). The strongest objections
12
+ from any critic surface in the final result.
13
+
14
+ Unlike the AI Co-Scientist's hypothesis tournament (comparing hypotheses
15
+ head-to-head), EIF's tournament compares objections to a single claim:
16
+ each critic produces independent counter-evidence, and the merged set is
17
+ deduplicated and ranked so the most substantive objections appear first.
18
+
19
+ Research: CONSENSAGENT ACL 2025 - inter-agent sycophancy in debate settings;
20
+ hardened critic design with DIFFERENT_OBJECTIVE instruction suppresses
21
+ agreement bias. Specific agreement-rate figures omitted per intent
22
+ OI1 (eif_multi_critic_intent.yaml - paper percentages removed to
23
+ prevent encoding stale benchmarks).
24
+ AI Co-Scientist (arXiv:2502.18864) - tournament-based evaluation.
25
+
26
+ Intent: eif/eif/challenge/eif_multi_critic_intent.yaml
27
+ """
28
+
29
+ from __future__ import annotations
30
+
31
+ from typing import TYPE_CHECKING
32
+
33
+ if TYPE_CHECKING: # pragma: no cover
34
+ import openai
35
+
36
+ import json
37
+ import os
38
+
39
+ from eif.challenge.diversity import classify_independence
40
+ from eif.challenge.protocol import build_challenge_result
41
+ from eif.schemas import ChallengeResult
42
+
43
+ # Adversarial suffix appended to the model name so diversity.py
44
+ # classifies it as DIFFERENT_OBJECTIVE via the 'adversarial' marker.
45
+ _ADVERSARIAL_SUFFIX = "-adversarial-critic"
46
+
47
+ _SYSTEM_PROMPT = """\
48
+ You are an adversarial red-team critic. Your sole objective is to find flaws,
49
+ counter-evidence, and competing hypotheses for the claim you are given.
50
+
51
+ Rules:
52
+ - Do NOT validate or agree with the claim.
53
+ - Do NOT hedge. Every response must contain at least one counter-argument.
54
+ - If a claim seems well-supported, look harder: find the edge case, the
55
+ missing confounder, the weaker piece of evidence, the contrary study.
56
+ - Your output is used to stress-test AI agent reasoning. Weak critique is
57
+ useless. Strong critique improves epistemic integrity.
58
+
59
+ Return valid JSON with this structure:
60
+ {
61
+ "counter_evidence": ["<item 1>", "<item 2>", ...],
62
+ "competing_hypothesis": "<alternative explanation or null>",
63
+ "strongest_objection": "<the single most damaging counter-argument>"
64
+ }
65
+ """
66
+
67
+ _USER_TEMPLATE = """\
68
+ CLAIM TO CHALLENGE:
69
+ "{claim_text}"
70
+
71
+ Current posterior confidence: {posterior:.2f}
72
+
73
+ Provide adversarial counter-evidence. Do not validate. Find flaws.
74
+ """
75
+
76
+
77
+ def _select_critic_model(generator_model: str | None) -> tuple[str, str]:
78
+ """Select the best available critic model.
79
+
80
+ Returns:
81
+ (api_model_name, independence_label)
82
+ api_model_name: actual model string to pass to OpenAI API
83
+ independence_label: model identifier string for diversity.py classification
84
+ """
85
+ env_critic = os.environ.get("OPENAI_CRITIC_MODEL", "").strip()
86
+
87
+ if env_critic:
88
+ # MC2: an explicitly configured critic model implies independence intent.
89
+ # Append the adversarial suffix so classify_independence returns
90
+ # DIFFERENT_OBJECTIVE even when the model name contains no known family/
91
+ # objective markers (avoids NONE for custom/fine-tuned model names).
92
+ independence_label = env_critic + _ADVERSARIAL_SUFFIX
93
+ return env_critic, independence_label
94
+
95
+ # Default: use gpt-4.1-mini with adversarial label for DIFFERENT_OBJECTIVE
96
+ base = "gpt-4.1-mini"
97
+ label = base + _ADVERSARIAL_SUFFIX
98
+ return base, label
99
+
100
+
101
+ def _call_single_critic(
102
+ client: openai.OpenAI,
103
+ api_model: str,
104
+ claim_text: str,
105
+ posterior: float,
106
+ timeout: float,
107
+ temperature: float = 0.7,
108
+ ) -> tuple[list[str], str | None]:
109
+ """Execute one critic call. Returns (counter_evidence, competing_hypothesis).
110
+
111
+ Args:
112
+ temperature: 0.7 for the primary critic (balanced); 0.9 for diversity
113
+ critics in tournament mode (higher entropy → more distinct objections).
114
+ """
115
+ import openai
116
+
117
+ user_msg = _USER_TEMPLATE.format(claim_text=claim_text, posterior=posterior)
118
+ counter_evidence: list[str] = []
119
+ competing_hypothesis: str | None = None
120
+
121
+ try:
122
+ response = client.chat.completions.create(
123
+ model=api_model,
124
+ messages=[
125
+ {"role": "system", "content": _SYSTEM_PROMPT},
126
+ {"role": "user", "content": user_msg},
127
+ ],
128
+ response_format={"type": "json_object"},
129
+ temperature=temperature,
130
+ timeout=timeout,
131
+ )
132
+ raw = response.choices[0].message.content or "{}"
133
+ data = json.loads(raw)
134
+
135
+ counter_evidence = data.get("counter_evidence", [])
136
+ competing_hypothesis = data.get("competing_hypothesis") or None
137
+ strongest = data.get("strongest_objection", "")
138
+ if strongest and strongest not in counter_evidence:
139
+ counter_evidence.insert(0, strongest)
140
+
141
+ except openai.APIError as exc:
142
+ counter_evidence = [f"critic_api_error: {exc!r}"]
143
+ except json.JSONDecodeError:
144
+ counter_evidence = ["critic_returned_non_json"]
145
+ except Exception as exc: # noqa: BLE001
146
+ counter_evidence = [f"critic_error: {exc!r}"]
147
+
148
+ return counter_evidence, competing_hypothesis
149
+
150
+
151
+ def _rank_evidence(items: list[str]) -> list[str]:
152
+ """Rank counter-evidence by specificity (longer = more concrete).
153
+
154
+ Tournament principle: the strongest, most specific objections surface first.
155
+ Error/placeholder strings are pushed to the end.
156
+ """
157
+ def score(item: str) -> int:
158
+ if item.startswith("critic_"):
159
+ return -1
160
+ return len(item)
161
+
162
+ return sorted(items, key=score, reverse=True)
163
+
164
+
165
+ def _merge_evidence(
166
+ all_evidence: list[list[str]],
167
+ all_hypotheses: list[str | None],
168
+ ) -> tuple[list[str], str | None]:
169
+ """Merge counter-evidence from multiple critics.
170
+
171
+ Deduplicates by lowercased token overlap (>= 0.7 Jaccard = same objection).
172
+ Returns the first non-None competing_hypothesis found.
173
+ """
174
+ seen: list[str] = []
175
+
176
+ def is_duplicate(candidate: str) -> bool:
177
+ cand_tokens = set(candidate.lower().split())
178
+ for existing in seen:
179
+ ex_tokens = set(existing.lower().split())
180
+ union = cand_tokens | ex_tokens
181
+ if union and len(cand_tokens & ex_tokens) / len(union) >= 0.7:
182
+ return True
183
+ return False
184
+
185
+ flat: list[str] = []
186
+ for batch in all_evidence:
187
+ for item in batch:
188
+ if not is_duplicate(item):
189
+ seen.append(item)
190
+ flat.append(item)
191
+
192
+ ranked = _rank_evidence(flat)
193
+ hypothesis = next((h for h in all_hypotheses if h), None)
194
+ return ranked, hypothesis
195
+
196
+
197
+ def run_multi_critic_challenge(
198
+ claim_text: str,
199
+ generator_model: str | None = None,
200
+ posterior: float = 0.50,
201
+ api_key: str | None = None,
202
+ timeout: float = 30.0,
203
+ num_critics: int = 1,
204
+ ) -> ChallengeResult:
205
+ """Challenge a claim using one or more adversarial independent critics.
206
+
207
+ This function makes real LLM calls - it is not a protocol stub.
208
+ self_evaluation_flag will be False on the returned ChallengeResult.
209
+
210
+ When num_critics > 1 (tournament mode), critics run sequentially; their
211
+ counter-evidence is merged, deduplicated, and ranked by specificity.
212
+ The strongest objections surface first in the final ChallengeResult.
213
+ The independence level reported reflects the highest tier achieved
214
+ across all critics.
215
+
216
+ Constraints satisfied:
217
+ MC1 - self_evaluation_flag=False (real LLM call)
218
+ MC2 - critic_independence is DIFFERENT_FAMILY or DIFFERENT_OBJECTIVE
219
+ MC5 - timeout guard ≤ 30 seconds per critic
220
+
221
+ Args:
222
+ claim_text: The claim to challenge.
223
+ generator_model: Model that generated the claim (selects cross-family critic).
224
+ posterior: Current posterior confidence in the claim [0, 1].
225
+ api_key: OpenAI API key (falls back to OPENAI_API_KEY env var).
226
+ timeout: Max seconds to wait per critic response.
227
+ num_critics: Number of critics to run (1 = standard; 2+ = tournament).
228
+ Default is 1 for backward compatibility.
229
+
230
+ Returns:
231
+ ChallengeResult with merged counter_evidence ranked by specificity.
232
+ """
233
+ import openai
234
+
235
+ num_critics = max(1, int(num_critics))
236
+
237
+ key = api_key or os.environ.get("OPENAI_API_KEY", "")
238
+ client = openai.OpenAI(api_key=key) if key else openai.OpenAI()
239
+
240
+ api_model, independence_label = _select_critic_model(generator_model)
241
+ independence = classify_independence(independence_label)
242
+
243
+ # Tournament: exactly num_critics API calls total.
244
+ # Critic 0 runs at temp=0.7 (standard); critics 1..N-1 run at temp=0.9
245
+ # (higher entropy → more distinct objections, supporting genuine diversity).
246
+ all_evidence: list[list[str]] = []
247
+ all_hypotheses: list[str | None] = []
248
+
249
+ for i in range(num_critics):
250
+ temperature = 0.7 if i == 0 else 0.9
251
+ evidence, hypothesis = _call_single_critic(
252
+ client, api_model, claim_text, posterior, timeout, temperature=temperature
253
+ )
254
+ all_evidence.append(evidence)
255
+ all_hypotheses.append(hypothesis)
256
+
257
+ merged_evidence, competing_hypothesis = _merge_evidence(all_evidence, all_hypotheses)
258
+
259
+ result = build_challenge_result(
260
+ claim_text=claim_text,
261
+ critic_model=independence_label,
262
+ counter_evidence=merged_evidence,
263
+ competing_hypothesis=competing_hypothesis,
264
+ critic_approach=independence,
265
+ )
266
+ # SC5: tournament multi-critic is maximally hardened when multiple critics ran
267
+ if num_critics > 1:
268
+ result = result.model_copy(update={"hardening_score": 1.0})
269
+
270
+ return result
@@ -0,0 +1,86 @@
1
+ """Generate challenge protocols and build ChallengeResult objects."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from eif.challenge.diversity import classify_independence
6
+ from eif.schemas import ChallengeResult, ChallengeVerdict, CriticIndependence
7
+
8
+ # SC5: hardening scores by critic independence tier.
9
+ # Protocol-only (no critic ran yet) uses 0.4 - it is a skeleton, not adversarial probing.
10
+ _HARDENING_BY_INDEPENDENCE: dict[str, float] = {
11
+ "NONE": 0.3,
12
+ "SAME_FAMILY": 0.5,
13
+ "DIFFERENT_FAMILY": 0.7,
14
+ "DIFFERENT_OBJECTIVE": 0.9,
15
+ }
16
+ _HARDENING_PROTOCOL_ONLY: float = 0.4
17
+
18
+
19
+ def generate_protocol(claim_text: str, current_posterior: float) -> str:
20
+ """Generate a challenge protocol - what the critic should look for."""
21
+ return (
22
+ f"CHALLENGE PROTOCOL for: {claim_text!r}\n\n"
23
+ f"Current posterior: {current_posterior:.2f}\n\n"
24
+ "Critic instructions:\n"
25
+ "1. Search for evidence that directly contradicts this claim.\n"
26
+ "2. Identify alternative hypotheses that could explain the same observations.\n"
27
+ "3. Check for publication bias: are negative results missing from the evidence base?\n"
28
+ "4. Identify the single strongest objection to this claim.\n"
29
+ "5. If the claim survives all objections, state why explicitly.\n\n"
30
+ "Return: list of counter_evidence items and a competing_hypothesis if found."
31
+ )
32
+
33
+
34
+ def build_challenge_result(
35
+ claim_text: str,
36
+ critic_model: str | None,
37
+ counter_evidence: list[str] | None,
38
+ competing_hypothesis: str | None = None,
39
+ critic_approach: CriticIndependence | None = None,
40
+ ) -> ChallengeResult:
41
+ """Build a ChallengeResult from critic output.
42
+
43
+ If counter_evidence is None, this is a protocol-only call (no critic ran yet).
44
+ hardening_score (SC5): reflects how adversarial the challenge was.
45
+ Protocol-only → 0.4; otherwise derived from critic_independence tier.
46
+ """
47
+ independence = critic_approach or classify_independence(critic_model)
48
+ self_eval = independence == "NONE"
49
+
50
+ if counter_evidence is None:
51
+ # Protocol-only: no critic ran; self_evaluation_flag must be True regardless
52
+ # of any critic_model label (MC6: protocol-only path always self-evaluates).
53
+ return ChallengeResult(
54
+ claim_text=claim_text,
55
+ critic_model=critic_model,
56
+ critic_independence=independence,
57
+ counter_evidence_found=False,
58
+ counter_evidence=[],
59
+ competing_hypothesis=competing_hypothesis,
60
+ verdict="SURVIVES",
61
+ self_evaluation_flag=True,
62
+ hardening_score=_HARDENING_PROTOCOL_ONLY,
63
+ )
64
+
65
+ counter_found = len(counter_evidence) > 0
66
+ verdict: ChallengeVerdict
67
+ if not counter_found:
68
+ verdict = "SURVIVES"
69
+ elif competing_hypothesis:
70
+ verdict = "DEFEATED"
71
+ else:
72
+ verdict = "NEEDS_REVISION"
73
+
74
+ hardening = _HARDENING_BY_INDEPENDENCE.get(str(independence), 0.7)
75
+
76
+ return ChallengeResult(
77
+ claim_text=claim_text,
78
+ critic_model=critic_model,
79
+ critic_independence=independence,
80
+ counter_evidence_found=counter_found,
81
+ counter_evidence=counter_evidence,
82
+ competing_hypothesis=competing_hypothesis,
83
+ verdict=verdict,
84
+ self_evaluation_flag=self_eval,
85
+ hardening_score=hardening,
86
+ )