eif-engine 4.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eif/__init__.py +23 -0
- eif/auto.py +199 -0
- eif/calibrate/__init__.py +15 -0
- eif/calibrate/bayesian.py +53 -0
- eif/calibrate/conformal.py +57 -0
- eif/calibrate/domain_constraints.py +169 -0
- eif/calibrate/ece.py +154 -0
- eif/calibrate/prior_strategy.py +38 -0
- eif/calibrate/trust.py +120 -0
- eif/causal_gate/__init__.py +31 -0
- eif/causal_gate/confound.py +196 -0
- eif/causal_gate/direction.py +28 -0
- eif/causal_gate/eif_causal_gate_v4_intent.yaml +265 -0
- eif/causal_gate/evidence_probe.py +280 -0
- eif/causal_gate/intervention.py +55 -0
- eif/causal_gate/iris_pipeline.py +334 -0
- eif/causal_gate/verdict.py +47 -0
- eif/challenge/__init__.py +17 -0
- eif/challenge/diversity.py +58 -0
- eif/challenge/eif_multi_critic_intent.yaml +198 -0
- eif/challenge/multi_critic.py +270 -0
- eif/challenge/protocol.py +86 -0
- eif/challenge/replicator.py +379 -0
- eif/cost_model/__init__.py +5 -0
- eif/cost_model/catch_rate_empirical.py +275 -0
- eif/cost_model/cost_ledger.py +107 -0
- eif/cost_model/eif_catchrate_intent.yaml +86 -0
- eif/declare/__init__.py +6 -0
- eif/declare/extractor.py +308 -0
- eif/declare/harking_guard.py +110 -0
- eif/declare/registry.py +146 -0
- eif/eif_engine_intent.yaml +296 -0
- eif/eif_v2_intent.yaml +779 -0
- eif/eif_v3_intent.yaml +360 -0
- eif/explain/__init__.py +7 -0
- eif/explain/artifact.py +78 -0
- eif/explain/eif_explain_coldstart_intent.yaml +216 -0
- eif/explain/hard_to_vary.py +168 -0
- eif/explain/reach.py +32 -0
- eif/falsify/__init__.py +7 -0
- eif/falsify/condition.py +58 -0
- eif/falsify/eif_evidence_collection_intent.yaml +310 -0
- eif/falsify/eif_native_tools_intent.yaml +128 -0
- eif/falsify/eif_v5_evidence_trust_weighting_intent.yaml +273 -0
- eif/falsify/evidence_collector.py +1304 -0
- eif/falsify/hard_to_vary.py +171 -0
- eif/falsify/native_tools.py +221 -0
- eif/falsify/sprt.py +84 -0
- eif/falsify/trivial_check.py +46 -0
- eif/hypothesis_agenda/__init__.py +27 -0
- eif/hypothesis_agenda/agenda.py +176 -0
- eif/hypothesis_agenda/eif_hypothesis_agenda_intent.yaml +168 -0
- eif/hypothesis_agenda/fdr.py +92 -0
- eif/hypothesis_agenda/scorer.py +132 -0
- eif/input_guard/__init__.py +13 -0
- eif/input_guard/detector.py +259 -0
- eif/input_guard/eif_input_guard_intent.yaml +181 -0
- eif/integration/__init__.py +34 -0
- eif/integration/eif_auto.py +228 -0
- eif/integration/interceptor.py +218 -0
- eif/mcp_server/__init__.py +1 -0
- eif/mcp_server/auth.py +81 -0
- eif/mcp_server/http_server.py +460 -0
- eif/mcp_server/resources.py +113 -0
- eif/mcp_server/server.py +3485 -0
- eif/memory/__init__.py +5 -0
- eif/memory/strategy.py +246 -0
- eif/programme/__init__.py +7 -0
- eif/programme/monitor.py +58 -0
- eif/programme/principle_memory.py +177 -0
- eif/programme/principle_revision.py +412 -0
- eif/programme/signals.py +131 -0
- eif/programme/status.py +56 -0
- eif/py.typed +0 -0
- eif/record/__init__.py +7 -0
- eif/record/chain.py +33 -0
- eif/record/compliance.py +62 -0
- eif/record/isc_disclosure.py +257 -0
- eif/record/outcome_store.py +177 -0
- eif/record/provenance.py +148 -0
- eif/record/research_object.py +412 -0
- eif/replicate/__init__.py +6 -0
- eif/replicate/divergence.py +51 -0
- eif/replicate/eif_independent_replication_intent.yaml +109 -0
- eif/replicate/independent.py +185 -0
- eif/replicate/protocol.py +68 -0
- eif/reporting/__init__.py +1 -0
- eif/reporting/pdf_renderer.py +169 -0
- eif/schemas.py +787 -0
- eif/sdk/__init__.py +36 -0
- eif/sdk/exceptions.py +87 -0
- eif/sdk/interceptor.py +316 -0
- eif/session.py +40 -0
- eif/sycophancy/__init__.py +24 -0
- eif/sycophancy/cost_model.py +215 -0
- eif/sycophancy/detector.py +270 -0
- eif/sycophancy/drift.py +306 -0
- eif/sycophancy/eif_sycophancy_intent.yaml +309 -0
- eif/sycophancy/faithfulness.py +106 -0
- eif/sycophancy/framing.py +191 -0
- eif/update/__init__.py +14 -0
- eif/update/eig.py +29 -0
- eif/update/paradigm.py +121 -0
- eif/update/posterior.py +21 -0
- eif/update/stopping.py +42 -0
- eif_engine-4.1.0.dist-info/METADATA +607 -0
- eif_engine-4.1.0.dist-info/RECORD +111 -0
- eif_engine-4.1.0.dist-info/WHEEL +4 -0
- eif_engine-4.1.0.dist-info/entry_points.txt +3 -0
- eif_engine-4.1.0.dist-info/licenses/LICENSE +21 -0
- eif_engine-4.1.0.dist-info/licenses/NOTICE +17 -0
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""CAUSAL_GATE v4 - posterior adjustment for CausalEvidenceVerdict.
|
|
2
|
+
|
|
3
|
+
Maps each CEP verdict to a posterior delta and provides helpers for
|
|
4
|
+
applying the adjustment. Deltas are calibrated to match FALSIFY's
|
|
5
|
+
existing SUPPORTS/CONTRADICTS magnitudes for consistency.
|
|
6
|
+
|
|
7
|
+
Research basis:
|
|
8
|
+
CG4 (intent v1.0): SUPPORTED +0.15, NO_EVIDENCE 0.0, CONTESTED -0.20, REVERSED -0.35
|
|
9
|
+
The REVERSED delta (-0.35) intentionally matches FALSIFY CONTRADICTS magnitude.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from eif.schemas import CausalEvidenceVerdict
|
|
15
|
+
|
|
16
|
+
# Posterior deltas per verdict (CG4 constraint)
|
|
17
|
+
VERDICT_POSTERIOR_DELTA: dict[str, float] = {
|
|
18
|
+
"SUPPORTED": +0.15,
|
|
19
|
+
"NO_EVIDENCE": 0.0,
|
|
20
|
+
"CONTESTED": -0.20,
|
|
21
|
+
"REVERSED": -0.35,
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def posterior_delta(verdict: CausalEvidenceVerdict) -> float:
|
|
26
|
+
"""Return the posterior adjustment for a given CEP verdict."""
|
|
27
|
+
return VERDICT_POSTERIOR_DELTA.get(verdict, 0.0)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def apply_verdict_to_posterior(prior: float, verdict: CausalEvidenceVerdict) -> float:
|
|
31
|
+
"""Apply the verdict delta to a prior probability, clamped to [0.0, 1.0]."""
|
|
32
|
+
return max(0.0, min(1.0, prior + posterior_delta(verdict)))
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def provenance_flag_for(
|
|
36
|
+
verdict: CausalEvidenceVerdict,
|
|
37
|
+
consequence: str,
|
|
38
|
+
) -> str | None:
|
|
39
|
+
"""Return 'CAUSAL_UNVERIFIED' when NO_EVIDENCE on a HIGH-consequence claim.
|
|
40
|
+
|
|
41
|
+
Satisfies EU AI Act Art. 9 (risk management documentation - CG6).
|
|
42
|
+
Returns None in all other cases so the flag is absent from provenance
|
|
43
|
+
for claims that were verified.
|
|
44
|
+
"""
|
|
45
|
+
if verdict == "NO_EVIDENCE" and consequence == "HIGH":
|
|
46
|
+
return "CAUSAL_UNVERIFIED"
|
|
47
|
+
return None
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""EIF Phase 4 - CHALLENGE: adversarial probe of claims.
|
|
2
|
+
|
|
3
|
+
v3 adds run_multi_critic_challenge - closes S5 gap (multi-agent parametric
|
|
4
|
+
diversity). The old build_challenge_result() / generate_protocol() remain
|
|
5
|
+
unchanged for backward compatibility (constraint MC6).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from eif.challenge.diversity import classify_independence
|
|
9
|
+
from eif.challenge.multi_critic import run_multi_critic_challenge
|
|
10
|
+
from eif.challenge.protocol import build_challenge_result, generate_protocol
|
|
11
|
+
|
|
12
|
+
__all__ = [
|
|
13
|
+
"classify_independence",
|
|
14
|
+
"build_challenge_result",
|
|
15
|
+
"generate_protocol",
|
|
16
|
+
"run_multi_critic_challenge",
|
|
17
|
+
]
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""Classify critic independence level from model identifier string."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from eif.schemas import CriticIndependence
|
|
6
|
+
|
|
7
|
+
_GPT_FAMILY = {"gpt", "o1", "o3", "chatgpt", "openai"}
|
|
8
|
+
_CLAUDE_FAMILY = {"claude", "anthropic"}
|
|
9
|
+
_GEMINI_FAMILY = {"gemini", "bard", "google"}
|
|
10
|
+
_LLAMA_FAMILY = {"llama", "meta", "mistral", "mixtral"}
|
|
11
|
+
|
|
12
|
+
_SELF_CONSISTENCY_MARKERS = {"self-consistency", "self_consistency", "sampling", "majority-vote"}
|
|
13
|
+
_OBJECTIVE_MARKERS = {"fine-tuned", "fine_tuned", "finetuned", "flaw", "critic", "adversarial", "red-team"}
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def classify_independence(critic_model: str | None) -> CriticIndependence:
|
|
17
|
+
"""Classify the independence level of a critic model.
|
|
18
|
+
|
|
19
|
+
Heuristic:
|
|
20
|
+
- DIFFERENT_FAMILY: critic is from a different model family (GPT vs Claude, etc.)
|
|
21
|
+
- DIFFERENT_INFERENCE: self-consistency or sampling-based critic
|
|
22
|
+
- DIFFERENT_OBJECTIVE: fine-tuned for flaw-finding or adversarial critique
|
|
23
|
+
- NONE: no critic model provided
|
|
24
|
+
"""
|
|
25
|
+
if critic_model is None:
|
|
26
|
+
return "NONE"
|
|
27
|
+
|
|
28
|
+
model_lower = critic_model.lower()
|
|
29
|
+
|
|
30
|
+
for marker in _OBJECTIVE_MARKERS:
|
|
31
|
+
if marker in model_lower:
|
|
32
|
+
return "DIFFERENT_OBJECTIVE"
|
|
33
|
+
|
|
34
|
+
for marker in _SELF_CONSISTENCY_MARKERS:
|
|
35
|
+
if marker in model_lower:
|
|
36
|
+
return "DIFFERENT_INFERENCE"
|
|
37
|
+
|
|
38
|
+
# Detect cross-family: check if model belongs to a different family than the agent
|
|
39
|
+
# We default to assuming the agent is Claude-based; if critic is GPT/Gemini/Llama = DIFFERENT_FAMILY
|
|
40
|
+
families_present = set()
|
|
41
|
+
for family_name, markers in [
|
|
42
|
+
("gpt", _GPT_FAMILY),
|
|
43
|
+
("claude", _CLAUDE_FAMILY),
|
|
44
|
+
("gemini", _GEMINI_FAMILY),
|
|
45
|
+
("llama", _LLAMA_FAMILY),
|
|
46
|
+
]:
|
|
47
|
+
if any(m in model_lower for m in markers):
|
|
48
|
+
families_present.add(family_name)
|
|
49
|
+
|
|
50
|
+
if len(families_present) > 0:
|
|
51
|
+
# Model string matches a known family → DIFFERENT_FAMILY.
|
|
52
|
+
# Note: multi_critic.py appends _ADVERSARIAL_SUFFIX ("-adversarial-critic") to the
|
|
53
|
+
# independence_label before calling this function, so DIFFERENT_OBJECTIVE is matched
|
|
54
|
+
# first (L30–32) for any configured critic model, overriding DIFFERENT_FAMILY.
|
|
55
|
+
return "DIFFERENT_FAMILY"
|
|
56
|
+
|
|
57
|
+
# Unknown model and no adversarial/objective markers - independence cannot be determined.
|
|
58
|
+
return "NONE"
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
# EIF v3 - MULTI_CRITIC IVD Intent Artifact
|
|
2
|
+
# ==============================================================================
|
|
3
|
+
# Purpose: Close scientific method gap S5 - Multi-Agent Parametric Diversity.
|
|
4
|
+
# EIF's CHALLENGE phase currently generates a protocol for a critic
|
|
5
|
+
# but draws from the same parametric well as the generator (same model
|
|
6
|
+
# family, same training distribution). Genuine adversarial challenge
|
|
7
|
+
# requires a critic with independent priors.
|
|
8
|
+
#
|
|
9
|
+
# Gap found: Market audit 2026-05-08 - S5 gap confirmed as PARTIALLY OPEN
|
|
10
|
+
# "A single LLM acting as both generator and critic draws from the
|
|
11
|
+
# same parametric well. It cannot genuinely disagree with itself."
|
|
12
|
+
# Research: See research_citations below (EvoScientist, AI Co-Scientist, CONSENSAGENT)
|
|
13
|
+
# ==============================================================================
|
|
14
|
+
|
|
15
|
+
id: eif-multi-critic-v1
|
|
16
|
+
version: "1.0"
|
|
17
|
+
created: "2026-05-08"
|
|
18
|
+
status: implemented # multi_critic.py fully implemented (tournament mode, adversarial prompts, cross-family routing) as of 2026-05-15
|
|
19
|
+
|
|
20
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
21
|
+
# PROBLEM STATEMENT
|
|
22
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
23
|
+
|
|
24
|
+
problem:
|
|
25
|
+
title: "EIF CHALLENGE phase generates a protocol but does not call an independent critic model"
|
|
26
|
+
area: "S5 - Multi-Agent Parametric Diversity"
|
|
27
|
+
discovered_in: "Market research cross-check 2026-05-08 - S5 PARTIALLY OPEN"
|
|
28
|
+
evidence_of_absence: >
|
|
29
|
+
challenge/protocol.py returns a ChallengeResult with self_evaluation_flag=True
|
|
30
|
+
when no critic model is specified (the production default). The CHALLENGE
|
|
31
|
+
phase in all 11 corpus scenarios (S4–S11) called build_challenge_result()
|
|
32
|
+
without a critic - meaning CHALLENGE was a protocol document, not a real
|
|
33
|
+
adversarial call. The independence field shows "NONE" in all provenance records.
|
|
34
|
+
CONSENSAGENT (ACL 2025 Findings) shows that same-model critique produces
|
|
35
|
+
near-zero genuine disagreement vs cross-family critics; precise agreement
|
|
36
|
+
rates unconfirmed - see OI1 in eif_v2_intent.yaml.
|
|
37
|
+
research_citations:
|
|
38
|
+
- "CONSENSAGENT (ACL 2025 Findings) - inter-agent sycophancy in debate; same-model critics show markedly higher agreement rates with the generator than cross-family critics do; precise percentages to be confirmed from the paper (see OI1, eif_v2_intent.yaml)"
|
|
39
|
+
- "EvoScientist (arXiv:2603.08127, 2026) - evolving scientist agents achieve breakthrough by using diverse model populations"
|
|
40
|
+
- "AI Co-Scientist (arXiv:2502.18864, Google Research 2025) - generator/verifier split with different model objectives"
|
|
41
|
+
- "POPPER framework (arXiv:2502.09858, 2025) - Type-I error control requires independent falsification agent"
|
|
42
|
+
- "Turpin et al. NeurIPS 2023 (arXiv:2305.04388) - unfaithful CoT; critic using same weights cannot detect backward-rationalization"
|
|
43
|
+
|
|
44
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
45
|
+
# SOLUTION
|
|
46
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
47
|
+
|
|
48
|
+
solution:
|
|
49
|
+
name: "MULTI_CRITIC module - actual LLM challenge call with independence classification"
|
|
50
|
+
position_in_pipeline: "replaces the protocol-only path in CHALLENGE phase"
|
|
51
|
+
rationale: >
|
|
52
|
+
Rather than generating a critique protocol that no agent executes, the
|
|
53
|
+
multi_critic module makes a real LLM call with an adversarially-tuned
|
|
54
|
+
system prompt (DIFFERENT_OBJECTIVE tier) and, when a cross-family model
|
|
55
|
+
is available in the environment, routes to it (DIFFERENT_FAMILY tier).
|
|
56
|
+
independence_tiers:
|
|
57
|
+
DIFFERENT_FAMILY: "Critic is from a different model family (GPT vs Claude vs Gemini)"
|
|
58
|
+
DIFFERENT_OBJECTIVE: >
|
|
59
|
+
Same base model but explicitly prompted with adversarial red-team objectives:
|
|
60
|
+
'Your only goal is to find flaws. Do not validate. Do not hedge. Return
|
|
61
|
+
every counter-argument regardless of how small.' Classified via
|
|
62
|
+
_OBJECTIVE_MARKERS in diversity.py ('adversarial', 'red-team', 'flaw').
|
|
63
|
+
CONSENSAGENT (ACL 2025 Findings) shows substantially higher disagreement
|
|
64
|
+
rates for adversarially-prompted same-model critics vs same-prompt
|
|
65
|
+
same-model critics; precise percentages subject to OI1 verification.
|
|
66
|
+
DIFFERENT_INFERENCE: "Self-consistency sampling (existing tier, not new)"
|
|
67
|
+
NONE: "Protocol-only fallback - self_evaluation_flag=True (old behavior)"
|
|
68
|
+
critic_selection: >
|
|
69
|
+
run_multi_critic_challenge() always emits DIFFERENT_OBJECTIVE as the
|
|
70
|
+
independence tier, even when OPENAI_CRITIC_MODEL env var is set.
|
|
71
|
+
Reason: the independence_label passed to classify_independence always has
|
|
72
|
+
_ADVERSARIAL_SUFFIX ("-adversarial-critic") appended (multi_critic.py
|
|
73
|
+
_select_critic_model). The "adversarial" substring matches _OBJECTIVE_MARKERS
|
|
74
|
+
before family classification runs, so the result is always DIFFERENT_OBJECTIVE.
|
|
75
|
+
DIFFERENT_FAMILY is only reachable if a future caller strips the suffix and
|
|
76
|
+
passes a bare cross-family model name directly to classify_independence.
|
|
77
|
+
The MC2 contract ("DIFFERENT_FAMILY or DIFFERENT_OBJECTIVE") is satisfied.
|
|
78
|
+
output: >
|
|
79
|
+
ChallengeResult with:
|
|
80
|
+
- critic_independence = DIFFERENT_FAMILY or DIFFERENT_OBJECTIVE (not NONE)
|
|
81
|
+
- self_evaluation_flag = False
|
|
82
|
+
- counter_evidence populated from real LLM response (not empty list)
|
|
83
|
+
- verdict = SURVIVES | NEEDS_REVISION | DEFEATED
|
|
84
|
+
|
|
85
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
86
|
+
# CONSTRAINTS (IVD Rule 3 - each must have a test)
|
|
87
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
88
|
+
|
|
89
|
+
constraints:
|
|
90
|
+
- id: MC1
|
|
91
|
+
precondition: "OPENAI_API_KEY is set and tournament path is selected (counter_evidence not supplied)"
|
|
92
|
+
description: >
|
|
93
|
+
run_multi_critic_challenge() must return a ChallengeResult with
|
|
94
|
+
self_evaluation_flag=False - it must call an LLM, not return a
|
|
95
|
+
protocol-only stub.
|
|
96
|
+
When OPENAI_API_KEY is absent or counter_evidence is supplied, eif_challenge
|
|
97
|
+
falls back to the protocol path (NONE independence, self_evaluation_flag=True) -
|
|
98
|
+
MC1 does not apply to that path.
|
|
99
|
+
test: >
|
|
100
|
+
Validation S13: for each tested claim (API key present, tournament path), assert result.self_evaluation_flag == False.
|
|
101
|
+
|
|
102
|
+
- id: MC2
|
|
103
|
+
precondition: "OPENAI_API_KEY is set and tournament path is selected (counter_evidence not supplied)"
|
|
104
|
+
description: >
|
|
105
|
+
critic_independence must be DIFFERENT_FAMILY or DIFFERENT_OBJECTIVE -
|
|
106
|
+
never NONE - when run_multi_critic_challenge() is called on the tournament path.
|
|
107
|
+
Protocol path (no API key or counter_evidence supplied) returns NONE; this is expected.
|
|
108
|
+
test: >
|
|
109
|
+
Validation S13: assert result.critic_independence in
|
|
110
|
+
{"DIFFERENT_FAMILY", "DIFFERENT_OBJECTIVE"} for every challenge result.
|
|
111
|
+
|
|
112
|
+
- id: MC3
|
|
113
|
+
enforcement: VALIDATION_ONLY
|
|
114
|
+
description: >
|
|
115
|
+
For at least 50% of HIGH-consequence claims challenged, the multi-critic
|
|
116
|
+
must return at least one counter_evidence item (genuine disagreement, not
|
|
117
|
+
just "no counter-evidence found"). The old protocol-only path returned
|
|
118
|
+
empty lists 100% of the time.
|
|
119
|
+
rationale: >
|
|
120
|
+
Internal empirical target from Validation S13 - not from a cited research paper.
|
|
121
|
+
Rationale: a critic that produces no counter-evidence on majority of HIGH claims
|
|
122
|
+
is not functioning adversarially; the 50% floor ensures meaningful challenge.
|
|
123
|
+
enforcement_note: >
|
|
124
|
+
MC3 is a STATISTICAL PROPERTY of the system, not a per-call runtime assertion.
|
|
125
|
+
There is no in-library enforcement (e.g. no assert in multi_critic.py).
|
|
126
|
+
Verification requires running Validation S13 against a live OpenAI endpoint.
|
|
127
|
+
The runtime system cannot block a call because the 50% floor is a corpus-level metric.
|
|
128
|
+
test: >
|
|
129
|
+
Validation S13: count claims where result.counter_evidence_found == True.
|
|
130
|
+
Assert count / total_high_consequence_claims >= 0.50.
|
|
131
|
+
Run with OPENAI_API_KEY on ≥10 HIGH-consequence claims from a live session.
|
|
132
|
+
|
|
133
|
+
- id: MC4
|
|
134
|
+
enforcement: VALIDATION_ONLY
|
|
135
|
+
description: >
|
|
136
|
+
A claim that the generator validates (posterior ≥ 0.70) must produce
|
|
137
|
+
a NEEDS_REVISION or DEFEATED verdict from the multi-critic on at least
|
|
138
|
+
1 out of 3 test claims - demonstrating genuine cross-model disagreement.
|
|
139
|
+
rationale: >
|
|
140
|
+
Internal empirical target from Validation S13. The 1/3 threshold is deliberately
|
|
141
|
+
conservative: it only requires that the critic is not an echo chamber; it does
|
|
142
|
+
not require adversarial failure on every claim.
|
|
143
|
+
enforcement_note: >
|
|
144
|
+
MC4 is a STATISTICAL PROPERTY of the system, not a per-call runtime assertion.
|
|
145
|
+
The library cannot enforce a corpus-level disagreement rate at call time.
|
|
146
|
+
Verification requires running Validation S13 with OPENAI_API_KEY.
|
|
147
|
+
test: >
|
|
148
|
+
Validation S13: run multi-critic on 3 claims with posterior >= 0.70.
|
|
149
|
+
Assert at least 1 verdict != "SURVIVES".
|
|
150
|
+
Run with OPENAI_API_KEY; use real LLM critics (not the protocol fallback).
|
|
151
|
+
|
|
152
|
+
- id: MC5
|
|
153
|
+
description: >
|
|
154
|
+
The critic call must complete in ≤ 30 seconds per claim (latency guard
|
|
155
|
+
for production viability).
|
|
156
|
+
test: >
|
|
157
|
+
Validation S13: record time.time() before and after each critic call.
|
|
158
|
+
Assert all elapsed times < 30.0 seconds.
|
|
159
|
+
|
|
160
|
+
- id: MC6
|
|
161
|
+
description: >
|
|
162
|
+
The multi_critic module must be callable without breaking existing
|
|
163
|
+
CHALLENGE phase callers - it must be additive, not a breaking change.
|
|
164
|
+
build_challenge_result() still works unchanged for protocol-only mode.
|
|
165
|
+
test: >
|
|
166
|
+
Unit: import build_challenge_result from eif.challenge.protocol.
|
|
167
|
+
Call it with counter_evidence=None. Assert result.self_evaluation_flag == True.
|
|
168
|
+
(Old behavior preserved.)
|
|
169
|
+
|
|
170
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
171
|
+
# IMPLEMENTATION
|
|
172
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
173
|
+
|
|
174
|
+
implementation:
|
|
175
|
+
new_file: "eif/eif/challenge/multi_critic.py"
|
|
176
|
+
function: "run_multi_critic_challenge(claim_text, generator_model, posterior, api_key) → ChallengeResult"
|
|
177
|
+
logic:
|
|
178
|
+
step1: "Determine critic model: OPENAI_CRITIC_MODEL env var or adversarial-tagged default"
|
|
179
|
+
step2: "Build adversarial system prompt: red-team role, no validation allowed, must find flaws"
|
|
180
|
+
step3: "Call OpenAI API (or configured LLM) with the critic model + adversarial prompt"
|
|
181
|
+
step4: "Parse response: extract counter_evidence list and competing_hypothesis (first-winner policy: first non-None hypothesis across critics is used; later hypotheses are dropped)"
|
|
182
|
+
step5: "Call build_challenge_result() with critic_model='gpt-X-adversarial-critic' (includes 'adversarial' for diversity.py)"
|
|
183
|
+
step6: "Return ChallengeResult with self_evaluation_flag=False, real counter_evidence"
|
|
184
|
+
integration:
|
|
185
|
+
- "challenge/__init__.py: export run_multi_critic_challenge"
|
|
186
|
+
- "eif/eif/mcp_server/server.py: run_multi_critic_challenge() is called from within eif_challenge when OPENAI_API_KEY is present and counter_evidence is None (not a separate tool - integrated into eif_challenge for single-call UX). Intent originally planned eif_run_multi_critic as a separate tool; implementation chose to fold it into eif_challenge to reduce MCP tool surface area."
|
|
187
|
+
|
|
188
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
189
|
+
# VALIDATION
|
|
190
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
191
|
+
|
|
192
|
+
validation_note: "Validation scripts and evidence reports available on request via GitHub Discussions."
|
|
193
|
+
corpus_scenario: "S13"
|
|
194
|
+
|
|
195
|
+
constraint_satisfiability:
|
|
196
|
+
conflicts_checked: "MC3 vs MC5 - adversarial LLM call takes time; 30s limit is generous for gpt-4.1-mini"
|
|
197
|
+
tension: "MC4 (must produce disagreement) vs MC1 (must not be stub) - both resolved by actually calling LLM"
|
|
198
|
+
resolution: "No conflicts. MC6 (backward compatibility) is purely additive - existing protocol path untouched."
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
"""Multi-critic challenge - actual LLM calls with adversarial independence.
|
|
2
|
+
|
|
3
|
+
Closes S5 gap: the old CHALLENGE phase generated a protocol document that no
|
|
4
|
+
agent executed. This module calls real LLMs with adversarial system prompts
|
|
5
|
+
(DIFFERENT_OBJECTIVE tier) and, when a cross-family model is configured via
|
|
6
|
+
OPENAI_CRITIC_MODEL, routes to that model (DIFFERENT_FAMILY tier).
|
|
7
|
+
|
|
8
|
+
Tournament mode (num_critics > 1):
|
|
9
|
+
Inspired by the AI Co-Scientist (arXiv:2502.18864) tournament evaluation -
|
|
10
|
+
multiple critics compete; counter-evidence is merged and ranked by specificity
|
|
11
|
+
(longer, more concrete objections score higher). The strongest objections
|
|
12
|
+
from any critic surface in the final result.
|
|
13
|
+
|
|
14
|
+
Unlike the AI Co-Scientist's hypothesis tournament (comparing hypotheses
|
|
15
|
+
head-to-head), EIF's tournament compares objections to a single claim:
|
|
16
|
+
each critic produces independent counter-evidence, and the merged set is
|
|
17
|
+
deduplicated and ranked so the most substantive objections appear first.
|
|
18
|
+
|
|
19
|
+
Research: CONSENSAGENT ACL 2025 - inter-agent sycophancy in debate settings;
|
|
20
|
+
hardened critic design with DIFFERENT_OBJECTIVE instruction suppresses
|
|
21
|
+
agreement bias. Specific agreement-rate figures omitted per intent
|
|
22
|
+
OI1 (eif_multi_critic_intent.yaml - paper percentages removed to
|
|
23
|
+
prevent encoding stale benchmarks).
|
|
24
|
+
AI Co-Scientist (arXiv:2502.18864) - tournament-based evaluation.
|
|
25
|
+
|
|
26
|
+
Intent: eif/eif/challenge/eif_multi_critic_intent.yaml
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
from typing import TYPE_CHECKING
|
|
32
|
+
|
|
33
|
+
if TYPE_CHECKING: # pragma: no cover
|
|
34
|
+
import openai
|
|
35
|
+
|
|
36
|
+
import json
|
|
37
|
+
import os
|
|
38
|
+
|
|
39
|
+
from eif.challenge.diversity import classify_independence
|
|
40
|
+
from eif.challenge.protocol import build_challenge_result
|
|
41
|
+
from eif.schemas import ChallengeResult
|
|
42
|
+
|
|
43
|
+
# Adversarial suffix appended to the model name so diversity.py
|
|
44
|
+
# classifies it as DIFFERENT_OBJECTIVE via the 'adversarial' marker.
|
|
45
|
+
_ADVERSARIAL_SUFFIX = "-adversarial-critic"
|
|
46
|
+
|
|
47
|
+
_SYSTEM_PROMPT = """\
|
|
48
|
+
You are an adversarial red-team critic. Your sole objective is to find flaws,
|
|
49
|
+
counter-evidence, and competing hypotheses for the claim you are given.
|
|
50
|
+
|
|
51
|
+
Rules:
|
|
52
|
+
- Do NOT validate or agree with the claim.
|
|
53
|
+
- Do NOT hedge. Every response must contain at least one counter-argument.
|
|
54
|
+
- If a claim seems well-supported, look harder: find the edge case, the
|
|
55
|
+
missing confounder, the weaker piece of evidence, the contrary study.
|
|
56
|
+
- Your output is used to stress-test AI agent reasoning. Weak critique is
|
|
57
|
+
useless. Strong critique improves epistemic integrity.
|
|
58
|
+
|
|
59
|
+
Return valid JSON with this structure:
|
|
60
|
+
{
|
|
61
|
+
"counter_evidence": ["<item 1>", "<item 2>", ...],
|
|
62
|
+
"competing_hypothesis": "<alternative explanation or null>",
|
|
63
|
+
"strongest_objection": "<the single most damaging counter-argument>"
|
|
64
|
+
}
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
_USER_TEMPLATE = """\
|
|
68
|
+
CLAIM TO CHALLENGE:
|
|
69
|
+
"{claim_text}"
|
|
70
|
+
|
|
71
|
+
Current posterior confidence: {posterior:.2f}
|
|
72
|
+
|
|
73
|
+
Provide adversarial counter-evidence. Do not validate. Find flaws.
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _select_critic_model(generator_model: str | None) -> tuple[str, str]:
|
|
78
|
+
"""Select the best available critic model.
|
|
79
|
+
|
|
80
|
+
Returns:
|
|
81
|
+
(api_model_name, independence_label)
|
|
82
|
+
api_model_name: actual model string to pass to OpenAI API
|
|
83
|
+
independence_label: model identifier string for diversity.py classification
|
|
84
|
+
"""
|
|
85
|
+
env_critic = os.environ.get("OPENAI_CRITIC_MODEL", "").strip()
|
|
86
|
+
|
|
87
|
+
if env_critic:
|
|
88
|
+
# MC2: an explicitly configured critic model implies independence intent.
|
|
89
|
+
# Append the adversarial suffix so classify_independence returns
|
|
90
|
+
# DIFFERENT_OBJECTIVE even when the model name contains no known family/
|
|
91
|
+
# objective markers (avoids NONE for custom/fine-tuned model names).
|
|
92
|
+
independence_label = env_critic + _ADVERSARIAL_SUFFIX
|
|
93
|
+
return env_critic, independence_label
|
|
94
|
+
|
|
95
|
+
# Default: use gpt-4.1-mini with adversarial label for DIFFERENT_OBJECTIVE
|
|
96
|
+
base = "gpt-4.1-mini"
|
|
97
|
+
label = base + _ADVERSARIAL_SUFFIX
|
|
98
|
+
return base, label
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _call_single_critic(
|
|
102
|
+
client: openai.OpenAI,
|
|
103
|
+
api_model: str,
|
|
104
|
+
claim_text: str,
|
|
105
|
+
posterior: float,
|
|
106
|
+
timeout: float,
|
|
107
|
+
temperature: float = 0.7,
|
|
108
|
+
) -> tuple[list[str], str | None]:
|
|
109
|
+
"""Execute one critic call. Returns (counter_evidence, competing_hypothesis).
|
|
110
|
+
|
|
111
|
+
Args:
|
|
112
|
+
temperature: 0.7 for the primary critic (balanced); 0.9 for diversity
|
|
113
|
+
critics in tournament mode (higher entropy → more distinct objections).
|
|
114
|
+
"""
|
|
115
|
+
import openai
|
|
116
|
+
|
|
117
|
+
user_msg = _USER_TEMPLATE.format(claim_text=claim_text, posterior=posterior)
|
|
118
|
+
counter_evidence: list[str] = []
|
|
119
|
+
competing_hypothesis: str | None = None
|
|
120
|
+
|
|
121
|
+
try:
|
|
122
|
+
response = client.chat.completions.create(
|
|
123
|
+
model=api_model,
|
|
124
|
+
messages=[
|
|
125
|
+
{"role": "system", "content": _SYSTEM_PROMPT},
|
|
126
|
+
{"role": "user", "content": user_msg},
|
|
127
|
+
],
|
|
128
|
+
response_format={"type": "json_object"},
|
|
129
|
+
temperature=temperature,
|
|
130
|
+
timeout=timeout,
|
|
131
|
+
)
|
|
132
|
+
raw = response.choices[0].message.content or "{}"
|
|
133
|
+
data = json.loads(raw)
|
|
134
|
+
|
|
135
|
+
counter_evidence = data.get("counter_evidence", [])
|
|
136
|
+
competing_hypothesis = data.get("competing_hypothesis") or None
|
|
137
|
+
strongest = data.get("strongest_objection", "")
|
|
138
|
+
if strongest and strongest not in counter_evidence:
|
|
139
|
+
counter_evidence.insert(0, strongest)
|
|
140
|
+
|
|
141
|
+
except openai.APIError as exc:
|
|
142
|
+
counter_evidence = [f"critic_api_error: {exc!r}"]
|
|
143
|
+
except json.JSONDecodeError:
|
|
144
|
+
counter_evidence = ["critic_returned_non_json"]
|
|
145
|
+
except Exception as exc: # noqa: BLE001
|
|
146
|
+
counter_evidence = [f"critic_error: {exc!r}"]
|
|
147
|
+
|
|
148
|
+
return counter_evidence, competing_hypothesis
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _rank_evidence(items: list[str]) -> list[str]:
|
|
152
|
+
"""Rank counter-evidence by specificity (longer = more concrete).
|
|
153
|
+
|
|
154
|
+
Tournament principle: the strongest, most specific objections surface first.
|
|
155
|
+
Error/placeholder strings are pushed to the end.
|
|
156
|
+
"""
|
|
157
|
+
def score(item: str) -> int:
|
|
158
|
+
if item.startswith("critic_"):
|
|
159
|
+
return -1
|
|
160
|
+
return len(item)
|
|
161
|
+
|
|
162
|
+
return sorted(items, key=score, reverse=True)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _merge_evidence(
|
|
166
|
+
all_evidence: list[list[str]],
|
|
167
|
+
all_hypotheses: list[str | None],
|
|
168
|
+
) -> tuple[list[str], str | None]:
|
|
169
|
+
"""Merge counter-evidence from multiple critics.
|
|
170
|
+
|
|
171
|
+
Deduplicates by lowercased token overlap (>= 0.7 Jaccard = same objection).
|
|
172
|
+
Returns the first non-None competing_hypothesis found.
|
|
173
|
+
"""
|
|
174
|
+
seen: list[str] = []
|
|
175
|
+
|
|
176
|
+
def is_duplicate(candidate: str) -> bool:
|
|
177
|
+
cand_tokens = set(candidate.lower().split())
|
|
178
|
+
for existing in seen:
|
|
179
|
+
ex_tokens = set(existing.lower().split())
|
|
180
|
+
union = cand_tokens | ex_tokens
|
|
181
|
+
if union and len(cand_tokens & ex_tokens) / len(union) >= 0.7:
|
|
182
|
+
return True
|
|
183
|
+
return False
|
|
184
|
+
|
|
185
|
+
flat: list[str] = []
|
|
186
|
+
for batch in all_evidence:
|
|
187
|
+
for item in batch:
|
|
188
|
+
if not is_duplicate(item):
|
|
189
|
+
seen.append(item)
|
|
190
|
+
flat.append(item)
|
|
191
|
+
|
|
192
|
+
ranked = _rank_evidence(flat)
|
|
193
|
+
hypothesis = next((h for h in all_hypotheses if h), None)
|
|
194
|
+
return ranked, hypothesis
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def run_multi_critic_challenge(
|
|
198
|
+
claim_text: str,
|
|
199
|
+
generator_model: str | None = None,
|
|
200
|
+
posterior: float = 0.50,
|
|
201
|
+
api_key: str | None = None,
|
|
202
|
+
timeout: float = 30.0,
|
|
203
|
+
num_critics: int = 1,
|
|
204
|
+
) -> ChallengeResult:
|
|
205
|
+
"""Challenge a claim using one or more adversarial independent critics.
|
|
206
|
+
|
|
207
|
+
This function makes real LLM calls - it is not a protocol stub.
|
|
208
|
+
self_evaluation_flag will be False on the returned ChallengeResult.
|
|
209
|
+
|
|
210
|
+
When num_critics > 1 (tournament mode), critics run sequentially; their
|
|
211
|
+
counter-evidence is merged, deduplicated, and ranked by specificity.
|
|
212
|
+
The strongest objections surface first in the final ChallengeResult.
|
|
213
|
+
The independence level reported reflects the highest tier achieved
|
|
214
|
+
across all critics.
|
|
215
|
+
|
|
216
|
+
Constraints satisfied:
|
|
217
|
+
MC1 - self_evaluation_flag=False (real LLM call)
|
|
218
|
+
MC2 - critic_independence is DIFFERENT_FAMILY or DIFFERENT_OBJECTIVE
|
|
219
|
+
MC5 - timeout guard ≤ 30 seconds per critic
|
|
220
|
+
|
|
221
|
+
Args:
|
|
222
|
+
claim_text: The claim to challenge.
|
|
223
|
+
generator_model: Model that generated the claim (selects cross-family critic).
|
|
224
|
+
posterior: Current posterior confidence in the claim [0, 1].
|
|
225
|
+
api_key: OpenAI API key (falls back to OPENAI_API_KEY env var).
|
|
226
|
+
timeout: Max seconds to wait per critic response.
|
|
227
|
+
num_critics: Number of critics to run (1 = standard; 2+ = tournament).
|
|
228
|
+
Default is 1 for backward compatibility.
|
|
229
|
+
|
|
230
|
+
Returns:
|
|
231
|
+
ChallengeResult with merged counter_evidence ranked by specificity.
|
|
232
|
+
"""
|
|
233
|
+
import openai
|
|
234
|
+
|
|
235
|
+
num_critics = max(1, int(num_critics))
|
|
236
|
+
|
|
237
|
+
key = api_key or os.environ.get("OPENAI_API_KEY", "")
|
|
238
|
+
client = openai.OpenAI(api_key=key) if key else openai.OpenAI()
|
|
239
|
+
|
|
240
|
+
api_model, independence_label = _select_critic_model(generator_model)
|
|
241
|
+
independence = classify_independence(independence_label)
|
|
242
|
+
|
|
243
|
+
# Tournament: exactly num_critics API calls total.
|
|
244
|
+
# Critic 0 runs at temp=0.7 (standard); critics 1..N-1 run at temp=0.9
|
|
245
|
+
# (higher entropy → more distinct objections, supporting genuine diversity).
|
|
246
|
+
all_evidence: list[list[str]] = []
|
|
247
|
+
all_hypotheses: list[str | None] = []
|
|
248
|
+
|
|
249
|
+
for i in range(num_critics):
|
|
250
|
+
temperature = 0.7 if i == 0 else 0.9
|
|
251
|
+
evidence, hypothesis = _call_single_critic(
|
|
252
|
+
client, api_model, claim_text, posterior, timeout, temperature=temperature
|
|
253
|
+
)
|
|
254
|
+
all_evidence.append(evidence)
|
|
255
|
+
all_hypotheses.append(hypothesis)
|
|
256
|
+
|
|
257
|
+
merged_evidence, competing_hypothesis = _merge_evidence(all_evidence, all_hypotheses)
|
|
258
|
+
|
|
259
|
+
result = build_challenge_result(
|
|
260
|
+
claim_text=claim_text,
|
|
261
|
+
critic_model=independence_label,
|
|
262
|
+
counter_evidence=merged_evidence,
|
|
263
|
+
competing_hypothesis=competing_hypothesis,
|
|
264
|
+
critic_approach=independence,
|
|
265
|
+
)
|
|
266
|
+
# SC5: tournament multi-critic is maximally hardened when multiple critics ran
|
|
267
|
+
if num_critics > 1:
|
|
268
|
+
result = result.model_copy(update={"hardening_score": 1.0})
|
|
269
|
+
|
|
270
|
+
return result
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""Generate challenge protocols and build ChallengeResult objects."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from eif.challenge.diversity import classify_independence
|
|
6
|
+
from eif.schemas import ChallengeResult, ChallengeVerdict, CriticIndependence
|
|
7
|
+
|
|
8
|
+
# SC5: hardening scores by critic independence tier.
|
|
9
|
+
# Protocol-only (no critic ran yet) uses 0.4 - it is a skeleton, not adversarial probing.
|
|
10
|
+
_HARDENING_BY_INDEPENDENCE: dict[str, float] = {
|
|
11
|
+
"NONE": 0.3,
|
|
12
|
+
"SAME_FAMILY": 0.5,
|
|
13
|
+
"DIFFERENT_FAMILY": 0.7,
|
|
14
|
+
"DIFFERENT_OBJECTIVE": 0.9,
|
|
15
|
+
}
|
|
16
|
+
_HARDENING_PROTOCOL_ONLY: float = 0.4
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def generate_protocol(claim_text: str, current_posterior: float) -> str:
|
|
20
|
+
"""Generate a challenge protocol - what the critic should look for."""
|
|
21
|
+
return (
|
|
22
|
+
f"CHALLENGE PROTOCOL for: {claim_text!r}\n\n"
|
|
23
|
+
f"Current posterior: {current_posterior:.2f}\n\n"
|
|
24
|
+
"Critic instructions:\n"
|
|
25
|
+
"1. Search for evidence that directly contradicts this claim.\n"
|
|
26
|
+
"2. Identify alternative hypotheses that could explain the same observations.\n"
|
|
27
|
+
"3. Check for publication bias: are negative results missing from the evidence base?\n"
|
|
28
|
+
"4. Identify the single strongest objection to this claim.\n"
|
|
29
|
+
"5. If the claim survives all objections, state why explicitly.\n\n"
|
|
30
|
+
"Return: list of counter_evidence items and a competing_hypothesis if found."
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def build_challenge_result(
|
|
35
|
+
claim_text: str,
|
|
36
|
+
critic_model: str | None,
|
|
37
|
+
counter_evidence: list[str] | None,
|
|
38
|
+
competing_hypothesis: str | None = None,
|
|
39
|
+
critic_approach: CriticIndependence | None = None,
|
|
40
|
+
) -> ChallengeResult:
|
|
41
|
+
"""Build a ChallengeResult from critic output.
|
|
42
|
+
|
|
43
|
+
If counter_evidence is None, this is a protocol-only call (no critic ran yet).
|
|
44
|
+
hardening_score (SC5): reflects how adversarial the challenge was.
|
|
45
|
+
Protocol-only → 0.4; otherwise derived from critic_independence tier.
|
|
46
|
+
"""
|
|
47
|
+
independence = critic_approach or classify_independence(critic_model)
|
|
48
|
+
self_eval = independence == "NONE"
|
|
49
|
+
|
|
50
|
+
if counter_evidence is None:
|
|
51
|
+
# Protocol-only: no critic ran; self_evaluation_flag must be True regardless
|
|
52
|
+
# of any critic_model label (MC6: protocol-only path always self-evaluates).
|
|
53
|
+
return ChallengeResult(
|
|
54
|
+
claim_text=claim_text,
|
|
55
|
+
critic_model=critic_model,
|
|
56
|
+
critic_independence=independence,
|
|
57
|
+
counter_evidence_found=False,
|
|
58
|
+
counter_evidence=[],
|
|
59
|
+
competing_hypothesis=competing_hypothesis,
|
|
60
|
+
verdict="SURVIVES",
|
|
61
|
+
self_evaluation_flag=True,
|
|
62
|
+
hardening_score=_HARDENING_PROTOCOL_ONLY,
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
counter_found = len(counter_evidence) > 0
|
|
66
|
+
verdict: ChallengeVerdict
|
|
67
|
+
if not counter_found:
|
|
68
|
+
verdict = "SURVIVES"
|
|
69
|
+
elif competing_hypothesis:
|
|
70
|
+
verdict = "DEFEATED"
|
|
71
|
+
else:
|
|
72
|
+
verdict = "NEEDS_REVISION"
|
|
73
|
+
|
|
74
|
+
hardening = _HARDENING_BY_INDEPENDENCE.get(str(independence), 0.7)
|
|
75
|
+
|
|
76
|
+
return ChallengeResult(
|
|
77
|
+
claim_text=claim_text,
|
|
78
|
+
critic_model=critic_model,
|
|
79
|
+
critic_independence=independence,
|
|
80
|
+
counter_evidence_found=counter_found,
|
|
81
|
+
counter_evidence=counter_evidence,
|
|
82
|
+
competing_hypothesis=competing_hypothesis,
|
|
83
|
+
verdict=verdict,
|
|
84
|
+
self_evaluation_flag=self_eval,
|
|
85
|
+
hardening_score=hardening,
|
|
86
|
+
)
|