eif-engine 4.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eif/__init__.py +23 -0
- eif/auto.py +199 -0
- eif/calibrate/__init__.py +15 -0
- eif/calibrate/bayesian.py +53 -0
- eif/calibrate/conformal.py +57 -0
- eif/calibrate/domain_constraints.py +169 -0
- eif/calibrate/ece.py +154 -0
- eif/calibrate/prior_strategy.py +38 -0
- eif/calibrate/trust.py +120 -0
- eif/causal_gate/__init__.py +31 -0
- eif/causal_gate/confound.py +196 -0
- eif/causal_gate/direction.py +28 -0
- eif/causal_gate/eif_causal_gate_v4_intent.yaml +265 -0
- eif/causal_gate/evidence_probe.py +280 -0
- eif/causal_gate/intervention.py +55 -0
- eif/causal_gate/iris_pipeline.py +334 -0
- eif/causal_gate/verdict.py +47 -0
- eif/challenge/__init__.py +17 -0
- eif/challenge/diversity.py +58 -0
- eif/challenge/eif_multi_critic_intent.yaml +198 -0
- eif/challenge/multi_critic.py +270 -0
- eif/challenge/protocol.py +86 -0
- eif/challenge/replicator.py +379 -0
- eif/cost_model/__init__.py +5 -0
- eif/cost_model/catch_rate_empirical.py +275 -0
- eif/cost_model/cost_ledger.py +107 -0
- eif/cost_model/eif_catchrate_intent.yaml +86 -0
- eif/declare/__init__.py +6 -0
- eif/declare/extractor.py +308 -0
- eif/declare/harking_guard.py +110 -0
- eif/declare/registry.py +146 -0
- eif/eif_engine_intent.yaml +296 -0
- eif/eif_v2_intent.yaml +779 -0
- eif/eif_v3_intent.yaml +360 -0
- eif/explain/__init__.py +7 -0
- eif/explain/artifact.py +78 -0
- eif/explain/eif_explain_coldstart_intent.yaml +216 -0
- eif/explain/hard_to_vary.py +168 -0
- eif/explain/reach.py +32 -0
- eif/falsify/__init__.py +7 -0
- eif/falsify/condition.py +58 -0
- eif/falsify/eif_evidence_collection_intent.yaml +310 -0
- eif/falsify/eif_native_tools_intent.yaml +128 -0
- eif/falsify/eif_v5_evidence_trust_weighting_intent.yaml +273 -0
- eif/falsify/evidence_collector.py +1304 -0
- eif/falsify/hard_to_vary.py +171 -0
- eif/falsify/native_tools.py +221 -0
- eif/falsify/sprt.py +84 -0
- eif/falsify/trivial_check.py +46 -0
- eif/hypothesis_agenda/__init__.py +27 -0
- eif/hypothesis_agenda/agenda.py +176 -0
- eif/hypothesis_agenda/eif_hypothesis_agenda_intent.yaml +168 -0
- eif/hypothesis_agenda/fdr.py +92 -0
- eif/hypothesis_agenda/scorer.py +132 -0
- eif/input_guard/__init__.py +13 -0
- eif/input_guard/detector.py +259 -0
- eif/input_guard/eif_input_guard_intent.yaml +181 -0
- eif/integration/__init__.py +34 -0
- eif/integration/eif_auto.py +228 -0
- eif/integration/interceptor.py +218 -0
- eif/mcp_server/__init__.py +1 -0
- eif/mcp_server/auth.py +81 -0
- eif/mcp_server/http_server.py +460 -0
- eif/mcp_server/resources.py +113 -0
- eif/mcp_server/server.py +3485 -0
- eif/memory/__init__.py +5 -0
- eif/memory/strategy.py +246 -0
- eif/programme/__init__.py +7 -0
- eif/programme/monitor.py +58 -0
- eif/programme/principle_memory.py +177 -0
- eif/programme/principle_revision.py +412 -0
- eif/programme/signals.py +131 -0
- eif/programme/status.py +56 -0
- eif/py.typed +0 -0
- eif/record/__init__.py +7 -0
- eif/record/chain.py +33 -0
- eif/record/compliance.py +62 -0
- eif/record/isc_disclosure.py +257 -0
- eif/record/outcome_store.py +177 -0
- eif/record/provenance.py +148 -0
- eif/record/research_object.py +412 -0
- eif/replicate/__init__.py +6 -0
- eif/replicate/divergence.py +51 -0
- eif/replicate/eif_independent_replication_intent.yaml +109 -0
- eif/replicate/independent.py +185 -0
- eif/replicate/protocol.py +68 -0
- eif/reporting/__init__.py +1 -0
- eif/reporting/pdf_renderer.py +169 -0
- eif/schemas.py +787 -0
- eif/sdk/__init__.py +36 -0
- eif/sdk/exceptions.py +87 -0
- eif/sdk/interceptor.py +316 -0
- eif/session.py +40 -0
- eif/sycophancy/__init__.py +24 -0
- eif/sycophancy/cost_model.py +215 -0
- eif/sycophancy/detector.py +270 -0
- eif/sycophancy/drift.py +306 -0
- eif/sycophancy/eif_sycophancy_intent.yaml +309 -0
- eif/sycophancy/faithfulness.py +106 -0
- eif/sycophancy/framing.py +191 -0
- eif/update/__init__.py +14 -0
- eif/update/eig.py +29 -0
- eif/update/paradigm.py +121 -0
- eif/update/posterior.py +21 -0
- eif/update/stopping.py +42 -0
- eif_engine-4.1.0.dist-info/METADATA +607 -0
- eif_engine-4.1.0.dist-info/RECORD +111 -0
- eif_engine-4.1.0.dist-info/WHEEL +4 -0
- eif_engine-4.1.0.dist-info/entry_points.txt +3 -0
- eif_engine-4.1.0.dist-info/licenses/LICENSE +21 -0
- eif_engine-4.1.0.dist-info/licenses/NOTICE +17 -0
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Prior strategy selection for Bayesian calibration."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from eif.schemas import PRIOR_EMPIRICAL_MIN, PriorStrategy
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def select_prior_strategy(
|
|
9
|
+
prior_provided: bool,
|
|
10
|
+
calibration_history_size: int,
|
|
11
|
+
) -> tuple[PriorStrategy, float]:
|
|
12
|
+
"""Select prior strategy and return (strategy, prior_value).
|
|
13
|
+
|
|
14
|
+
Selection rules:
|
|
15
|
+
- "domain" if user provides a prior (prior_provided=True)
|
|
16
|
+
- "empirical_bayes" if history >= PRIOR_EMPIRICAL_MIN (= 10)
|
|
17
|
+
- "max_entropy" otherwise (uniform prior = 0.5)
|
|
18
|
+
|
|
19
|
+
Returns a (strategy, prior_value) where prior_value is 0.5 for max_entropy
|
|
20
|
+
and empirical_bayes (caller must supply the actual value for domain).
|
|
21
|
+
"""
|
|
22
|
+
if prior_provided:
|
|
23
|
+
return "domain", 0.5 # caller provides actual value
|
|
24
|
+
|
|
25
|
+
if calibration_history_size >= PRIOR_EMPIRICAL_MIN:
|
|
26
|
+
return "empirical_bayes", 0.5
|
|
27
|
+
|
|
28
|
+
return "max_entropy", 0.5
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def empirical_bayes_prior(history_posteriors: list[float]) -> float:
|
|
32
|
+
"""Estimate empirical Bayes prior from calibration history.
|
|
33
|
+
|
|
34
|
+
Simple hyperparameter estimation: mean of past posteriors.
|
|
35
|
+
"""
|
|
36
|
+
if not history_posteriors:
|
|
37
|
+
return 0.5
|
|
38
|
+
return sum(history_posteriors) / len(history_posteriors)
|
eif/calibrate/trust.py
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
"""Evidence trust weighting - tier- and confidence-aware Bayesian likelihood.
|
|
2
|
+
|
|
3
|
+
The eif_verify orchestrator previously used a flat likelihood of 0.8 (SUPPORTS)
|
|
4
|
+
/ 0.2 (CONTRADICTS) for every probed claim, discarding the evidence's tier and
|
|
5
|
+
confidence. A single P3 web-search keyword match then produced the SAME posterior
|
|
6
|
+
as a P1 code-execution result. This module replaces that flat update with a
|
|
7
|
+
deterministic, zero-LLM trust function that weights the Bayesian likelihood by:
|
|
8
|
+
|
|
9
|
+
(a) the evidence tier - direct observation (P0/P1/P2) > web search (P3) >
|
|
10
|
+
parametric (P4); and
|
|
11
|
+
(b) the evidence confidence reported by the collector; and
|
|
12
|
+
(c) whether a web result was corroborated by >= 2 independent sources.
|
|
13
|
+
|
|
14
|
+
Design (with prior 0.5, compute_posterior(0.5, supports, L) ≈ L):
|
|
15
|
+
|
|
16
|
+
trust = tier_weight * clamp(confidence, 0, 1)
|
|
17
|
+
likelihood = 0.5 + P_SPAN * trust (SUPPORTS)
|
|
18
|
+
= 0.5 - P_SPAN * trust (CONTRADICTS)
|
|
19
|
+
|
|
20
|
+
with P_SPAN = 0.35 and the tier weights below. This yields:
|
|
21
|
+
|
|
22
|
+
P1 conf 0.9 → trust 0.81 → L ≈ 0.7835 → posterior ≈ 0.78 (>= ACT)
|
|
23
|
+
P3 single conf 0.65 → trust 0.3575 → L ≈ 0.6251 → posterior ≈ 0.63 (< ACT)
|
|
24
|
+
P3 corroborated conf 0.75→ trust 0.6375 → L ≈ 0.7231 → posterior ≈ 0.72 (>= ACT)
|
|
25
|
+
|
|
26
|
+
so a single uncorroborated web match cannot reach the ACT threshold (0.70) while
|
|
27
|
+
direct-observation tiers and corroborated web evidence can. See
|
|
28
|
+
eif/falsify/eif_v5_evidence_trust_weighting_intent.yaml (TW1, TW2, TW7).
|
|
29
|
+
|
|
30
|
+
Research basis:
|
|
31
|
+
PoisonedRAG (arXiv:2402.07867, USENIX Security 2025): injecting as few as 5 -
|
|
32
|
+
and in single-doc variants, 1 - crafted documents into a retrieval corpus
|
|
33
|
+
yields ~90% attack success. A verifier that treats one retrieved match as
|
|
34
|
+
decisive is directly exploitable; single-source web evidence must be
|
|
35
|
+
discounted and gated on corroboration.
|
|
36
|
+
The Power of Noise (arXiv:2401.14887): distracting / low-relevance retrieved
|
|
37
|
+
documents degrade RAG accuracy even alongside a correct document. Single
|
|
38
|
+
keyword-match web evidence is the noisy regime and must not carry the weight
|
|
39
|
+
of direct observation.
|
|
40
|
+
|
|
41
|
+
Invariant: this module makes ZERO LLM calls and ZERO network calls - it is pure
|
|
42
|
+
arithmetic, preserving tech_spec.md C1 (zero-LLM) and C4 (zero-network) for the
|
|
43
|
+
engine.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
from __future__ import annotations
|
|
47
|
+
|
|
48
|
+
# Distance the likelihood may move from the 0.5 prior at full trust.
|
|
49
|
+
# With prior 0.5 the posterior ≈ likelihood, so P_SPAN bounds the per-probe
|
|
50
|
+
# posterior swing. 0.35 places a max-trust SUPPORTS at L=0.85 (matching the prior
|
|
51
|
+
# flat-0.8 ceiling for the strongest tier) while leaving headroom above ACT.
|
|
52
|
+
P_SPAN: float = 0.35
|
|
53
|
+
|
|
54
|
+
# Fallibilism bound - consistent with eif/calibrate/bayesian.py _FALLIBILISM_EPS.
|
|
55
|
+
_EPS: float = 1e-6
|
|
56
|
+
|
|
57
|
+
# Tier trust weights. Higher = the tier moves the posterior further from the
|
|
58
|
+
# 0.5 prior. P3 web search has two weights: uncorroborated (single source) vs
|
|
59
|
+
# corroborated (>= 2 independent registrable domains).
|
|
60
|
+
TIER_TRUST_WEIGHTS: dict[str, float] = {
|
|
61
|
+
"P0_HOST_TOOL": 1.0, # authoritative host tool - direct data access
|
|
62
|
+
"P1_NATIVE_CODE": 0.9, # deterministic code execution against real data
|
|
63
|
+
"P1_CODE_EXECUTION": 0.9, # collector alias for the code tier
|
|
64
|
+
"P2_TOOL_OUTPUT": 0.8, # coordinator-retrieved observations
|
|
65
|
+
"P2_EXPERIMENTAL": 0.8, # experimental / RCT-category evidence
|
|
66
|
+
"P4_PARAMETRIC": 0.5, # non-independent LLM probe - capped weak
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
# P3 web search weights (selected by `corroborated`):
|
|
70
|
+
_P3_TIER = "P3_WEB_SEARCH"
|
|
71
|
+
_P3_WEIGHT_SINGLE: float = 0.55 # one source - below ACT after P_SPAN
|
|
72
|
+
_P3_WEIGHT_CORROBORATED: float = 0.85 # >= 2 independent domains - can reach ACT
|
|
73
|
+
|
|
74
|
+
# Unknown tiers move the posterior nowhere (trust 0 → likelihood 0.5).
|
|
75
|
+
_UNKNOWN_WEIGHT: float = 0.5
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _tier_weight(probe_tier: str, corroborated: bool) -> float:
|
|
79
|
+
"""Return the trust weight for a probe tier (P3 depends on corroboration)."""
|
|
80
|
+
if probe_tier == _P3_TIER:
|
|
81
|
+
return _P3_WEIGHT_CORROBORATED if corroborated else _P3_WEIGHT_SINGLE
|
|
82
|
+
return TIER_TRUST_WEIGHTS.get(probe_tier, _UNKNOWN_WEIGHT)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def tier_confidence_likelihood(
|
|
86
|
+
probe_tier: str,
|
|
87
|
+
confidence: float,
|
|
88
|
+
evidence_supports: bool,
|
|
89
|
+
corroborated: bool = False,
|
|
90
|
+
) -> float:
|
|
91
|
+
"""Compute a Bayesian likelihood P(E|H) weighted by evidence tier + confidence.
|
|
92
|
+
|
|
93
|
+
Pure, deterministic, zero-LLM. Intended as the `likelihood_estimate` argument
|
|
94
|
+
to compute_posterior(prior, evidence_supports, likelihood_estimate=...).
|
|
95
|
+
|
|
96
|
+
Args:
|
|
97
|
+
probe_tier: evidence tier string (e.g. "P1_NATIVE_CODE",
|
|
98
|
+
"P3_WEB_SEARCH"). Unknown tiers → no posterior movement.
|
|
99
|
+
confidence: 0.0–1.0 confidence reported by the evidence collector
|
|
100
|
+
(clamped to [0, 1]).
|
|
101
|
+
evidence_supports: True if the evidence SUPPORTS the claim, False if it
|
|
102
|
+
CONTRADICTS.
|
|
103
|
+
corroborated: True when the (web) evidence was confirmed by >= 2
|
|
104
|
+
independent registrable domains. Only affects P3.
|
|
105
|
+
|
|
106
|
+
Returns:
|
|
107
|
+
Likelihood in the open interval (1e-6, 1-1e-6). With prior 0.5,
|
|
108
|
+
compute_posterior(0.5, evidence_supports, likelihood) ≈ likelihood.
|
|
109
|
+
"""
|
|
110
|
+
weight = _tier_weight(probe_tier, corroborated)
|
|
111
|
+
conf = max(0.0, min(1.0, confidence))
|
|
112
|
+
trust = weight * conf
|
|
113
|
+
|
|
114
|
+
if evidence_supports:
|
|
115
|
+
likelihood = 0.5 + P_SPAN * trust
|
|
116
|
+
else:
|
|
117
|
+
likelihood = 0.5 - P_SPAN * trust
|
|
118
|
+
|
|
119
|
+
# Fallibilism bound - never assert exact 0 or 1 (consistent with bayesian.py).
|
|
120
|
+
return max(_EPS, min(1.0 - _EPS, likelihood))
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""EIF Phase 2.5 - CAUSAL GATE: check causal reasoning validity.
|
|
2
|
+
|
|
3
|
+
v4 adds the Causal Evidence Probe (CEP): when a HIGH-consequence claim is
|
|
4
|
+
classified at INTERVENTION or COUNTERFACTUAL level, CEP searches for
|
|
5
|
+
independent causal evidence (RCTs, meta-analyses) and returns a
|
|
6
|
+
structured CausalEvidenceResult with posterior adjustment.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from eif.causal_gate.confound import check_confounders
|
|
10
|
+
from eif.causal_gate.direction import check_direction
|
|
11
|
+
from eif.causal_gate.evidence_probe import run_causal_evidence_probe, should_probe
|
|
12
|
+
from eif.causal_gate.intervention import check_intervention, classify_causal_level
|
|
13
|
+
from eif.causal_gate.verdict import (
|
|
14
|
+
apply_verdict_to_posterior,
|
|
15
|
+
posterior_delta,
|
|
16
|
+
provenance_flag_for,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
# v3 (unchanged)
|
|
21
|
+
"check_direction",
|
|
22
|
+
"check_confounders",
|
|
23
|
+
"classify_causal_level",
|
|
24
|
+
"check_intervention",
|
|
25
|
+
# v4 CEP
|
|
26
|
+
"run_causal_evidence_probe",
|
|
27
|
+
"should_probe",
|
|
28
|
+
"apply_verdict_to_posterior",
|
|
29
|
+
"posterior_delta",
|
|
30
|
+
"provenance_flag_for",
|
|
31
|
+
]
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
"""Confounder detection for causal hypotheses.
|
|
2
|
+
|
|
3
|
+
Two complementary strategies (IRIS-inspired, arXiv:2510.09217):
|
|
4
|
+
|
|
5
|
+
1. check_confounders - verifies that analyst-supplied confounders are
|
|
6
|
+
acknowledged in the hypothesis text (original).
|
|
7
|
+
|
|
8
|
+
2. discover_confounders - actively searches P3 for *unknown* confounders
|
|
9
|
+
using the cause/effect/domain extracted from the
|
|
10
|
+
claim. Returns a deduplicated list of confounder
|
|
11
|
+
candidates not already in the hypothesis.
|
|
12
|
+
Called unconditionally by eif_causal_gate for all
|
|
13
|
+
causal levels (not just INTERVENTION/COUNTERFACTUAL);
|
|
14
|
+
the CEP (CausalEvidenceProbe) is the tier that is
|
|
15
|
+
gated on HIGH-consequence INTERVENTION/COUNTERFACTUAL.
|
|
16
|
+
|
|
17
|
+
Research: IRIS (arXiv:2510.09217, ACL 2025) - iterative missing variable
|
|
18
|
+
identification from unstructured evidence. Key finding: the confounders most
|
|
19
|
+
likely to invalidate a causal claim are the ones *not in the dataset*, not the
|
|
20
|
+
ones already named by the analyst.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import logging
|
|
26
|
+
import re
|
|
27
|
+
from collections.abc import Callable
|
|
28
|
+
|
|
29
|
+
_log = logging.getLogger(__name__)
|
|
30
|
+
|
|
31
|
+
# Search suffix for confounder discovery
|
|
32
|
+
_CONFOUNDER_SUFFIX = "confounders mediators alternative explanations spurious correlation"
|
|
33
|
+
|
|
34
|
+
_EXTRACT_CONFOUNDERS_PROMPT = """\
|
|
35
|
+
A researcher claims: "{cause}" causes "{effect}" in the domain of "{domain}".
|
|
36
|
+
|
|
37
|
+
Based on the following search results, identify confounders, mediators, or
|
|
38
|
+
alternative explanations that could invalidate this causal claim:
|
|
39
|
+
|
|
40
|
+
---
|
|
41
|
+
{search_results}
|
|
42
|
+
---
|
|
43
|
+
|
|
44
|
+
Return a JSON array of confounder strings. Each item should be a short phrase
|
|
45
|
+
describing a specific variable or mechanism. Maximum 5 items. If no confounders
|
|
46
|
+
are identifiable from the results, return [].
|
|
47
|
+
|
|
48
|
+
Example output: ["selection bias", "socioeconomic status", "publication date effects"]
|
|
49
|
+
|
|
50
|
+
Return ONLY the JSON array, no commentary.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def check_confounders(
|
|
55
|
+
hypothesis: str,
|
|
56
|
+
potential_confounders: list[str] | None,
|
|
57
|
+
) -> list[str]:
|
|
58
|
+
"""Return analyst-supplied confounders not acknowledged in the hypothesis.
|
|
59
|
+
|
|
60
|
+
A confounder is considered documented if its name appears in the hypothesis
|
|
61
|
+
text. Undocumented confounders are flagged for review.
|
|
62
|
+
"""
|
|
63
|
+
if not potential_confounders:
|
|
64
|
+
return []
|
|
65
|
+
|
|
66
|
+
hypothesis_lower = hypothesis.lower()
|
|
67
|
+
return [c for c in potential_confounders if c.lower() not in hypothesis_lower]
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def discover_confounders(
|
|
71
|
+
cause: str,
|
|
72
|
+
effect: str,
|
|
73
|
+
domain: str,
|
|
74
|
+
hypothesis: str,
|
|
75
|
+
search_fn: Callable[[str], str] | None = None,
|
|
76
|
+
llm_fn: Callable[[str], str] | None = None,
|
|
77
|
+
) -> list[str]:
|
|
78
|
+
"""Actively discover unknown confounders via P3 search + LLM extraction.
|
|
79
|
+
|
|
80
|
+
This implements the IRIS missing variable identification step: search for
|
|
81
|
+
confounders that are not in the current dataset (hypothesis text) but are
|
|
82
|
+
known to the scientific community for this cause→effect relationship.
|
|
83
|
+
|
|
84
|
+
Args:
|
|
85
|
+
cause: extracted cause variable
|
|
86
|
+
effect: extracted effect variable
|
|
87
|
+
domain: domain context (e.g. "healthcare", "finance")
|
|
88
|
+
hypothesis: original hypothesis text - used to filter out already-
|
|
89
|
+
acknowledged confounders from the discovered list
|
|
90
|
+
search_fn: optional P3 search override (default: DDGS native_search)
|
|
91
|
+
llm_fn: optional LLM extractor (if None, uses heuristic extraction)
|
|
92
|
+
|
|
93
|
+
Returns:
|
|
94
|
+
List of newly-discovered confounder strings, excluding any already
|
|
95
|
+
mentioned in the hypothesis. Empty list if none found or on error.
|
|
96
|
+
"""
|
|
97
|
+
query = f"{cause} {effect} {_CONFOUNDER_SUFFIX} {domain}".strip()
|
|
98
|
+
|
|
99
|
+
# Run P3 search
|
|
100
|
+
search_results = ""
|
|
101
|
+
try:
|
|
102
|
+
if search_fn is not None:
|
|
103
|
+
search_results = search_fn(query)
|
|
104
|
+
else:
|
|
105
|
+
from eif.falsify.native_tools import native_search # noqa: PLC0415
|
|
106
|
+
search_results = native_search(query, max_results=5)
|
|
107
|
+
except Exception: # noqa: BLE001
|
|
108
|
+
_log.debug("discover_confounders: P3 search unavailable - returning []")
|
|
109
|
+
return []
|
|
110
|
+
|
|
111
|
+
if not search_results.strip():
|
|
112
|
+
return []
|
|
113
|
+
|
|
114
|
+
# Extract confounder candidates
|
|
115
|
+
candidates: list[str] = []
|
|
116
|
+
if llm_fn is not None:
|
|
117
|
+
candidates = _extract_via_llm(cause, effect, domain, search_results, llm_fn)
|
|
118
|
+
else:
|
|
119
|
+
candidates = _extract_heuristic(search_results)
|
|
120
|
+
|
|
121
|
+
# Filter out confounders already present in the hypothesis (case-insensitive)
|
|
122
|
+
hypothesis_lower = hypothesis.lower()
|
|
123
|
+
novel = [c for c in candidates if c.lower() not in hypothesis_lower]
|
|
124
|
+
|
|
125
|
+
if novel:
|
|
126
|
+
_log.info(
|
|
127
|
+
"IRIS: discovered %d novel confounder(s) for '%s → %s': %s",
|
|
128
|
+
len(novel), cause, effect, novel,
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
return novel[:5]
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _extract_via_llm(
|
|
135
|
+
cause: str,
|
|
136
|
+
effect: str,
|
|
137
|
+
domain: str,
|
|
138
|
+
search_results: str,
|
|
139
|
+
llm_fn: Callable[[str], str],
|
|
140
|
+
) -> list[str]:
|
|
141
|
+
"""Use LLM to extract confounder candidates from search results."""
|
|
142
|
+
import json
|
|
143
|
+
|
|
144
|
+
prompt = _EXTRACT_CONFOUNDERS_PROMPT.format(
|
|
145
|
+
cause=cause,
|
|
146
|
+
effect=effect,
|
|
147
|
+
domain=domain,
|
|
148
|
+
search_results=search_results[:3000],
|
|
149
|
+
)
|
|
150
|
+
try:
|
|
151
|
+
raw = llm_fn(prompt)
|
|
152
|
+
json_text = re.sub(r"```(?:json)?\s*(.*?)\s*```", r"\1", raw, flags=re.DOTALL).strip()
|
|
153
|
+
parsed = json.loads(json_text)
|
|
154
|
+
if isinstance(parsed, list):
|
|
155
|
+
return [str(item).strip() for item in parsed if item]
|
|
156
|
+
except Exception: # noqa: BLE001
|
|
157
|
+
_log.debug("discover_confounders: LLM extraction failed - falling back to heuristic")
|
|
158
|
+
|
|
159
|
+
return _extract_heuristic(search_results)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
# Common confounder signal phrases used in the heuristic fallback
|
|
163
|
+
_CONFOUNDER_SIGNALS = [
|
|
164
|
+
r"confound(?:ed|er|ing)\s+(?:by|with|variable)?\s*([\w\s]{4,40})",
|
|
165
|
+
r"spurious(?:ly)?\s+(?:correlated?|due to)\s*([\w\s]{4,40})",
|
|
166
|
+
r"mediat(?:ed|or|ing)\s+(?:by|through)?\s*([\w\s]{4,40})",
|
|
167
|
+
r"alternative\s+explanation[:\s]+([\w\s]{4,60})",
|
|
168
|
+
r"control(?:led|ling)?\s+for\s+([\w\s]{4,40})",
|
|
169
|
+
]
|
|
170
|
+
_CONFOUNDER_RE = re.compile(
|
|
171
|
+
"|".join(_CONFOUNDER_SIGNALS),
|
|
172
|
+
re.IGNORECASE,
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _extract_heuristic(search_results: str) -> list[str]:
|
|
177
|
+
"""Extract confounder phrases from raw search text using regex signals."""
|
|
178
|
+
found: list[str] = []
|
|
179
|
+
for match in _CONFOUNDER_RE.finditer(search_results):
|
|
180
|
+
# Take the first non-None group
|
|
181
|
+
phrase = next((g for g in match.groups() if g), None)
|
|
182
|
+
if phrase:
|
|
183
|
+
cleaned = phrase.strip().rstrip(".,;")
|
|
184
|
+
if 4 <= len(cleaned) <= 60:
|
|
185
|
+
found.append(cleaned)
|
|
186
|
+
|
|
187
|
+
# Deduplicate preserving order
|
|
188
|
+
seen: set[str] = set()
|
|
189
|
+
unique: list[str] = []
|
|
190
|
+
for item in found:
|
|
191
|
+
key = item.lower()
|
|
192
|
+
if key not in seen:
|
|
193
|
+
seen.add(key)
|
|
194
|
+
unique.append(item)
|
|
195
|
+
|
|
196
|
+
return unique[:5]
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""Check if evidence supports the claimed causal direction."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def check_direction(
|
|
7
|
+
cause_variable: str,
|
|
8
|
+
effect_variable: str,
|
|
9
|
+
evidence_text: str,
|
|
10
|
+
) -> bool:
|
|
11
|
+
"""Return True if the cause appears before the effect in the evidence text.
|
|
12
|
+
|
|
13
|
+
This is a simple heuristic: temporal ordering in text often reflects
|
|
14
|
+
causal ordering. A cause mentioned before its effect is consistent with
|
|
15
|
+
the claimed direction.
|
|
16
|
+
"""
|
|
17
|
+
cause_lower = cause_variable.lower()
|
|
18
|
+
effect_lower = effect_variable.lower()
|
|
19
|
+
text_lower = evidence_text.lower()
|
|
20
|
+
|
|
21
|
+
cause_pos = text_lower.find(cause_lower)
|
|
22
|
+
effect_pos = text_lower.find(effect_lower)
|
|
23
|
+
|
|
24
|
+
if cause_pos == -1 or effect_pos == -1:
|
|
25
|
+
# Can't determine direction from text - default to True (undetectable)
|
|
26
|
+
return True
|
|
27
|
+
|
|
28
|
+
return cause_pos < effect_pos
|
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
# EIF CAUSAL_GATE v4 - Evidence-Backed Causal Verification
|
|
2
|
+
# IVD Intent Artifact
|
|
3
|
+
# ==============================================================================
|
|
4
|
+
# When CAUSAL_GATE flags a causal claim, v4 goes beyond flagging: it searches
|
|
5
|
+
# for independent causal evidence (RCTs, meta-analyses, natural experiments,
|
|
6
|
+
# established DAGs) and returns a structured verdict, closing the loop from
|
|
7
|
+
# "this looks like a causal overclaim" to "here is or isn't independent
|
|
8
|
+
# evidence for the causal mechanism."
|
|
9
|
+
# ==============================================================================
|
|
10
|
+
|
|
11
|
+
name: "eif_causal_gate_v4"
|
|
12
|
+
version: "1.0"
|
|
13
|
+
status: "implemented" # CEP wired into eif_causal_gate MCP tool (2026-05-16); fires when consequence=HIGH + INTERVENTION/COUNTERFACTUAL; uses gpt-4.1-mini LLM callable; skips gracefully if no API key
|
|
14
|
+
created: "2026-05-08"
|
|
15
|
+
implemented: "2026-05-16"
|
|
16
|
+
|
|
17
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
18
|
+
# PROBLEM STATEMENT
|
|
19
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
20
|
+
|
|
21
|
+
problem: >
|
|
22
|
+
CAUSAL_GATE v3 classifies Pearl's causal hierarchy level and flags
|
|
23
|
+
undocumented confounders, but it stops at flagging. It outputs "reviewer
|
|
24
|
+
should verify" without actually verifying. This is peer review, not
|
|
25
|
+
experimentation. The scientific method requires that causal claims be
|
|
26
|
+
tested against independent evidence - not just identified and passed
|
|
27
|
+
through.
|
|
28
|
+
|
|
29
|
+
Research confirms:
|
|
30
|
+
- LLMs confuse P(Y|X) with P(Y|do(X)) systematically ("Rung Collapse",
|
|
31
|
+
ERM arXiv:2602.11675, 2025)
|
|
32
|
+
- Best LLM achieves F1=0.535 on real-world causal inference (ReCITE,
|
|
33
|
+
arXiv:2505.18931, 2025) - barely above chance
|
|
34
|
+
- LLMs treat conjunctive causes as disjunctive (arXiv:2505.09614, 2025)
|
|
35
|
+
- Multi-agent debate improves causal accuracy 78% → 87% (CRAwDAD,
|
|
36
|
+
arXiv:2511.22854, 2025) - but still below acceptable threshold
|
|
37
|
+
- Reasoning traces are often "Reasoning Theater" - correct conclusions
|
|
38
|
+
from contradictory internal logic (Ariadne, arXiv:2601.02314, 2026)
|
|
39
|
+
|
|
40
|
+
The gap: when CAUSAL_GATE fires on a HIGH-consequence causal claim, EIF
|
|
41
|
+
should execute a targeted evidence search for the specific causal
|
|
42
|
+
relationship and return a structured verdict. The infrastructure (P3
|
|
43
|
+
web search via DDGS, P1 code execution) already exists in FALSIFY.
|
|
44
|
+
|
|
45
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
46
|
+
# DESIGN DECISIONS
|
|
47
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
48
|
+
|
|
49
|
+
design:
|
|
50
|
+
approach: >
|
|
51
|
+
When CAUSAL_GATE flags a claim at INTERVENTION or COUNTERFACTUAL level
|
|
52
|
+
with HIGH consequence, v4 launches a Causal Evidence Probe (CEP) that:
|
|
53
|
+
1. Formulates a falsifiable causal query from the claim
|
|
54
|
+
2. Searches for RCTs, meta-analyses, natural experiments, or established
|
|
55
|
+
DAGs via P3 (DDGS native search)
|
|
56
|
+
3. Classifies the evidence found using Pearl's hierarchy
|
|
57
|
+
4. Returns a CausalVerdict: SUPPORTED, CONTESTED, NO_EVIDENCE, REVERSED
|
|
58
|
+
|
|
59
|
+
key_insight: >
|
|
60
|
+
We do NOT attempt to build a causal DAG or run do-calculus on the claim
|
|
61
|
+
programmatically. That requires domain-specific structural models.
|
|
62
|
+
Instead, we search for whether the causal question has already been
|
|
63
|
+
answered by the scientific community (RCTs, Cochrane reviews, natural
|
|
64
|
+
experiments, replicated observational studies with IV). This is the
|
|
65
|
+
correct application of the scientific method: check if someone already
|
|
66
|
+
ran the experiment, not re-derive causality from first principles.
|
|
67
|
+
|
|
68
|
+
llm_role_in_cep: >
|
|
69
|
+
CEP uses an LLM in two bounded, non-evidential roles only:
|
|
70
|
+
(1) extract_causal_pair - parses free-text claim into (cause, effect, domain)
|
|
71
|
+
for query formulation. This is NLP structuring, not evidence generation.
|
|
72
|
+
(2) classify_evidence - reads DDGS search snippets and classifies whether they
|
|
73
|
+
contain RCT/meta-analysis methodology. This is a text classifier reading
|
|
74
|
+
external content; the LLM does not supply causal facts from parametric memory.
|
|
75
|
+
evidence_source is always "P3_WEB_SEARCH" because facts come from DDGS.
|
|
76
|
+
If no OPENAI_API_KEY is set, extract_causal_pair falls back to a heuristic
|
|
77
|
+
(full claim as cause) and classify_evidence returns NO_EVIDENCE.
|
|
78
|
+
|
|
79
|
+
non_goals:
|
|
80
|
+
- "Do NOT build causal DAGs from agent conversations"
|
|
81
|
+
- "Do NOT run do-calculus - requires structural model we don't have"
|
|
82
|
+
- >
|
|
83
|
+
Do NOT use LLM parametric memory as causal evidence (that would be P4 - circular:
|
|
84
|
+
the same model that made the claim cannot validate it from its own training).
|
|
85
|
+
PERMITTED: lightweight LLM calls to structure or classify P3 search snippets
|
|
86
|
+
(e.g. extract_causal_pair, classify_evidence in evidence_probe.py). These are
|
|
87
|
+
NLP helpers operating on external DDGS results - the evidence_source is
|
|
88
|
+
P3_WEB_SEARCH. The LLM reads real external text; it does not supply causal facts
|
|
89
|
+
from parametric knowledge.
|
|
90
|
+
- "Do NOT slow down LOW-consequence causal claims - only probe HIGH"
|
|
91
|
+
|
|
92
|
+
trusted_sources:
|
|
93
|
+
libraries:
|
|
94
|
+
- name: "DoWhy"
|
|
95
|
+
github: "py-why/dowhy"
|
|
96
|
+
stars: 8049
|
|
97
|
+
use: "Reference for causal reasoning vocabulary and refutation patterns; not a runtime dependency"
|
|
98
|
+
- name: "causal-learn"
|
|
99
|
+
github: "py-why/causal-learn"
|
|
100
|
+
stars: 1579
|
|
101
|
+
use: "Reference for DAG structure patterns; not a runtime dependency"
|
|
102
|
+
research:
|
|
103
|
+
- "Pearl (2009) - Causality: Models, Reasoning, and Inference (canonical)"
|
|
104
|
+
- "ERM (arXiv:2602.11675, 2025) - Rung Collapse detection via regret minimization"
|
|
105
|
+
- "CRAwDAD (arXiv:2511.22854, 2025) - Dual-agent causal debate (78%→87%)"
|
|
106
|
+
- "Ariadne (arXiv:2601.02314, 2026) - Structural causal audit of reasoning faithfulness"
|
|
107
|
+
- "ReCITE (arXiv:2505.18931, 2025) - Real-world causal inference benchmark (F1=0.535)"
|
|
108
|
+
- "DoVerifier (arXiv:2601.21210, 2026) - Symbolic verification of do-calculus expressions"
|
|
109
|
+
- "CausalBench (OpenReview, 2025) - Multi-dimensional causal reasoning evaluation"
|
|
110
|
+
|
|
111
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
112
|
+
# CONSTRAINTS
|
|
113
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
114
|
+
|
|
115
|
+
constraints:
|
|
116
|
+
|
|
117
|
+
- id: CG1
|
|
118
|
+
description: >
|
|
119
|
+
Causal Evidence Probe fires only on HIGH-consequence claims classified
|
|
120
|
+
at INTERVENTION or COUNTERFACTUAL level. LOW/MEDIUM claims and
|
|
121
|
+
ASSOCIATION-level claims bypass the probe entirely.
|
|
122
|
+
test: >
|
|
123
|
+
Construct 6 test claims (2 LOW/ASSOCIATION, 2 MEDIUM/INTERVENTION,
|
|
124
|
+
2 HIGH/INTERVENTION). Only the HIGH/INTERVENTION pair triggers CEP.
|
|
125
|
+
Verify the others return the v3 flag-only behavior unchanged.
|
|
126
|
+
|
|
127
|
+
- id: CG2
|
|
128
|
+
description: >
|
|
129
|
+
CEP uses only P3 (DDGS native) for evidence search. It MUST NOT use
|
|
130
|
+
P4 (parametric LLM judgment) as causal evidence. The search query
|
|
131
|
+
must include "RCT OR meta-analysis OR randomized OR natural experiment
|
|
132
|
+
OR causal" alongside the domain-specific terms.
|
|
133
|
+
test: >
|
|
134
|
+
Run CEP on "microservices reduce latency" (HIGH). Verify the search
|
|
135
|
+
query contains causal evidence keywords. Verify no P4 fallback in the
|
|
136
|
+
evidence chain. Verify evidence_source field = P3_WEB_SEARCH.
|
|
137
|
+
|
|
138
|
+
- id: CG3
|
|
139
|
+
description: >
|
|
140
|
+
CausalVerdict is one of: SUPPORTED (RCT/meta-analysis found supporting
|
|
141
|
+
the direction), CONTESTED (evidence found but contradicts or shows
|
|
142
|
+
null effect), NO_EVIDENCE (no causal studies found for this specific
|
|
143
|
+
relationship), REVERSED (evidence supports opposite causal direction).
|
|
144
|
+
test: >
|
|
145
|
+
Run CEP on a known causal relationship ("smoking causes lung cancer")
|
|
146
|
+
and verify SUPPORTED. Run on a known non-causal correlation ("ice cream
|
|
147
|
+
sales cause drowning") and verify CONTESTED or NO_EVIDENCE.
|
|
148
|
+
|
|
149
|
+
- id: CG4
|
|
150
|
+
description: >
|
|
151
|
+
CausalVerdict feeds into CALIBRATE. SUPPORTED adds +0.15 to posterior.
|
|
152
|
+
NO_EVIDENCE is neutral (0.0). CONTESTED degrades posterior by -0.20.
|
|
153
|
+
REVERSED degrades by -0.35 (same as CONTRADICTS from FALSIFY).
|
|
154
|
+
test: >
|
|
155
|
+
Construct a claim at posterior=0.55 (REVISE zone). Apply SUPPORTED →
|
|
156
|
+
verify 0.70. Apply CONTESTED → verify 0.35 (HALT boundary). Apply
|
|
157
|
+
REVERSED → verify 0.20 (deep HALT).
|
|
158
|
+
|
|
159
|
+
- id: CG5
|
|
160
|
+
description: >
|
|
161
|
+
CEP runs under a hard 3-second asyncio.wait_for wall-clock cap.
|
|
162
|
+
If CEP exceeds this limit (DDGS/OpenAI latency spike), it is
|
|
163
|
+
skipped gracefully - the causal gate result is returned without
|
|
164
|
+
a CEP result (causal_evidence=null, cep_delta not applied).
|
|
165
|
+
The 3s cap is also the expected p95 latency target (< 3s mean).
|
|
166
|
+
It reuses the existing native_search (DDGS) function from
|
|
167
|
+
native_tools.py. No new network dependencies.
|
|
168
|
+
implementation_note: >
|
|
169
|
+
The cap is a hard cutoff, not a mean target. Under sustained
|
|
170
|
+
load CEP may skip more frequently. Acceptable trade-off: CEP is
|
|
171
|
+
optional evidence enrichment, not a required gate.
|
|
172
|
+
test: >
|
|
173
|
+
Time 10 CEP executions. Verify each completes in < 3s (hard cap enforced
|
|
174
|
+
by asyncio.wait_for). Verify timeout causes graceful skip, not error.
|
|
175
|
+
Verify import path uses eif.falsify.native_tools.native_search.
|
|
176
|
+
|
|
177
|
+
- id: CG6
|
|
178
|
+
description: >
|
|
179
|
+
When CEP returns NO_EVIDENCE and the claim is HIGH-consequence, the
|
|
180
|
+
RECORD phase must note "CAUSAL_UNVERIFIED" in the provenance output.
|
|
181
|
+
This satisfies EU AI Act Art. 9 (risk management) - the system
|
|
182
|
+
explicitly documents that a high-risk causal claim could not be
|
|
183
|
+
independently verified.
|
|
184
|
+
test: >
|
|
185
|
+
Run a HIGH claim through CEP where DDGS returns no RCT/meta-analysis.
|
|
186
|
+
Verify the assembled ProvenanceRecord has:
|
|
187
|
+
record.causal_evidence_result.verdict == "NO_EVIDENCE"
|
|
188
|
+
record.causal_unverified == True
|
|
189
|
+
(There is no top-level causal_verdict key on ProvenanceRecord - the
|
|
190
|
+
verdict lives on the nested CausalEvidenceResult model.)
|
|
191
|
+
|
|
192
|
+
- id: CG7
|
|
193
|
+
description: >
|
|
194
|
+
CEP must extract and cite the specific study found (title, year, journal
|
|
195
|
+
or arXiv ID if available) in the CausalVerdict output. Verdicts without
|
|
196
|
+
a citation are rejected and downgraded to NO_EVIDENCE.
|
|
197
|
+
test: >
|
|
198
|
+
Run CEP on "statins reduce cardiovascular mortality". Verify the
|
|
199
|
+
SUPPORTED verdict includes at least one citation with study name and year.
|
|
200
|
+
|
|
201
|
+
- id: CG8
|
|
202
|
+
description: >
|
|
203
|
+
CEP (Causal Evidence Probe) is gracefully optional when OPENAI_API_KEY is absent.
|
|
204
|
+
When the key is not set, eif_causal_gate must return the normal CausalGateResult
|
|
205
|
+
with causal_evidence=None (skipping CEP entirely), without raising an error or
|
|
206
|
+
degrading other causal gate behavior. This allows CEP-free operation in environments
|
|
207
|
+
without an OpenAI API key (e.g. offline, CI, cost-sensitive deployments).
|
|
208
|
+
test: >
|
|
209
|
+
Call eif_causal_gate with OPENAI_API_KEY unset (or empty).
|
|
210
|
+
Assert the tool returns a valid response dict with causal_evidence=None.
|
|
211
|
+
Assert no exception is raised. Assert the CausalGateResult fields
|
|
212
|
+
(verdict, causal_level, confounders) are populated normally.
|
|
213
|
+
|
|
214
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
215
|
+
# ARCHITECTURE
|
|
216
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
217
|
+
|
|
218
|
+
architecture:
|
|
219
|
+
new_files:
|
|
220
|
+
- "eif/eif/causal_gate/evidence_probe.py - CEP implementation"
|
|
221
|
+
- "eif/eif/causal_gate/verdict.py - CausalVerdict schema + posterior adjustment"
|
|
222
|
+
modified_files:
|
|
223
|
+
- "eif/eif/causal_gate/__init__.py - export CEP"
|
|
224
|
+
- "eif/eif/schemas.py - add CausalVerdict enum + CausalEvidenceResult model"
|
|
225
|
+
- "eif/eif/record/provenance.py - add CAUSAL_UNVERIFIED flag"
|
|
226
|
+
dependencies:
|
|
227
|
+
- "eif.falsify.native_tools.native_search (existing - DDGS)"
|
|
228
|
+
- "No new pip dependencies"
|
|
229
|
+
|
|
230
|
+
flow: |
|
|
231
|
+
DECLARE → claims extracted
|
|
232
|
+
↓
|
|
233
|
+
CAUSAL_GATE v3 (existing) → flags causal issues
|
|
234
|
+
↓ (if causal_level >= INTERVENTION AND consequence == HIGH)
|
|
235
|
+
CAUSAL_GATE v4 CEP:
|
|
236
|
+
1. Extract (cause, effect, domain) from claim text
|
|
237
|
+
2. Formulate P3 search query: "{cause} {effect} RCT OR meta-analysis"
|
|
238
|
+
3. native_search(query) → search results
|
|
239
|
+
4. LLM classify search results → CausalVerdict + citation
|
|
240
|
+
5. Feed verdict into CALIBRATE posterior adjustment
|
|
241
|
+
6. (Ancillary) discover_confounders() runs novel-confounder discovery via
|
|
242
|
+
IRIS-inspired active search - surfaces unknown-unknown confounders beyond
|
|
243
|
+
the static list; results appear as discovered_confounders[] in output
|
|
244
|
+
↓
|
|
245
|
+
CALIBRATE → route (ACT / REVISE / HALT)
|
|
246
|
+
↓
|
|
247
|
+
RECORD → provenance includes causal_evidence_result.verdict and causal_unverified flag
|
|
248
|
+
|
|
249
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
250
|
+
# CONSTRAINT SATISFIABILITY
|
|
251
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
252
|
+
|
|
253
|
+
constraint_satisfiability:
|
|
254
|
+
conflicts_checked:
|
|
255
|
+
- "CG1 (only HIGH) vs CG5 (< 3s): compatible - fewer probes means lower avg latency"
|
|
256
|
+
- "CG2 (no P4) vs CG4 (verdict classification): the search is P3, the classification
|
|
257
|
+
of search results uses an LLM - but the evidence itself is external (P3 source).
|
|
258
|
+
This is the same pattern as FALSIFY: LLM interprets external evidence, not self-validates."
|
|
259
|
+
- "CG7 (citation required) vs CG3 (NO_EVIDENCE): compatible - NO_EVIDENCE means
|
|
260
|
+
no citation was possible, which is the correct state"
|
|
261
|
+
tensions:
|
|
262
|
+
- "CG2 vs classification step: LLM judges whether a P3 result constitutes causal
|
|
263
|
+
support. This is analogous to FALSIFY's LLM judging P2/P3 evidence as SUPPORTS/
|
|
264
|
+
CONTRADICTS. Accepted trade-off - documented in design_decisions."
|
|
265
|
+
resolution: "All constraints simultaneously satisfiable."
|