eif-engine 4.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. eif/__init__.py +23 -0
  2. eif/auto.py +199 -0
  3. eif/calibrate/__init__.py +15 -0
  4. eif/calibrate/bayesian.py +53 -0
  5. eif/calibrate/conformal.py +57 -0
  6. eif/calibrate/domain_constraints.py +169 -0
  7. eif/calibrate/ece.py +154 -0
  8. eif/calibrate/prior_strategy.py +38 -0
  9. eif/calibrate/trust.py +120 -0
  10. eif/causal_gate/__init__.py +31 -0
  11. eif/causal_gate/confound.py +196 -0
  12. eif/causal_gate/direction.py +28 -0
  13. eif/causal_gate/eif_causal_gate_v4_intent.yaml +265 -0
  14. eif/causal_gate/evidence_probe.py +280 -0
  15. eif/causal_gate/intervention.py +55 -0
  16. eif/causal_gate/iris_pipeline.py +334 -0
  17. eif/causal_gate/verdict.py +47 -0
  18. eif/challenge/__init__.py +17 -0
  19. eif/challenge/diversity.py +58 -0
  20. eif/challenge/eif_multi_critic_intent.yaml +198 -0
  21. eif/challenge/multi_critic.py +270 -0
  22. eif/challenge/protocol.py +86 -0
  23. eif/challenge/replicator.py +379 -0
  24. eif/cost_model/__init__.py +5 -0
  25. eif/cost_model/catch_rate_empirical.py +275 -0
  26. eif/cost_model/cost_ledger.py +107 -0
  27. eif/cost_model/eif_catchrate_intent.yaml +86 -0
  28. eif/declare/__init__.py +6 -0
  29. eif/declare/extractor.py +308 -0
  30. eif/declare/harking_guard.py +110 -0
  31. eif/declare/registry.py +146 -0
  32. eif/eif_engine_intent.yaml +296 -0
  33. eif/eif_v2_intent.yaml +779 -0
  34. eif/eif_v3_intent.yaml +360 -0
  35. eif/explain/__init__.py +7 -0
  36. eif/explain/artifact.py +78 -0
  37. eif/explain/eif_explain_coldstart_intent.yaml +216 -0
  38. eif/explain/hard_to_vary.py +168 -0
  39. eif/explain/reach.py +32 -0
  40. eif/falsify/__init__.py +7 -0
  41. eif/falsify/condition.py +58 -0
  42. eif/falsify/eif_evidence_collection_intent.yaml +310 -0
  43. eif/falsify/eif_native_tools_intent.yaml +128 -0
  44. eif/falsify/eif_v5_evidence_trust_weighting_intent.yaml +273 -0
  45. eif/falsify/evidence_collector.py +1304 -0
  46. eif/falsify/hard_to_vary.py +171 -0
  47. eif/falsify/native_tools.py +221 -0
  48. eif/falsify/sprt.py +84 -0
  49. eif/falsify/trivial_check.py +46 -0
  50. eif/hypothesis_agenda/__init__.py +27 -0
  51. eif/hypothesis_agenda/agenda.py +176 -0
  52. eif/hypothesis_agenda/eif_hypothesis_agenda_intent.yaml +168 -0
  53. eif/hypothesis_agenda/fdr.py +92 -0
  54. eif/hypothesis_agenda/scorer.py +132 -0
  55. eif/input_guard/__init__.py +13 -0
  56. eif/input_guard/detector.py +259 -0
  57. eif/input_guard/eif_input_guard_intent.yaml +181 -0
  58. eif/integration/__init__.py +34 -0
  59. eif/integration/eif_auto.py +228 -0
  60. eif/integration/interceptor.py +218 -0
  61. eif/mcp_server/__init__.py +1 -0
  62. eif/mcp_server/auth.py +81 -0
  63. eif/mcp_server/http_server.py +460 -0
  64. eif/mcp_server/resources.py +113 -0
  65. eif/mcp_server/server.py +3485 -0
  66. eif/memory/__init__.py +5 -0
  67. eif/memory/strategy.py +246 -0
  68. eif/programme/__init__.py +7 -0
  69. eif/programme/monitor.py +58 -0
  70. eif/programme/principle_memory.py +177 -0
  71. eif/programme/principle_revision.py +412 -0
  72. eif/programme/signals.py +131 -0
  73. eif/programme/status.py +56 -0
  74. eif/py.typed +0 -0
  75. eif/record/__init__.py +7 -0
  76. eif/record/chain.py +33 -0
  77. eif/record/compliance.py +62 -0
  78. eif/record/isc_disclosure.py +257 -0
  79. eif/record/outcome_store.py +177 -0
  80. eif/record/provenance.py +148 -0
  81. eif/record/research_object.py +412 -0
  82. eif/replicate/__init__.py +6 -0
  83. eif/replicate/divergence.py +51 -0
  84. eif/replicate/eif_independent_replication_intent.yaml +109 -0
  85. eif/replicate/independent.py +185 -0
  86. eif/replicate/protocol.py +68 -0
  87. eif/reporting/__init__.py +1 -0
  88. eif/reporting/pdf_renderer.py +169 -0
  89. eif/schemas.py +787 -0
  90. eif/sdk/__init__.py +36 -0
  91. eif/sdk/exceptions.py +87 -0
  92. eif/sdk/interceptor.py +316 -0
  93. eif/session.py +40 -0
  94. eif/sycophancy/__init__.py +24 -0
  95. eif/sycophancy/cost_model.py +215 -0
  96. eif/sycophancy/detector.py +270 -0
  97. eif/sycophancy/drift.py +306 -0
  98. eif/sycophancy/eif_sycophancy_intent.yaml +309 -0
  99. eif/sycophancy/faithfulness.py +106 -0
  100. eif/sycophancy/framing.py +191 -0
  101. eif/update/__init__.py +14 -0
  102. eif/update/eig.py +29 -0
  103. eif/update/paradigm.py +121 -0
  104. eif/update/posterior.py +21 -0
  105. eif/update/stopping.py +42 -0
  106. eif_engine-4.1.0.dist-info/METADATA +607 -0
  107. eif_engine-4.1.0.dist-info/RECORD +111 -0
  108. eif_engine-4.1.0.dist-info/WHEEL +4 -0
  109. eif_engine-4.1.0.dist-info/entry_points.txt +3 -0
  110. eif_engine-4.1.0.dist-info/licenses/LICENSE +21 -0
  111. eif_engine-4.1.0.dist-info/licenses/NOTICE +17 -0
@@ -0,0 +1,38 @@
1
+ """Prior strategy selection for Bayesian calibration."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from eif.schemas import PRIOR_EMPIRICAL_MIN, PriorStrategy
6
+
7
+
8
+ def select_prior_strategy(
9
+ prior_provided: bool,
10
+ calibration_history_size: int,
11
+ ) -> tuple[PriorStrategy, float]:
12
+ """Select prior strategy and return (strategy, prior_value).
13
+
14
+ Selection rules:
15
+ - "domain" if user provides a prior (prior_provided=True)
16
+ - "empirical_bayes" if history >= PRIOR_EMPIRICAL_MIN (= 10)
17
+ - "max_entropy" otherwise (uniform prior = 0.5)
18
+
19
+ Returns a (strategy, prior_value) where prior_value is 0.5 for max_entropy
20
+ and empirical_bayes (caller must supply the actual value for domain).
21
+ """
22
+ if prior_provided:
23
+ return "domain", 0.5 # caller provides actual value
24
+
25
+ if calibration_history_size >= PRIOR_EMPIRICAL_MIN:
26
+ return "empirical_bayes", 0.5
27
+
28
+ return "max_entropy", 0.5
29
+
30
+
31
+ def empirical_bayes_prior(history_posteriors: list[float]) -> float:
32
+ """Estimate empirical Bayes prior from calibration history.
33
+
34
+ Simple hyperparameter estimation: mean of past posteriors.
35
+ """
36
+ if not history_posteriors:
37
+ return 0.5
38
+ return sum(history_posteriors) / len(history_posteriors)
eif/calibrate/trust.py ADDED
@@ -0,0 +1,120 @@
1
+ """Evidence trust weighting - tier- and confidence-aware Bayesian likelihood.
2
+
3
+ The eif_verify orchestrator previously used a flat likelihood of 0.8 (SUPPORTS)
4
+ / 0.2 (CONTRADICTS) for every probed claim, discarding the evidence's tier and
5
+ confidence. A single P3 web-search keyword match then produced the SAME posterior
6
+ as a P1 code-execution result. This module replaces that flat update with a
7
+ deterministic, zero-LLM trust function that weights the Bayesian likelihood by:
8
+
9
+ (a) the evidence tier - direct observation (P0/P1/P2) > web search (P3) >
10
+ parametric (P4); and
11
+ (b) the evidence confidence reported by the collector; and
12
+ (c) whether a web result was corroborated by >= 2 independent sources.
13
+
14
+ Design (with prior 0.5, compute_posterior(0.5, supports, L) ≈ L):
15
+
16
+ trust = tier_weight * clamp(confidence, 0, 1)
17
+ likelihood = 0.5 + P_SPAN * trust (SUPPORTS)
18
+ = 0.5 - P_SPAN * trust (CONTRADICTS)
19
+
20
+ with P_SPAN = 0.35 and the tier weights below. This yields:
21
+
22
+ P1 conf 0.9 → trust 0.81 → L ≈ 0.7835 → posterior ≈ 0.78 (>= ACT)
23
+ P3 single conf 0.65 → trust 0.3575 → L ≈ 0.6251 → posterior ≈ 0.63 (< ACT)
24
+ P3 corroborated conf 0.75→ trust 0.6375 → L ≈ 0.7231 → posterior ≈ 0.72 (>= ACT)
25
+
26
+ so a single uncorroborated web match cannot reach the ACT threshold (0.70) while
27
+ direct-observation tiers and corroborated web evidence can. See
28
+ eif/falsify/eif_v5_evidence_trust_weighting_intent.yaml (TW1, TW2, TW7).
29
+
30
+ Research basis:
31
+ PoisonedRAG (arXiv:2402.07867, USENIX Security 2025): injecting as few as 5 -
32
+ and in single-doc variants, 1 - crafted documents into a retrieval corpus
33
+ yields ~90% attack success. A verifier that treats one retrieved match as
34
+ decisive is directly exploitable; single-source web evidence must be
35
+ discounted and gated on corroboration.
36
+ The Power of Noise (arXiv:2401.14887): distracting / low-relevance retrieved
37
+ documents degrade RAG accuracy even alongside a correct document. Single
38
+ keyword-match web evidence is the noisy regime and must not carry the weight
39
+ of direct observation.
40
+
41
+ Invariant: this module makes ZERO LLM calls and ZERO network calls - it is pure
42
+ arithmetic, preserving tech_spec.md C1 (zero-LLM) and C4 (zero-network) for the
43
+ engine.
44
+ """
45
+
46
+ from __future__ import annotations
47
+
48
+ # Distance the likelihood may move from the 0.5 prior at full trust.
49
+ # With prior 0.5 the posterior ≈ likelihood, so P_SPAN bounds the per-probe
50
+ # posterior swing. 0.35 places a max-trust SUPPORTS at L=0.85 (matching the prior
51
+ # flat-0.8 ceiling for the strongest tier) while leaving headroom above ACT.
52
+ P_SPAN: float = 0.35
53
+
54
+ # Fallibilism bound - consistent with eif/calibrate/bayesian.py _FALLIBILISM_EPS.
55
+ _EPS: float = 1e-6
56
+
57
+ # Tier trust weights. Higher = the tier moves the posterior further from the
58
+ # 0.5 prior. P3 web search has two weights: uncorroborated (single source) vs
59
+ # corroborated (>= 2 independent registrable domains).
60
+ TIER_TRUST_WEIGHTS: dict[str, float] = {
61
+ "P0_HOST_TOOL": 1.0, # authoritative host tool - direct data access
62
+ "P1_NATIVE_CODE": 0.9, # deterministic code execution against real data
63
+ "P1_CODE_EXECUTION": 0.9, # collector alias for the code tier
64
+ "P2_TOOL_OUTPUT": 0.8, # coordinator-retrieved observations
65
+ "P2_EXPERIMENTAL": 0.8, # experimental / RCT-category evidence
66
+ "P4_PARAMETRIC": 0.5, # non-independent LLM probe - capped weak
67
+ }
68
+
69
+ # P3 web search weights (selected by `corroborated`):
70
+ _P3_TIER = "P3_WEB_SEARCH"
71
+ _P3_WEIGHT_SINGLE: float = 0.55 # one source - below ACT after P_SPAN
72
+ _P3_WEIGHT_CORROBORATED: float = 0.85 # >= 2 independent domains - can reach ACT
73
+
74
+ # Unknown tiers move the posterior nowhere (trust 0 → likelihood 0.5).
75
+ _UNKNOWN_WEIGHT: float = 0.5
76
+
77
+
78
+ def _tier_weight(probe_tier: str, corroborated: bool) -> float:
79
+ """Return the trust weight for a probe tier (P3 depends on corroboration)."""
80
+ if probe_tier == _P3_TIER:
81
+ return _P3_WEIGHT_CORROBORATED if corroborated else _P3_WEIGHT_SINGLE
82
+ return TIER_TRUST_WEIGHTS.get(probe_tier, _UNKNOWN_WEIGHT)
83
+
84
+
85
+ def tier_confidence_likelihood(
86
+ probe_tier: str,
87
+ confidence: float,
88
+ evidence_supports: bool,
89
+ corroborated: bool = False,
90
+ ) -> float:
91
+ """Compute a Bayesian likelihood P(E|H) weighted by evidence tier + confidence.
92
+
93
+ Pure, deterministic, zero-LLM. Intended as the `likelihood_estimate` argument
94
+ to compute_posterior(prior, evidence_supports, likelihood_estimate=...).
95
+
96
+ Args:
97
+ probe_tier: evidence tier string (e.g. "P1_NATIVE_CODE",
98
+ "P3_WEB_SEARCH"). Unknown tiers → no posterior movement.
99
+ confidence: 0.0–1.0 confidence reported by the evidence collector
100
+ (clamped to [0, 1]).
101
+ evidence_supports: True if the evidence SUPPORTS the claim, False if it
102
+ CONTRADICTS.
103
+ corroborated: True when the (web) evidence was confirmed by >= 2
104
+ independent registrable domains. Only affects P3.
105
+
106
+ Returns:
107
+ Likelihood in the open interval (1e-6, 1-1e-6). With prior 0.5,
108
+ compute_posterior(0.5, evidence_supports, likelihood) ≈ likelihood.
109
+ """
110
+ weight = _tier_weight(probe_tier, corroborated)
111
+ conf = max(0.0, min(1.0, confidence))
112
+ trust = weight * conf
113
+
114
+ if evidence_supports:
115
+ likelihood = 0.5 + P_SPAN * trust
116
+ else:
117
+ likelihood = 0.5 - P_SPAN * trust
118
+
119
+ # Fallibilism bound - never assert exact 0 or 1 (consistent with bayesian.py).
120
+ return max(_EPS, min(1.0 - _EPS, likelihood))
@@ -0,0 +1,31 @@
1
+ """EIF Phase 2.5 - CAUSAL GATE: check causal reasoning validity.
2
+
3
+ v4 adds the Causal Evidence Probe (CEP): when a HIGH-consequence claim is
4
+ classified at INTERVENTION or COUNTERFACTUAL level, CEP searches for
5
+ independent causal evidence (RCTs, meta-analyses) and returns a
6
+ structured CausalEvidenceResult with posterior adjustment.
7
+ """
8
+
9
+ from eif.causal_gate.confound import check_confounders
10
+ from eif.causal_gate.direction import check_direction
11
+ from eif.causal_gate.evidence_probe import run_causal_evidence_probe, should_probe
12
+ from eif.causal_gate.intervention import check_intervention, classify_causal_level
13
+ from eif.causal_gate.verdict import (
14
+ apply_verdict_to_posterior,
15
+ posterior_delta,
16
+ provenance_flag_for,
17
+ )
18
+
19
+ __all__ = [
20
+ # v3 (unchanged)
21
+ "check_direction",
22
+ "check_confounders",
23
+ "classify_causal_level",
24
+ "check_intervention",
25
+ # v4 CEP
26
+ "run_causal_evidence_probe",
27
+ "should_probe",
28
+ "apply_verdict_to_posterior",
29
+ "posterior_delta",
30
+ "provenance_flag_for",
31
+ ]
@@ -0,0 +1,196 @@
1
+ """Confounder detection for causal hypotheses.
2
+
3
+ Two complementary strategies (IRIS-inspired, arXiv:2510.09217):
4
+
5
+ 1. check_confounders - verifies that analyst-supplied confounders are
6
+ acknowledged in the hypothesis text (original).
7
+
8
+ 2. discover_confounders - actively searches P3 for *unknown* confounders
9
+ using the cause/effect/domain extracted from the
10
+ claim. Returns a deduplicated list of confounder
11
+ candidates not already in the hypothesis.
12
+ Called unconditionally by eif_causal_gate for all
13
+ causal levels (not just INTERVENTION/COUNTERFACTUAL);
14
+ the CEP (CausalEvidenceProbe) is the tier that is
15
+ gated on HIGH-consequence INTERVENTION/COUNTERFACTUAL.
16
+
17
+ Research: IRIS (arXiv:2510.09217, ACL 2025) - iterative missing variable
18
+ identification from unstructured evidence. Key finding: the confounders most
19
+ likely to invalidate a causal claim are the ones *not in the dataset*, not the
20
+ ones already named by the analyst.
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ import logging
26
+ import re
27
+ from collections.abc import Callable
28
+
29
+ _log = logging.getLogger(__name__)
30
+
31
+ # Search suffix for confounder discovery
32
+ _CONFOUNDER_SUFFIX = "confounders mediators alternative explanations spurious correlation"
33
+
34
+ _EXTRACT_CONFOUNDERS_PROMPT = """\
35
+ A researcher claims: "{cause}" causes "{effect}" in the domain of "{domain}".
36
+
37
+ Based on the following search results, identify confounders, mediators, or
38
+ alternative explanations that could invalidate this causal claim:
39
+
40
+ ---
41
+ {search_results}
42
+ ---
43
+
44
+ Return a JSON array of confounder strings. Each item should be a short phrase
45
+ describing a specific variable or mechanism. Maximum 5 items. If no confounders
46
+ are identifiable from the results, return [].
47
+
48
+ Example output: ["selection bias", "socioeconomic status", "publication date effects"]
49
+
50
+ Return ONLY the JSON array, no commentary.
51
+ """
52
+
53
+
54
+ def check_confounders(
55
+ hypothesis: str,
56
+ potential_confounders: list[str] | None,
57
+ ) -> list[str]:
58
+ """Return analyst-supplied confounders not acknowledged in the hypothesis.
59
+
60
+ A confounder is considered documented if its name appears in the hypothesis
61
+ text. Undocumented confounders are flagged for review.
62
+ """
63
+ if not potential_confounders:
64
+ return []
65
+
66
+ hypothesis_lower = hypothesis.lower()
67
+ return [c for c in potential_confounders if c.lower() not in hypothesis_lower]
68
+
69
+
70
+ def discover_confounders(
71
+ cause: str,
72
+ effect: str,
73
+ domain: str,
74
+ hypothesis: str,
75
+ search_fn: Callable[[str], str] | None = None,
76
+ llm_fn: Callable[[str], str] | None = None,
77
+ ) -> list[str]:
78
+ """Actively discover unknown confounders via P3 search + LLM extraction.
79
+
80
+ This implements the IRIS missing variable identification step: search for
81
+ confounders that are not in the current dataset (hypothesis text) but are
82
+ known to the scientific community for this cause→effect relationship.
83
+
84
+ Args:
85
+ cause: extracted cause variable
86
+ effect: extracted effect variable
87
+ domain: domain context (e.g. "healthcare", "finance")
88
+ hypothesis: original hypothesis text - used to filter out already-
89
+ acknowledged confounders from the discovered list
90
+ search_fn: optional P3 search override (default: DDGS native_search)
91
+ llm_fn: optional LLM extractor (if None, uses heuristic extraction)
92
+
93
+ Returns:
94
+ List of newly-discovered confounder strings, excluding any already
95
+ mentioned in the hypothesis. Empty list if none found or on error.
96
+ """
97
+ query = f"{cause} {effect} {_CONFOUNDER_SUFFIX} {domain}".strip()
98
+
99
+ # Run P3 search
100
+ search_results = ""
101
+ try:
102
+ if search_fn is not None:
103
+ search_results = search_fn(query)
104
+ else:
105
+ from eif.falsify.native_tools import native_search # noqa: PLC0415
106
+ search_results = native_search(query, max_results=5)
107
+ except Exception: # noqa: BLE001
108
+ _log.debug("discover_confounders: P3 search unavailable - returning []")
109
+ return []
110
+
111
+ if not search_results.strip():
112
+ return []
113
+
114
+ # Extract confounder candidates
115
+ candidates: list[str] = []
116
+ if llm_fn is not None:
117
+ candidates = _extract_via_llm(cause, effect, domain, search_results, llm_fn)
118
+ else:
119
+ candidates = _extract_heuristic(search_results)
120
+
121
+ # Filter out confounders already present in the hypothesis (case-insensitive)
122
+ hypothesis_lower = hypothesis.lower()
123
+ novel = [c for c in candidates if c.lower() not in hypothesis_lower]
124
+
125
+ if novel:
126
+ _log.info(
127
+ "IRIS: discovered %d novel confounder(s) for '%s → %s': %s",
128
+ len(novel), cause, effect, novel,
129
+ )
130
+
131
+ return novel[:5]
132
+
133
+
134
+ def _extract_via_llm(
135
+ cause: str,
136
+ effect: str,
137
+ domain: str,
138
+ search_results: str,
139
+ llm_fn: Callable[[str], str],
140
+ ) -> list[str]:
141
+ """Use LLM to extract confounder candidates from search results."""
142
+ import json
143
+
144
+ prompt = _EXTRACT_CONFOUNDERS_PROMPT.format(
145
+ cause=cause,
146
+ effect=effect,
147
+ domain=domain,
148
+ search_results=search_results[:3000],
149
+ )
150
+ try:
151
+ raw = llm_fn(prompt)
152
+ json_text = re.sub(r"```(?:json)?\s*(.*?)\s*```", r"\1", raw, flags=re.DOTALL).strip()
153
+ parsed = json.loads(json_text)
154
+ if isinstance(parsed, list):
155
+ return [str(item).strip() for item in parsed if item]
156
+ except Exception: # noqa: BLE001
157
+ _log.debug("discover_confounders: LLM extraction failed - falling back to heuristic")
158
+
159
+ return _extract_heuristic(search_results)
160
+
161
+
162
+ # Common confounder signal phrases used in the heuristic fallback
163
+ _CONFOUNDER_SIGNALS = [
164
+ r"confound(?:ed|er|ing)\s+(?:by|with|variable)?\s*([\w\s]{4,40})",
165
+ r"spurious(?:ly)?\s+(?:correlated?|due to)\s*([\w\s]{4,40})",
166
+ r"mediat(?:ed|or|ing)\s+(?:by|through)?\s*([\w\s]{4,40})",
167
+ r"alternative\s+explanation[:\s]+([\w\s]{4,60})",
168
+ r"control(?:led|ling)?\s+for\s+([\w\s]{4,40})",
169
+ ]
170
+ _CONFOUNDER_RE = re.compile(
171
+ "|".join(_CONFOUNDER_SIGNALS),
172
+ re.IGNORECASE,
173
+ )
174
+
175
+
176
+ def _extract_heuristic(search_results: str) -> list[str]:
177
+ """Extract confounder phrases from raw search text using regex signals."""
178
+ found: list[str] = []
179
+ for match in _CONFOUNDER_RE.finditer(search_results):
180
+ # Take the first non-None group
181
+ phrase = next((g for g in match.groups() if g), None)
182
+ if phrase:
183
+ cleaned = phrase.strip().rstrip(".,;")
184
+ if 4 <= len(cleaned) <= 60:
185
+ found.append(cleaned)
186
+
187
+ # Deduplicate preserving order
188
+ seen: set[str] = set()
189
+ unique: list[str] = []
190
+ for item in found:
191
+ key = item.lower()
192
+ if key not in seen:
193
+ seen.add(key)
194
+ unique.append(item)
195
+
196
+ return unique[:5]
@@ -0,0 +1,28 @@
1
+ """Check if evidence supports the claimed causal direction."""
2
+
3
+ from __future__ import annotations
4
+
5
+
6
+ def check_direction(
7
+ cause_variable: str,
8
+ effect_variable: str,
9
+ evidence_text: str,
10
+ ) -> bool:
11
+ """Return True if the cause appears before the effect in the evidence text.
12
+
13
+ This is a simple heuristic: temporal ordering in text often reflects
14
+ causal ordering. A cause mentioned before its effect is consistent with
15
+ the claimed direction.
16
+ """
17
+ cause_lower = cause_variable.lower()
18
+ effect_lower = effect_variable.lower()
19
+ text_lower = evidence_text.lower()
20
+
21
+ cause_pos = text_lower.find(cause_lower)
22
+ effect_pos = text_lower.find(effect_lower)
23
+
24
+ if cause_pos == -1 or effect_pos == -1:
25
+ # Can't determine direction from text - default to True (undetectable)
26
+ return True
27
+
28
+ return cause_pos < effect_pos
@@ -0,0 +1,265 @@
1
+ # EIF CAUSAL_GATE v4 - Evidence-Backed Causal Verification
2
+ # IVD Intent Artifact
3
+ # ==============================================================================
4
+ # When CAUSAL_GATE flags a causal claim, v4 goes beyond flagging: it searches
5
+ # for independent causal evidence (RCTs, meta-analyses, natural experiments,
6
+ # established DAGs) and returns a structured verdict, closing the loop from
7
+ # "this looks like a causal overclaim" to "here is or isn't independent
8
+ # evidence for the causal mechanism."
9
+ # ==============================================================================
10
+
11
+ name: "eif_causal_gate_v4"
12
+ version: "1.0"
13
+ status: "implemented" # CEP wired into eif_causal_gate MCP tool (2026-05-16); fires when consequence=HIGH + INTERVENTION/COUNTERFACTUAL; uses gpt-4.1-mini LLM callable; skips gracefully if no API key
14
+ created: "2026-05-08"
15
+ implemented: "2026-05-16"
16
+
17
+ # ─────────────────────────────────────────────────────────────────────────────
18
+ # PROBLEM STATEMENT
19
+ # ─────────────────────────────────────────────────────────────────────────────
20
+
21
+ problem: >
22
+ CAUSAL_GATE v3 classifies Pearl's causal hierarchy level and flags
23
+ undocumented confounders, but it stops at flagging. It outputs "reviewer
24
+ should verify" without actually verifying. This is peer review, not
25
+ experimentation. The scientific method requires that causal claims be
26
+ tested against independent evidence - not just identified and passed
27
+ through.
28
+
29
+ Research confirms:
30
+ - LLMs confuse P(Y|X) with P(Y|do(X)) systematically ("Rung Collapse",
31
+ ERM arXiv:2602.11675, 2025)
32
+ - Best LLM achieves F1=0.535 on real-world causal inference (ReCITE,
33
+ arXiv:2505.18931, 2025) - barely above chance
34
+ - LLMs treat conjunctive causes as disjunctive (arXiv:2505.09614, 2025)
35
+ - Multi-agent debate improves causal accuracy 78% → 87% (CRAwDAD,
36
+ arXiv:2511.22854, 2025) - but still below acceptable threshold
37
+ - Reasoning traces are often "Reasoning Theater" - correct conclusions
38
+ from contradictory internal logic (Ariadne, arXiv:2601.02314, 2026)
39
+
40
+ The gap: when CAUSAL_GATE fires on a HIGH-consequence causal claim, EIF
41
+ should execute a targeted evidence search for the specific causal
42
+ relationship and return a structured verdict. The infrastructure (P3
43
+ web search via DDGS, P1 code execution) already exists in FALSIFY.
44
+
45
+ # ─────────────────────────────────────────────────────────────────────────────
46
+ # DESIGN DECISIONS
47
+ # ─────────────────────────────────────────────────────────────────────────────
48
+
49
+ design:
50
+ approach: >
51
+ When CAUSAL_GATE flags a claim at INTERVENTION or COUNTERFACTUAL level
52
+ with HIGH consequence, v4 launches a Causal Evidence Probe (CEP) that:
53
+ 1. Formulates a falsifiable causal query from the claim
54
+ 2. Searches for RCTs, meta-analyses, natural experiments, or established
55
+ DAGs via P3 (DDGS native search)
56
+ 3. Classifies the evidence found using Pearl's hierarchy
57
+ 4. Returns a CausalVerdict: SUPPORTED, CONTESTED, NO_EVIDENCE, REVERSED
58
+
59
+ key_insight: >
60
+ We do NOT attempt to build a causal DAG or run do-calculus on the claim
61
+ programmatically. That requires domain-specific structural models.
62
+ Instead, we search for whether the causal question has already been
63
+ answered by the scientific community (RCTs, Cochrane reviews, natural
64
+ experiments, replicated observational studies with IV). This is the
65
+ correct application of the scientific method: check if someone already
66
+ ran the experiment, not re-derive causality from first principles.
67
+
68
+ llm_role_in_cep: >
69
+ CEP uses an LLM in two bounded, non-evidential roles only:
70
+ (1) extract_causal_pair - parses free-text claim into (cause, effect, domain)
71
+ for query formulation. This is NLP structuring, not evidence generation.
72
+ (2) classify_evidence - reads DDGS search snippets and classifies whether they
73
+ contain RCT/meta-analysis methodology. This is a text classifier reading
74
+ external content; the LLM does not supply causal facts from parametric memory.
75
+ evidence_source is always "P3_WEB_SEARCH" because facts come from DDGS.
76
+ If no OPENAI_API_KEY is set, extract_causal_pair falls back to a heuristic
77
+ (full claim as cause) and classify_evidence returns NO_EVIDENCE.
78
+
79
+ non_goals:
80
+ - "Do NOT build causal DAGs from agent conversations"
81
+ - "Do NOT run do-calculus - requires structural model we don't have"
82
+ - >
83
+ Do NOT use LLM parametric memory as causal evidence (that would be P4 - circular:
84
+ the same model that made the claim cannot validate it from its own training).
85
+ PERMITTED: lightweight LLM calls to structure or classify P3 search snippets
86
+ (e.g. extract_causal_pair, classify_evidence in evidence_probe.py). These are
87
+ NLP helpers operating on external DDGS results - the evidence_source is
88
+ P3_WEB_SEARCH. The LLM reads real external text; it does not supply causal facts
89
+ from parametric knowledge.
90
+ - "Do NOT slow down LOW-consequence causal claims - only probe HIGH"
91
+
92
+ trusted_sources:
93
+ libraries:
94
+ - name: "DoWhy"
95
+ github: "py-why/dowhy"
96
+ stars: 8049
97
+ use: "Reference for causal reasoning vocabulary and refutation patterns; not a runtime dependency"
98
+ - name: "causal-learn"
99
+ github: "py-why/causal-learn"
100
+ stars: 1579
101
+ use: "Reference for DAG structure patterns; not a runtime dependency"
102
+ research:
103
+ - "Pearl (2009) - Causality: Models, Reasoning, and Inference (canonical)"
104
+ - "ERM (arXiv:2602.11675, 2025) - Rung Collapse detection via regret minimization"
105
+ - "CRAwDAD (arXiv:2511.22854, 2025) - Dual-agent causal debate (78%→87%)"
106
+ - "Ariadne (arXiv:2601.02314, 2026) - Structural causal audit of reasoning faithfulness"
107
+ - "ReCITE (arXiv:2505.18931, 2025) - Real-world causal inference benchmark (F1=0.535)"
108
+ - "DoVerifier (arXiv:2601.21210, 2026) - Symbolic verification of do-calculus expressions"
109
+ - "CausalBench (OpenReview, 2025) - Multi-dimensional causal reasoning evaluation"
110
+
111
+ # ─────────────────────────────────────────────────────────────────────────────
112
+ # CONSTRAINTS
113
+ # ─────────────────────────────────────────────────────────────────────────────
114
+
115
+ constraints:
116
+
117
+ - id: CG1
118
+ description: >
119
+ Causal Evidence Probe fires only on HIGH-consequence claims classified
120
+ at INTERVENTION or COUNTERFACTUAL level. LOW/MEDIUM claims and
121
+ ASSOCIATION-level claims bypass the probe entirely.
122
+ test: >
123
+ Construct 6 test claims (2 LOW/ASSOCIATION, 2 MEDIUM/INTERVENTION,
124
+ 2 HIGH/INTERVENTION). Only the HIGH/INTERVENTION pair triggers CEP.
125
+ Verify the others return the v3 flag-only behavior unchanged.
126
+
127
+ - id: CG2
128
+ description: >
129
+ CEP uses only P3 (DDGS native) for evidence search. It MUST NOT use
130
+ P4 (parametric LLM judgment) as causal evidence. The search query
131
+ must include "RCT OR meta-analysis OR randomized OR natural experiment
132
+ OR causal" alongside the domain-specific terms.
133
+ test: >
134
+ Run CEP on "microservices reduce latency" (HIGH). Verify the search
135
+ query contains causal evidence keywords. Verify no P4 fallback in the
136
+ evidence chain. Verify evidence_source field = P3_WEB_SEARCH.
137
+
138
+ - id: CG3
139
+ description: >
140
+ CausalVerdict is one of: SUPPORTED (RCT/meta-analysis found supporting
141
+ the direction), CONTESTED (evidence found but contradicts or shows
142
+ null effect), NO_EVIDENCE (no causal studies found for this specific
143
+ relationship), REVERSED (evidence supports opposite causal direction).
144
+ test: >
145
+ Run CEP on a known causal relationship ("smoking causes lung cancer")
146
+ and verify SUPPORTED. Run on a known non-causal correlation ("ice cream
147
+ sales cause drowning") and verify CONTESTED or NO_EVIDENCE.
148
+
149
+ - id: CG4
150
+ description: >
151
+ CausalVerdict feeds into CALIBRATE. SUPPORTED adds +0.15 to posterior.
152
+ NO_EVIDENCE is neutral (0.0). CONTESTED degrades posterior by -0.20.
153
+ REVERSED degrades by -0.35 (same as CONTRADICTS from FALSIFY).
154
+ test: >
155
+ Construct a claim at posterior=0.55 (REVISE zone). Apply SUPPORTED →
156
+ verify 0.70. Apply CONTESTED → verify 0.35 (HALT boundary). Apply
157
+ REVERSED → verify 0.20 (deep HALT).
158
+
159
+ - id: CG5
160
+ description: >
161
+ CEP runs under a hard 3-second asyncio.wait_for wall-clock cap.
162
+ If CEP exceeds this limit (DDGS/OpenAI latency spike), it is
163
+ skipped gracefully - the causal gate result is returned without
164
+ a CEP result (causal_evidence=null, cep_delta not applied).
165
+ The 3s cap is also the expected p95 latency target (< 3s mean).
166
+ It reuses the existing native_search (DDGS) function from
167
+ native_tools.py. No new network dependencies.
168
+ implementation_note: >
169
+ The cap is a hard cutoff, not a mean target. Under sustained
170
+ load CEP may skip more frequently. Acceptable trade-off: CEP is
171
+ optional evidence enrichment, not a required gate.
172
+ test: >
173
+ Time 10 CEP executions. Verify each completes in < 3s (hard cap enforced
174
+ by asyncio.wait_for). Verify timeout causes graceful skip, not error.
175
+ Verify import path uses eif.falsify.native_tools.native_search.
176
+
177
+ - id: CG6
178
+ description: >
179
+ When CEP returns NO_EVIDENCE and the claim is HIGH-consequence, the
180
+ RECORD phase must note "CAUSAL_UNVERIFIED" in the provenance output.
181
+ This satisfies EU AI Act Art. 9 (risk management) - the system
182
+ explicitly documents that a high-risk causal claim could not be
183
+ independently verified.
184
+ test: >
185
+ Run a HIGH claim through CEP where DDGS returns no RCT/meta-analysis.
186
+ Verify the assembled ProvenanceRecord has:
187
+ record.causal_evidence_result.verdict == "NO_EVIDENCE"
188
+ record.causal_unverified == True
189
+ (There is no top-level causal_verdict key on ProvenanceRecord - the
190
+ verdict lives on the nested CausalEvidenceResult model.)
191
+
192
+ - id: CG7
193
+ description: >
194
+ CEP must extract and cite the specific study found (title, year, journal
195
+ or arXiv ID if available) in the CausalVerdict output. Verdicts without
196
+ a citation are rejected and downgraded to NO_EVIDENCE.
197
+ test: >
198
+ Run CEP on "statins reduce cardiovascular mortality". Verify the
199
+ SUPPORTED verdict includes at least one citation with study name and year.
200
+
201
+ - id: CG8
202
+ description: >
203
+ CEP (Causal Evidence Probe) is gracefully optional when OPENAI_API_KEY is absent.
204
+ When the key is not set, eif_causal_gate must return the normal CausalGateResult
205
+ with causal_evidence=None (skipping CEP entirely), without raising an error or
206
+ degrading other causal gate behavior. This allows CEP-free operation in environments
207
+ without an OpenAI API key (e.g. offline, CI, cost-sensitive deployments).
208
+ test: >
209
+ Call eif_causal_gate with OPENAI_API_KEY unset (or empty).
210
+ Assert the tool returns a valid response dict with causal_evidence=None.
211
+ Assert no exception is raised. Assert the CausalGateResult fields
212
+ (verdict, causal_level, confounders) are populated normally.
213
+
214
+ # ─────────────────────────────────────────────────────────────────────────────
215
+ # ARCHITECTURE
216
+ # ─────────────────────────────────────────────────────────────────────────────
217
+
218
+ architecture:
219
+ new_files:
220
+ - "eif/eif/causal_gate/evidence_probe.py - CEP implementation"
221
+ - "eif/eif/causal_gate/verdict.py - CausalVerdict schema + posterior adjustment"
222
+ modified_files:
223
+ - "eif/eif/causal_gate/__init__.py - export CEP"
224
+ - "eif/eif/schemas.py - add CausalVerdict enum + CausalEvidenceResult model"
225
+ - "eif/eif/record/provenance.py - add CAUSAL_UNVERIFIED flag"
226
+ dependencies:
227
+ - "eif.falsify.native_tools.native_search (existing - DDGS)"
228
+ - "No new pip dependencies"
229
+
230
+ flow: |
231
+ DECLARE → claims extracted
232
+ ↓
233
+ CAUSAL_GATE v3 (existing) → flags causal issues
234
+ ↓ (if causal_level >= INTERVENTION AND consequence == HIGH)
235
+ CAUSAL_GATE v4 CEP:
236
+ 1. Extract (cause, effect, domain) from claim text
237
+ 2. Formulate P3 search query: "{cause} {effect} RCT OR meta-analysis"
238
+ 3. native_search(query) → search results
239
+ 4. LLM classify search results → CausalVerdict + citation
240
+ 5. Feed verdict into CALIBRATE posterior adjustment
241
+ 6. (Ancillary) discover_confounders() runs novel-confounder discovery via
242
+ IRIS-inspired active search - surfaces unknown-unknown confounders beyond
243
+ the static list; results appear as discovered_confounders[] in output
244
+ ↓
245
+ CALIBRATE → route (ACT / REVISE / HALT)
246
+ ↓
247
+ RECORD → provenance includes causal_evidence_result.verdict and causal_unverified flag
248
+
249
+ # ─────────────────────────────────────────────────────────────────────────────
250
+ # CONSTRAINT SATISFIABILITY
251
+ # ─────────────────────────────────────────────────────────────────────────────
252
+
253
+ constraint_satisfiability:
254
+ conflicts_checked:
255
+ - "CG1 (only HIGH) vs CG5 (< 3s): compatible - fewer probes means lower avg latency"
256
+ - "CG2 (no P4) vs CG4 (verdict classification): the search is P3, the classification
257
+ of search results uses an LLM - but the evidence itself is external (P3 source).
258
+ This is the same pattern as FALSIFY: LLM interprets external evidence, not self-validates."
259
+ - "CG7 (citation required) vs CG3 (NO_EVIDENCE): compatible - NO_EVIDENCE means
260
+ no citation was possible, which is the correct state"
261
+ tensions:
262
+ - "CG2 vs classification step: LLM judges whether a P3 result constitutes causal
263
+ support. This is analogous to FALSIFY's LLM judging P2/P3 evidence as SUPPORTS/
264
+ CONTRADICTS. Accepted trade-off - documented in design_decisions."
265
+ resolution: "All constraints simultaneously satisfiable."