eif-engine 4.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eif/__init__.py +23 -0
- eif/auto.py +199 -0
- eif/calibrate/__init__.py +15 -0
- eif/calibrate/bayesian.py +53 -0
- eif/calibrate/conformal.py +57 -0
- eif/calibrate/domain_constraints.py +169 -0
- eif/calibrate/ece.py +154 -0
- eif/calibrate/prior_strategy.py +38 -0
- eif/calibrate/trust.py +120 -0
- eif/causal_gate/__init__.py +31 -0
- eif/causal_gate/confound.py +196 -0
- eif/causal_gate/direction.py +28 -0
- eif/causal_gate/eif_causal_gate_v4_intent.yaml +265 -0
- eif/causal_gate/evidence_probe.py +280 -0
- eif/causal_gate/intervention.py +55 -0
- eif/causal_gate/iris_pipeline.py +334 -0
- eif/causal_gate/verdict.py +47 -0
- eif/challenge/__init__.py +17 -0
- eif/challenge/diversity.py +58 -0
- eif/challenge/eif_multi_critic_intent.yaml +198 -0
- eif/challenge/multi_critic.py +270 -0
- eif/challenge/protocol.py +86 -0
- eif/challenge/replicator.py +379 -0
- eif/cost_model/__init__.py +5 -0
- eif/cost_model/catch_rate_empirical.py +275 -0
- eif/cost_model/cost_ledger.py +107 -0
- eif/cost_model/eif_catchrate_intent.yaml +86 -0
- eif/declare/__init__.py +6 -0
- eif/declare/extractor.py +308 -0
- eif/declare/harking_guard.py +110 -0
- eif/declare/registry.py +146 -0
- eif/eif_engine_intent.yaml +296 -0
- eif/eif_v2_intent.yaml +779 -0
- eif/eif_v3_intent.yaml +360 -0
- eif/explain/__init__.py +7 -0
- eif/explain/artifact.py +78 -0
- eif/explain/eif_explain_coldstart_intent.yaml +216 -0
- eif/explain/hard_to_vary.py +168 -0
- eif/explain/reach.py +32 -0
- eif/falsify/__init__.py +7 -0
- eif/falsify/condition.py +58 -0
- eif/falsify/eif_evidence_collection_intent.yaml +310 -0
- eif/falsify/eif_native_tools_intent.yaml +128 -0
- eif/falsify/eif_v5_evidence_trust_weighting_intent.yaml +273 -0
- eif/falsify/evidence_collector.py +1304 -0
- eif/falsify/hard_to_vary.py +171 -0
- eif/falsify/native_tools.py +221 -0
- eif/falsify/sprt.py +84 -0
- eif/falsify/trivial_check.py +46 -0
- eif/hypothesis_agenda/__init__.py +27 -0
- eif/hypothesis_agenda/agenda.py +176 -0
- eif/hypothesis_agenda/eif_hypothesis_agenda_intent.yaml +168 -0
- eif/hypothesis_agenda/fdr.py +92 -0
- eif/hypothesis_agenda/scorer.py +132 -0
- eif/input_guard/__init__.py +13 -0
- eif/input_guard/detector.py +259 -0
- eif/input_guard/eif_input_guard_intent.yaml +181 -0
- eif/integration/__init__.py +34 -0
- eif/integration/eif_auto.py +228 -0
- eif/integration/interceptor.py +218 -0
- eif/mcp_server/__init__.py +1 -0
- eif/mcp_server/auth.py +81 -0
- eif/mcp_server/http_server.py +460 -0
- eif/mcp_server/resources.py +113 -0
- eif/mcp_server/server.py +3485 -0
- eif/memory/__init__.py +5 -0
- eif/memory/strategy.py +246 -0
- eif/programme/__init__.py +7 -0
- eif/programme/monitor.py +58 -0
- eif/programme/principle_memory.py +177 -0
- eif/programme/principle_revision.py +412 -0
- eif/programme/signals.py +131 -0
- eif/programme/status.py +56 -0
- eif/py.typed +0 -0
- eif/record/__init__.py +7 -0
- eif/record/chain.py +33 -0
- eif/record/compliance.py +62 -0
- eif/record/isc_disclosure.py +257 -0
- eif/record/outcome_store.py +177 -0
- eif/record/provenance.py +148 -0
- eif/record/research_object.py +412 -0
- eif/replicate/__init__.py +6 -0
- eif/replicate/divergence.py +51 -0
- eif/replicate/eif_independent_replication_intent.yaml +109 -0
- eif/replicate/independent.py +185 -0
- eif/replicate/protocol.py +68 -0
- eif/reporting/__init__.py +1 -0
- eif/reporting/pdf_renderer.py +169 -0
- eif/schemas.py +787 -0
- eif/sdk/__init__.py +36 -0
- eif/sdk/exceptions.py +87 -0
- eif/sdk/interceptor.py +316 -0
- eif/session.py +40 -0
- eif/sycophancy/__init__.py +24 -0
- eif/sycophancy/cost_model.py +215 -0
- eif/sycophancy/detector.py +270 -0
- eif/sycophancy/drift.py +306 -0
- eif/sycophancy/eif_sycophancy_intent.yaml +309 -0
- eif/sycophancy/faithfulness.py +106 -0
- eif/sycophancy/framing.py +191 -0
- eif/update/__init__.py +14 -0
- eif/update/eig.py +29 -0
- eif/update/paradigm.py +121 -0
- eif/update/posterior.py +21 -0
- eif/update/stopping.py +42 -0
- eif_engine-4.1.0.dist-info/METADATA +607 -0
- eif_engine-4.1.0.dist-info/RECORD +111 -0
- eif_engine-4.1.0.dist-info/WHEEL +4 -0
- eif_engine-4.1.0.dist-info/entry_points.txt +3 -0
- eif_engine-4.1.0.dist-info/licenses/LICENSE +21 -0
- eif_engine-4.1.0.dist-info/licenses/NOTICE +17 -0
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
"""CAUSAL_GATE v4 - Causal Evidence Probe (CEP).
|
|
2
|
+
|
|
3
|
+
Fires when a claim is classified at INTERVENTION or COUNTERFACTUAL level
|
|
4
|
+
AND consequence_of_wrong == "HIGH". Searches for independent causal evidence
|
|
5
|
+
(RCTs, meta-analyses, natural experiments) via P3 DDGS and returns a
|
|
6
|
+
structured CausalEvidenceResult.
|
|
7
|
+
|
|
8
|
+
Design: We do NOT build DAGs or run do-calculus. We search for whether the
|
|
9
|
+
causal question has already been answered by the scientific community.
|
|
10
|
+
See: eif/eif/causal_gate/eif_causal_gate_v4_intent.yaml (CG1–CG7)
|
|
11
|
+
|
|
12
|
+
Research basis:
|
|
13
|
+
ERM (arXiv:2602.11675) - Rung Collapse: LLMs confuse P(Y|X) with P(Y|do(X))
|
|
14
|
+
ReCITE (arXiv:2505.18931) - Best LLM F1=0.535 on causal extraction
|
|
15
|
+
Ariadne (arXiv:2601.02314) - Reasoning traces unfaithful to causal logic
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import json
|
|
21
|
+
import logging
|
|
22
|
+
import re
|
|
23
|
+
from collections.abc import Callable
|
|
24
|
+
from datetime import datetime
|
|
25
|
+
|
|
26
|
+
from eif.causal_gate.confound import discover_confounders
|
|
27
|
+
from eif.causal_gate.verdict import posterior_delta, provenance_flag_for
|
|
28
|
+
from eif.schemas import CausalEvidenceResult, CausalEvidenceVerdict, CausalLevel
|
|
29
|
+
|
|
30
|
+
_log = logging.getLogger(__name__)
|
|
31
|
+
|
|
32
|
+
# Trigger conditions for CEP (CG1)
|
|
33
|
+
CEP_TRIGGER_LEVELS: frozenset[str] = frozenset({"INTERVENTION", "COUNTERFACTUAL"})
|
|
34
|
+
CEP_TRIGGER_CONSEQUENCE: str = "HIGH"
|
|
35
|
+
|
|
36
|
+
# Causal evidence search keyword suffix (CG2)
|
|
37
|
+
_CAUSAL_SEARCH_SUFFIX = "RCT OR meta-analysis OR randomized OR natural experiment OR causal"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
41
|
+
# Public trigger guard
|
|
42
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
43
|
+
|
|
44
|
+
def should_probe(causal_level: CausalLevel, consequence: str) -> bool:
|
|
45
|
+
"""Return True if the claim meets the CEP trigger conditions (CG1)."""
|
|
46
|
+
return causal_level in CEP_TRIGGER_LEVELS and consequence == CEP_TRIGGER_CONSEQUENCE
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
50
|
+
# Step 1: extract (cause, effect, domain) from claim text
|
|
51
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
52
|
+
|
|
53
|
+
_EXTRACT_PROMPT = """\
|
|
54
|
+
Given this causal claim: "{claim_text}"
|
|
55
|
+
|
|
56
|
+
Extract exactly three fields:
|
|
57
|
+
- cause: the variable proposed as the cause
|
|
58
|
+
- effect: the variable proposed as the effect
|
|
59
|
+
- domain: the field or sector (e.g. healthcare, engineering, finance, software)
|
|
60
|
+
|
|
61
|
+
Return only valid JSON with keys "cause", "effect", "domain". No commentary.
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def extract_causal_pair(
|
|
66
|
+
claim_text: str,
|
|
67
|
+
llm_fn: Callable[[str], str],
|
|
68
|
+
) -> tuple[str, str, str]:
|
|
69
|
+
"""Extract (cause, effect, domain) from a causal claim via a lightweight LLM call.
|
|
70
|
+
|
|
71
|
+
Args:
|
|
72
|
+
claim_text: raw claim text, e.g. "microservices reduce deployment failures"
|
|
73
|
+
llm_fn: callable(prompt) -> response_text (injectable for testing)
|
|
74
|
+
|
|
75
|
+
Returns:
|
|
76
|
+
(cause, effect, domain) tuple. Falls back to heuristics if LLM fails.
|
|
77
|
+
"""
|
|
78
|
+
prompt = _EXTRACT_PROMPT.format(claim_text=claim_text)
|
|
79
|
+
try:
|
|
80
|
+
raw = llm_fn(prompt)
|
|
81
|
+
# Accept raw JSON or JSON wrapped in markdown fences
|
|
82
|
+
json_text = re.sub(r"```(?:json)?\s*(.*?)\s*```", r"\1", raw, flags=re.DOTALL).strip()
|
|
83
|
+
parsed = json.loads(json_text)
|
|
84
|
+
cause = str(parsed.get("cause", "")).strip()
|
|
85
|
+
effect = str(parsed.get("effect", "")).strip()
|
|
86
|
+
domain = str(parsed.get("domain", "")).strip()
|
|
87
|
+
if cause and effect:
|
|
88
|
+
return cause, effect, domain or "general"
|
|
89
|
+
except Exception: # noqa: BLE001
|
|
90
|
+
_log.debug("extract_causal_pair: LLM parse failed, falling back to heuristic")
|
|
91
|
+
|
|
92
|
+
# Heuristic fallback: use the full claim as cause, "outcome" as effect
|
|
93
|
+
return claim_text, "outcome", "general"
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
97
|
+
# Step 2: formulate search query (CG2)
|
|
98
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
99
|
+
|
|
100
|
+
def formulate_search_query(cause: str, effect: str, domain: str) -> str:
|
|
101
|
+
"""Build a P3 search query targeting causal evidence for the (cause, effect) pair.
|
|
102
|
+
|
|
103
|
+
The suffix ensures results are filtered toward RCTs and meta-analyses (CG2).
|
|
104
|
+
"""
|
|
105
|
+
return f"{cause} {effect} {_CAUSAL_SEARCH_SUFFIX} {domain}".strip()
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
109
|
+
# Step 3: P3 search via DDGS native_search
|
|
110
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
111
|
+
|
|
112
|
+
def _run_search(query: str, search_fn: Callable[[str], str] | None = None) -> str:
|
|
113
|
+
"""Execute the P3 DDGS search. Falls back gracefully to empty string on failure."""
|
|
114
|
+
if search_fn is not None:
|
|
115
|
+
return search_fn(query)
|
|
116
|
+
|
|
117
|
+
# Default: use EIF native_search (DDGS, no API key)
|
|
118
|
+
try:
|
|
119
|
+
from eif.falsify.native_tools import native_search # noqa: PLC0415
|
|
120
|
+
return native_search(query, max_results=5)
|
|
121
|
+
except Exception: # noqa: BLE001
|
|
122
|
+
_log.warning("CEP: native_search unavailable - returning empty results")
|
|
123
|
+
return ""
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
127
|
+
# Step 4: classify search results into CausalEvidenceVerdict (CG3, CG7)
|
|
128
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
129
|
+
|
|
130
|
+
_CLASSIFY_PROMPT = """\
|
|
131
|
+
You are an evidence classifier for a causal inference system.
|
|
132
|
+
|
|
133
|
+
Given these search results about whether "{cause}" causally affects "{effect}":
|
|
134
|
+
|
|
135
|
+
---
|
|
136
|
+
{search_results}
|
|
137
|
+
---
|
|
138
|
+
|
|
139
|
+
Classify the causal evidence into exactly one verdict:
|
|
140
|
+
- SUPPORTED: An RCT, meta-analysis, natural experiment, or Cochrane review was found
|
|
141
|
+
that supports this causal direction
|
|
142
|
+
- CONTESTED: Evidence was found but shows a null effect, conflicting results, or
|
|
143
|
+
disputes the claimed direction
|
|
144
|
+
- NO_EVIDENCE: No causal studies (RCTs, meta-analyses, natural experiments) were found
|
|
145
|
+
for this specific relationship - observational associations alone do not qualify
|
|
146
|
+
- REVERSED: Evidence supports the opposite causal direction
|
|
147
|
+
|
|
148
|
+
Rules:
|
|
149
|
+
1. Only use SUPPORTED if actual causal study methodology is evident (RCT, natural
|
|
150
|
+
experiment, IV, regression discontinuity, etc.) - not mere correlation
|
|
151
|
+
2. Citation must include study name/title and year if verdict is SUPPORTED or REVERSED
|
|
152
|
+
3. If the search results contain only observational data or no relevant studies,
|
|
153
|
+
return NO_EVIDENCE
|
|
154
|
+
|
|
155
|
+
Return only valid JSON: {{"verdict": "...", "citation": "study title (year)" or null}}
|
|
156
|
+
No commentary outside the JSON.
|
|
157
|
+
"""
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def classify_evidence(
|
|
161
|
+
cause: str,
|
|
162
|
+
effect: str,
|
|
163
|
+
search_results: str,
|
|
164
|
+
llm_fn: Callable[[str], str],
|
|
165
|
+
) -> tuple[CausalEvidenceVerdict, str | None]:
|
|
166
|
+
"""Classify P3 search results into a CausalEvidenceVerdict + citation (CG3, CG7).
|
|
167
|
+
|
|
168
|
+
Args:
|
|
169
|
+
cause: extracted cause variable
|
|
170
|
+
effect: extracted effect variable
|
|
171
|
+
search_results: concatenated P3 search output
|
|
172
|
+
llm_fn: callable(prompt) -> response_text
|
|
173
|
+
|
|
174
|
+
Returns:
|
|
175
|
+
(verdict, citation) - citation is None when verdict is NO_EVIDENCE.
|
|
176
|
+
"""
|
|
177
|
+
if not search_results.strip():
|
|
178
|
+
return "NO_EVIDENCE", None
|
|
179
|
+
|
|
180
|
+
prompt = _CLASSIFY_PROMPT.format(
|
|
181
|
+
cause=cause,
|
|
182
|
+
effect=effect,
|
|
183
|
+
search_results=search_results[:3000], # guard against token overflow
|
|
184
|
+
)
|
|
185
|
+
try:
|
|
186
|
+
raw = llm_fn(prompt)
|
|
187
|
+
json_text = re.sub(r"```(?:json)?\s*(.*?)\s*```", r"\1", raw, flags=re.DOTALL).strip()
|
|
188
|
+
parsed = json.loads(json_text)
|
|
189
|
+
raw_verdict = str(parsed.get("verdict", "NO_EVIDENCE")).upper()
|
|
190
|
+
citation = parsed.get("citation") or None
|
|
191
|
+
|
|
192
|
+
valid_verdicts: set[str] = {"SUPPORTED", "CONTESTED", "NO_EVIDENCE", "REVERSED"}
|
|
193
|
+
if raw_verdict not in valid_verdicts:
|
|
194
|
+
raw_verdict = "NO_EVIDENCE"
|
|
195
|
+
|
|
196
|
+
# CG7: all non-neutral verdicts must carry a citation; CONTESTED without
|
|
197
|
+
# a citation is also unreliable (neither side citable → no evidence).
|
|
198
|
+
if raw_verdict in ("SUPPORTED", "REVERSED", "CONTESTED") and not citation:
|
|
199
|
+
_log.warning("CEP: %s verdict has no citation - downgraded to NO_EVIDENCE", raw_verdict)
|
|
200
|
+
raw_verdict = "NO_EVIDENCE"
|
|
201
|
+
|
|
202
|
+
return raw_verdict, citation # type: ignore[return-value]
|
|
203
|
+
|
|
204
|
+
except Exception: # noqa: BLE001
|
|
205
|
+
_log.debug("CEP: classify_evidence LLM parse failed - returning NO_EVIDENCE")
|
|
206
|
+
return "NO_EVIDENCE", None
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
210
|
+
# Main entry point
|
|
211
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
212
|
+
|
|
213
|
+
def run_causal_evidence_probe(
|
|
214
|
+
claim_text: str,
|
|
215
|
+
causal_level: CausalLevel,
|
|
216
|
+
consequence: str,
|
|
217
|
+
llm_fn: Callable[[str], str],
|
|
218
|
+
search_fn: Callable[[str], str] | None = None,
|
|
219
|
+
) -> CausalEvidenceResult | None:
|
|
220
|
+
"""Run the Causal Evidence Probe (CEP) for a single claim.
|
|
221
|
+
|
|
222
|
+
Returns None immediately if the claim does not meet trigger conditions (CG1).
|
|
223
|
+
Returns a CausalEvidenceResult with verdict, citation, posterior_delta, and
|
|
224
|
+
provenance_flag otherwise.
|
|
225
|
+
|
|
226
|
+
Args:
|
|
227
|
+
claim_text: raw claim text from DECLARE
|
|
228
|
+
causal_level: ASSOCIATION / INTERVENTION / COUNTERFACTUAL (from CAUSAL_GATE v3)
|
|
229
|
+
consequence: HIGH / MEDIUM / LOW (from Claim.consequence_of_wrong)
|
|
230
|
+
llm_fn: callable(prompt) -> str - lightweight LLM (e.g. gpt-4.1-mini)
|
|
231
|
+
search_fn: optional override for P3 search (default: DDGS native_search)
|
|
232
|
+
|
|
233
|
+
Raises:
|
|
234
|
+
Nothing - all failures are caught and result in NO_EVIDENCE verdict.
|
|
235
|
+
"""
|
|
236
|
+
if not should_probe(causal_level, consequence):
|
|
237
|
+
return None
|
|
238
|
+
|
|
239
|
+
_log.info("CEP: probing claim=%r level=%s consequence=%s", claim_text, causal_level, consequence)
|
|
240
|
+
|
|
241
|
+
cause, effect, domain = extract_causal_pair(claim_text, llm_fn)
|
|
242
|
+
query = formulate_search_query(cause, effect, domain)
|
|
243
|
+
search_results = _run_search(query, search_fn)
|
|
244
|
+
verdict, citation = classify_evidence(cause, effect, search_results, llm_fn)
|
|
245
|
+
|
|
246
|
+
# IRIS step: discover confounders not already named in the claim
|
|
247
|
+
novel_confounders = discover_confounders(
|
|
248
|
+
cause=cause,
|
|
249
|
+
effect=effect,
|
|
250
|
+
domain=domain,
|
|
251
|
+
hypothesis=claim_text,
|
|
252
|
+
search_fn=search_fn,
|
|
253
|
+
llm_fn=llm_fn,
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
delta = posterior_delta(verdict)
|
|
257
|
+
flag = provenance_flag_for(verdict, consequence)
|
|
258
|
+
|
|
259
|
+
result = CausalEvidenceResult(
|
|
260
|
+
claim_text=claim_text,
|
|
261
|
+
cause=cause,
|
|
262
|
+
effect=effect,
|
|
263
|
+
domain=domain,
|
|
264
|
+
causal_level=causal_level,
|
|
265
|
+
search_query=query,
|
|
266
|
+
verdict=verdict,
|
|
267
|
+
citation=citation,
|
|
268
|
+
evidence_summary=search_results[:500] if search_results else "",
|
|
269
|
+
evidence_source="P3_WEB_SEARCH",
|
|
270
|
+
posterior_delta=delta,
|
|
271
|
+
provenance_flag=flag,
|
|
272
|
+
discovered_confounders=novel_confounders,
|
|
273
|
+
probed_at=datetime.utcnow(),
|
|
274
|
+
)
|
|
275
|
+
|
|
276
|
+
_log.info(
|
|
277
|
+
"CEP result: verdict=%s citation=%r delta=%+.2f flag=%s",
|
|
278
|
+
verdict, citation, delta, flag,
|
|
279
|
+
)
|
|
280
|
+
return result
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""Classify Pearl causal level and detect intervention requirements."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from eif.schemas import CausalLevel
|
|
6
|
+
|
|
7
|
+
_COUNTERFACTUAL_MARKERS = {"would have", "counterfactual", "if x had", "had not", "would not have"}
|
|
8
|
+
_INTERVENTION_MARKERS = {"do(", "causes", "interventions", "intervention", "randomized", "rct", "assign"}
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def classify_causal_level(hypothesis: str) -> CausalLevel:
|
|
12
|
+
"""Classify hypothesis at Pearl's Ladder of Causation.
|
|
13
|
+
|
|
14
|
+
Level 3 COUNTERFACTUAL: "would have", "counterfactual", structural model language
|
|
15
|
+
Level 2 INTERVENTION: "do(", "causes", "interventions", RCT language
|
|
16
|
+
Level 1 ASSOCIATION: everything else (correlation, observational)
|
|
17
|
+
"""
|
|
18
|
+
h_lower = hypothesis.lower()
|
|
19
|
+
|
|
20
|
+
for marker in _COUNTERFACTUAL_MARKERS:
|
|
21
|
+
if marker in h_lower:
|
|
22
|
+
return "COUNTERFACTUAL"
|
|
23
|
+
|
|
24
|
+
for marker in _INTERVENTION_MARKERS:
|
|
25
|
+
if marker in h_lower:
|
|
26
|
+
return "INTERVENTION"
|
|
27
|
+
|
|
28
|
+
return "ASSOCIATION"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def check_intervention(
|
|
32
|
+
hypothesis: str,
|
|
33
|
+
causal_level: CausalLevel,
|
|
34
|
+
evidence_level: CausalLevel = "ASSOCIATION",
|
|
35
|
+
) -> tuple[bool, bool]:
|
|
36
|
+
"""Check if the hypothesis requires do-calculus but evidence is only observational.
|
|
37
|
+
|
|
38
|
+
Returns:
|
|
39
|
+
(intervention_required, disjunctive_bias_flag)
|
|
40
|
+
|
|
41
|
+
intervention_required: True when causal_level > evidence_level
|
|
42
|
+
disjunctive_bias_flag: True when "and" in hypothesis and evidence is ASSOCIATION
|
|
43
|
+
- LLMs systematically treat conjunctive causes as disjunctive (arXiv:2505.09614)
|
|
44
|
+
"""
|
|
45
|
+
level_order = {"ASSOCIATION": 0, "INTERVENTION": 1, "COUNTERFACTUAL": 2}
|
|
46
|
+
|
|
47
|
+
intervention_required = level_order[causal_level] > level_order[evidence_level]
|
|
48
|
+
|
|
49
|
+
h_lower = hypothesis.lower()
|
|
50
|
+
disjunctive_bias_flag = (
|
|
51
|
+
" and " in h_lower
|
|
52
|
+
and evidence_level == "ASSOCIATION"
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
return intervention_required, disjunctive_bias_flag
|
|
@@ -0,0 +1,334 @@
|
|
|
1
|
+
"""IRIS Full Iterative Causal Discovery Pipeline - V2 (F2).
|
|
2
|
+
|
|
3
|
+
V1 discover_confounders() runs a single web query + heuristic extraction.
|
|
4
|
+
V2 implements the full 6-step IRIS loop (arXiv:2510.09217, ACL 2025):
|
|
5
|
+
|
|
6
|
+
Step 1: Document collection - 3-5 native_search queries for (cause, effect, domain).
|
|
7
|
+
Step 2: Statistical causal discovery - extract quantitative claims from documents.
|
|
8
|
+
Step 3: LLM Pearl-ladder classification - classify each claim on the Pearl causal
|
|
9
|
+
hierarchy: ASSOCIATION / INTERVENTION / COUNTERFACTUAL.
|
|
10
|
+
Step 4: Missing variable identification - ask what variables might confound
|
|
11
|
+
cause→effect that are not in the literature.
|
|
12
|
+
Step 5: Causal graph expansion - add missing variables; re-query for each.
|
|
13
|
+
Step 6: Verifiable output - CausalGraph with confidence scores per edge.
|
|
14
|
+
|
|
15
|
+
Max 2 iterations of steps 4–6 (budget control: F2C3).
|
|
16
|
+
|
|
17
|
+
Research: IRIS arXiv:2510.09217 (ACL 2025) - iterative framework for verifiable
|
|
18
|
+
causal discovery in non-tabular data. Statistical extraction → LLM Pearl-ladder
|
|
19
|
+
classification → missing-variable identification → graph expansion.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import json
|
|
25
|
+
import logging
|
|
26
|
+
import re
|
|
27
|
+
from collections.abc import Callable
|
|
28
|
+
|
|
29
|
+
from eif.schemas import CausalEdge, CausalGraph
|
|
30
|
+
|
|
31
|
+
_log = logging.getLogger(__name__)
|
|
32
|
+
|
|
33
|
+
# Causal signal patterns for heuristic extraction (step 2 fallback)
|
|
34
|
+
_CAUSAL_PATTERNS = re.compile(
|
|
35
|
+
r"(?:causes?|leads? to|results? in|increases?|decreases?|reduces?|affects?|"
|
|
36
|
+
r"associated with|correlat(?:ed|es?) with|predicts?|explains?)\s+"
|
|
37
|
+
r"([\w\s]{3,50})",
|
|
38
|
+
re.IGNORECASE,
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
_PEARL_LEVELS = {"ASSOCIATION", "INTERVENTION", "COUNTERFACTUAL"}
|
|
42
|
+
|
|
43
|
+
_CLASSIFY_PROMPT = """\
|
|
44
|
+
Given this causal claim, classify it on the Pearl causal ladder.
|
|
45
|
+
|
|
46
|
+
Claim: "{claim}"
|
|
47
|
+
Cause: "{cause}"
|
|
48
|
+
Effect: "{effect}"
|
|
49
|
+
|
|
50
|
+
Pearl levels:
|
|
51
|
+
- ASSOCIATION: observed correlation; no manipulation; e.g. "X correlates with Y"
|
|
52
|
+
- INTERVENTION: do-calculus; intervention study or experiment; e.g. "giving X reduces Y"
|
|
53
|
+
- COUNTERFACTUAL: what-if reasoning; controls for unobserved; e.g. "had X not occurred, Y would not"
|
|
54
|
+
|
|
55
|
+
Return JSON: {{"level": "ASSOCIATION" | "INTERVENTION" | "COUNTERFACTUAL", "confidence": 0.0–1.0}}
|
|
56
|
+
Return ONLY the JSON, no commentary.
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
_MISSING_VAR_PROMPT = """\
|
|
60
|
+
A researcher claims: "{cause}" causes "{effect}" in the domain of "{domain}".
|
|
61
|
+
|
|
62
|
+
Current known variables in this analysis: {known_vars}
|
|
63
|
+
|
|
64
|
+
What variables might confound this causal relationship that are NOT already listed above?
|
|
65
|
+
Focus on variables that:
|
|
66
|
+
1. Are known risk factors for the effect
|
|
67
|
+
2. Are also correlated with the cause
|
|
68
|
+
3. Are NOT in the current variable list
|
|
69
|
+
|
|
70
|
+
Return JSON array of strings. Maximum 5 items. If none, return [].
|
|
71
|
+
Example: ["socioeconomic status", "baseline health", "age"]
|
|
72
|
+
Return ONLY the JSON array.
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _default_search(query: str, max_results: int = 5) -> str:
|
|
77
|
+
"""Use DDGS native_search as default P3 document retrieval."""
|
|
78
|
+
try:
|
|
79
|
+
from eif.falsify.native_tools import native_search # noqa: PLC0415
|
|
80
|
+
return native_search(query, max_results=max_results)
|
|
81
|
+
except Exception: # noqa: BLE001
|
|
82
|
+
_log.debug("IRIS: native_search unavailable")
|
|
83
|
+
return ""
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _extract_quantitative_claims(text: str, cause: str, effect: str) -> list[str]:
|
|
87
|
+
"""Step 2: Extract quantitative or directional causal claims from text."""
|
|
88
|
+
claims: list[str] = []
|
|
89
|
+
for match in _CAUSAL_PATTERNS.finditer(text):
|
|
90
|
+
phrase = match.group(0).strip()
|
|
91
|
+
if len(phrase) >= 10:
|
|
92
|
+
claims.append(phrase[:200])
|
|
93
|
+
|
|
94
|
+
# Also extract sentences mentioning both cause and effect
|
|
95
|
+
sentences = re.split(r"(?<=[.!?])\s+", text)
|
|
96
|
+
c_low, e_low = cause.lower(), effect.lower()
|
|
97
|
+
for sent in sentences:
|
|
98
|
+
s_low = sent.lower()
|
|
99
|
+
if c_low in s_low and e_low in s_low and 20 <= len(sent) <= 300:
|
|
100
|
+
claims.append(sent.strip())
|
|
101
|
+
|
|
102
|
+
# Deduplicate
|
|
103
|
+
seen: set[str] = set()
|
|
104
|
+
unique: list[str] = []
|
|
105
|
+
for c in claims:
|
|
106
|
+
key = c.lower()[:80]
|
|
107
|
+
if key not in seen:
|
|
108
|
+
seen.add(key)
|
|
109
|
+
unique.append(c)
|
|
110
|
+
|
|
111
|
+
return unique[:10]
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _classify_pearl_level(
|
|
115
|
+
claim: str,
|
|
116
|
+
cause: str,
|
|
117
|
+
effect: str,
|
|
118
|
+
llm_fn: Callable | None,
|
|
119
|
+
) -> tuple[str, float]:
|
|
120
|
+
"""Step 3: Classify a claim on the Pearl causal ladder."""
|
|
121
|
+
if llm_fn is not None:
|
|
122
|
+
prompt = _CLASSIFY_PROMPT.format(claim=claim[:300], cause=cause, effect=effect)
|
|
123
|
+
try:
|
|
124
|
+
raw = llm_fn(prompt, "iris_pearl_classify")
|
|
125
|
+
text = raw if isinstance(raw, str) else json.dumps(raw)
|
|
126
|
+
text = re.sub(r"```(?:json)?\s*(.*?)\s*```", r"\1", text, flags=re.DOTALL).strip()
|
|
127
|
+
parsed = json.loads(text)
|
|
128
|
+
level = str(parsed.get("level", "ASSOCIATION")).upper()
|
|
129
|
+
if level not in _PEARL_LEVELS:
|
|
130
|
+
level = "ASSOCIATION"
|
|
131
|
+
confidence = float(parsed.get("confidence", 0.5))
|
|
132
|
+
return level, max(0.0, min(1.0, confidence))
|
|
133
|
+
except Exception: # noqa: BLE001
|
|
134
|
+
pass
|
|
135
|
+
|
|
136
|
+
# Heuristic fallback
|
|
137
|
+
claim_lower = claim.lower()
|
|
138
|
+
if any(w in claim_lower for w in ["had not", "without", "counterfactual", "what if"]):
|
|
139
|
+
return "COUNTERFACTUAL", 0.55
|
|
140
|
+
if any(w in claim_lower for w in ["randomiz", "experiment", "trial", "intervention", "giving", "administered"]):
|
|
141
|
+
return "INTERVENTION", 0.60
|
|
142
|
+
return "ASSOCIATION", 0.50
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _identify_missing_variables(
|
|
146
|
+
cause: str,
|
|
147
|
+
effect: str,
|
|
148
|
+
domain: str,
|
|
149
|
+
known_vars: list[str],
|
|
150
|
+
llm_fn: Callable | None,
|
|
151
|
+
search_fn: Callable | None,
|
|
152
|
+
) -> list[str]:
|
|
153
|
+
"""Step 4: Identify confounders not in the current variable set."""
|
|
154
|
+
if llm_fn is not None:
|
|
155
|
+
prompt = _MISSING_VAR_PROMPT.format(
|
|
156
|
+
cause=cause,
|
|
157
|
+
effect=effect,
|
|
158
|
+
domain=domain,
|
|
159
|
+
known_vars=", ".join(known_vars) if known_vars else "none",
|
|
160
|
+
)
|
|
161
|
+
try:
|
|
162
|
+
raw = llm_fn(prompt, "iris_missing_vars")
|
|
163
|
+
text = raw if isinstance(raw, str) else json.dumps(raw)
|
|
164
|
+
text = re.sub(r"```(?:json)?\s*(.*?)\s*```", r"\1", text, flags=re.DOTALL).strip()
|
|
165
|
+
parsed = json.loads(text)
|
|
166
|
+
if isinstance(parsed, list):
|
|
167
|
+
return [str(v).strip() for v in parsed if v and str(v).strip() not in known_vars][:5]
|
|
168
|
+
except Exception: # noqa: BLE001
|
|
169
|
+
pass
|
|
170
|
+
|
|
171
|
+
# Heuristic: search for "confounders" and extract
|
|
172
|
+
sf = search_fn or _default_search
|
|
173
|
+
try:
|
|
174
|
+
results = sf(f"{cause} {effect} confounders {domain}")
|
|
175
|
+
from eif.causal_gate.confound import _extract_heuristic # noqa: PLC0415
|
|
176
|
+
candidates = _extract_heuristic(results)
|
|
177
|
+
known_lower = {v.lower() for v in known_vars}
|
|
178
|
+
return [c for c in candidates if c.lower() not in known_lower][:5]
|
|
179
|
+
except Exception: # noqa: BLE001
|
|
180
|
+
return []
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def run(
|
|
184
|
+
cause: str,
|
|
185
|
+
effect: str,
|
|
186
|
+
domain: str = "",
|
|
187
|
+
search_fn: Callable | None = None,
|
|
188
|
+
llm_fn: Callable | None = None,
|
|
189
|
+
max_iterations: int = 2,
|
|
190
|
+
) -> CausalGraph:
|
|
191
|
+
"""Execute the full 6-step IRIS pipeline.
|
|
192
|
+
|
|
193
|
+
Args:
|
|
194
|
+
cause: Cause variable name.
|
|
195
|
+
effect: Effect variable name.
|
|
196
|
+
domain: Domain context (e.g. "healthcare", "finance").
|
|
197
|
+
search_fn: Override for P3 document retrieval; defaults to native_search.
|
|
198
|
+
llm_fn: Optional LLM function for Pearl classification and variable
|
|
199
|
+
identification. When None, heuristic fallbacks are used.
|
|
200
|
+
max_iterations: Maximum causal graph expansion iterations (F2C3: ≤ 2).
|
|
201
|
+
|
|
202
|
+
Returns:
|
|
203
|
+
CausalGraph with edges, missing_variables, and confidence_scores.
|
|
204
|
+
"""
|
|
205
|
+
max_iterations = min(max_iterations, 2) # F2C3: hard cap at 2
|
|
206
|
+
sf = search_fn or _default_search
|
|
207
|
+
domain_str = domain or "general"
|
|
208
|
+
|
|
209
|
+
_log.info("IRIS: starting pipeline for '%s → %s' [domain=%s]", cause, effect, domain_str)
|
|
210
|
+
|
|
211
|
+
# ─ Step 1: Document collection ────────────────────────────────────────────
|
|
212
|
+
queries = [
|
|
213
|
+
f"{cause} {effect} causal relationship {domain_str}",
|
|
214
|
+
f"{cause} causes {effect} evidence {domain_str}",
|
|
215
|
+
f"{effect} risk factors {cause} {domain_str}",
|
|
216
|
+
]
|
|
217
|
+
all_docs = ""
|
|
218
|
+
docs_retrieved = 0
|
|
219
|
+
for q in queries:
|
|
220
|
+
try:
|
|
221
|
+
chunk = sf(q)
|
|
222
|
+
if chunk.strip():
|
|
223
|
+
all_docs += "\n" + chunk
|
|
224
|
+
docs_retrieved += len(chunk.split())
|
|
225
|
+
except Exception: # noqa: BLE001
|
|
226
|
+
pass
|
|
227
|
+
|
|
228
|
+
_log.debug("IRIS step 1: retrieved ~%d words", docs_retrieved)
|
|
229
|
+
|
|
230
|
+
# ─ Step 2: Statistical causal discovery ───────────────────────────────────
|
|
231
|
+
raw_claims = _extract_quantitative_claims(all_docs, cause, effect)
|
|
232
|
+
_log.debug("IRIS step 2: extracted %d causal claims", len(raw_claims))
|
|
233
|
+
|
|
234
|
+
# ─ Step 3: LLM Pearl-ladder classification ────────────────────────────────
|
|
235
|
+
edges: list[CausalEdge] = []
|
|
236
|
+
confidence_scores: dict[str, float] = {}
|
|
237
|
+
for claim in raw_claims:
|
|
238
|
+
level, conf = _classify_pearl_level(claim, cause, effect, llm_fn)
|
|
239
|
+
edge_key = f"{cause}->{effect}:{level}"
|
|
240
|
+
if edge_key not in confidence_scores or conf > confidence_scores[edge_key]:
|
|
241
|
+
confidence_scores[edge_key] = conf
|
|
242
|
+
edges.append(CausalEdge(
|
|
243
|
+
cause=cause,
|
|
244
|
+
effect=effect,
|
|
245
|
+
level=level, # type: ignore[arg-type]
|
|
246
|
+
evidence_count=1,
|
|
247
|
+
confidence=conf,
|
|
248
|
+
))
|
|
249
|
+
|
|
250
|
+
# Consolidate duplicate (cause, effect, level) edges
|
|
251
|
+
consolidated: dict[str, CausalEdge] = {}
|
|
252
|
+
for e in edges:
|
|
253
|
+
key = f"{e.cause}->{e.effect}:{e.level}"
|
|
254
|
+
if key not in consolidated:
|
|
255
|
+
consolidated[key] = e
|
|
256
|
+
else:
|
|
257
|
+
existing = consolidated[key]
|
|
258
|
+
consolidated[key] = CausalEdge(
|
|
259
|
+
cause=e.cause,
|
|
260
|
+
effect=e.effect,
|
|
261
|
+
level=e.level,
|
|
262
|
+
evidence_count=existing.evidence_count + 1,
|
|
263
|
+
confidence=max(existing.confidence, e.confidence),
|
|
264
|
+
)
|
|
265
|
+
edges = list(consolidated.values())
|
|
266
|
+
|
|
267
|
+
# Ensure at least one edge even if no claims were extracted
|
|
268
|
+
if not edges:
|
|
269
|
+
edges.append(CausalEdge(
|
|
270
|
+
cause=cause, effect=effect, level="ASSOCIATION", evidence_count=0, confidence=0.0,
|
|
271
|
+
))
|
|
272
|
+
_log.debug("IRIS step 3: no claims found - inserted zero-confidence ASSOCIATION edge")
|
|
273
|
+
|
|
274
|
+
_log.debug("IRIS step 3: classified into %d distinct edges", len(edges))
|
|
275
|
+
|
|
276
|
+
# ─ Steps 4–6: Iterative graph expansion ───────────────────────────────────
|
|
277
|
+
all_missing: list[str] = []
|
|
278
|
+
known_vars = [cause, effect]
|
|
279
|
+
iterations_run = 0
|
|
280
|
+
|
|
281
|
+
for iteration in range(max_iterations):
|
|
282
|
+
# Step 4: Missing variable identification
|
|
283
|
+
new_vars = _identify_missing_variables(cause, effect, domain_str, known_vars, llm_fn, sf)
|
|
284
|
+
if not new_vars:
|
|
285
|
+
_log.debug("IRIS step 4 (iter %d): no new variables found - stopping early", iteration + 1)
|
|
286
|
+
iterations_run = iteration + 1
|
|
287
|
+
break
|
|
288
|
+
|
|
289
|
+
all_missing.extend(v for v in new_vars if v not in all_missing)
|
|
290
|
+
known_vars.extend(v for v in new_vars if v not in known_vars)
|
|
291
|
+
|
|
292
|
+
# Step 5: Graph expansion - query for each new variable's relationship
|
|
293
|
+
for var in new_vars:
|
|
294
|
+
try:
|
|
295
|
+
var_docs = sf(f"{var} {cause} {effect} confound {domain_str}")
|
|
296
|
+
if var_docs.strip():
|
|
297
|
+
var_claims = _extract_quantitative_claims(var_docs, var, effect)
|
|
298
|
+
for claim in var_claims[:3]:
|
|
299
|
+
level, conf = _classify_pearl_level(claim, var, effect, llm_fn)
|
|
300
|
+
edges.append(CausalEdge(
|
|
301
|
+
cause=var, effect=effect,
|
|
302
|
+
level=level, # type: ignore[arg-type]
|
|
303
|
+
evidence_count=1,
|
|
304
|
+
confidence=conf * 0.8, # discount indirect evidence
|
|
305
|
+
))
|
|
306
|
+
key = f"{var}->{effect}:{level}"
|
|
307
|
+
confidence_scores[key] = conf * 0.8
|
|
308
|
+
except Exception: # noqa: BLE001
|
|
309
|
+
pass
|
|
310
|
+
|
|
311
|
+
iterations_run = iteration + 1
|
|
312
|
+
_log.debug(
|
|
313
|
+
"IRIS step 6 (iter %d): added %d new variables, %d total edges",
|
|
314
|
+
iteration + 1, len(new_vars), len(edges),
|
|
315
|
+
)
|
|
316
|
+
|
|
317
|
+
if iterations_run == 0:
|
|
318
|
+
iterations_run = 1 # Steps 4–6 ran at least once (found nothing)
|
|
319
|
+
|
|
320
|
+
_log.info(
|
|
321
|
+
"IRIS: pipeline complete - %d edges, %d missing vars, %d iteration(s)",
|
|
322
|
+
len(edges), len(all_missing), iterations_run,
|
|
323
|
+
)
|
|
324
|
+
|
|
325
|
+
return CausalGraph(
|
|
326
|
+
cause=cause,
|
|
327
|
+
effect=effect,
|
|
328
|
+
domain=domain_str,
|
|
329
|
+
edges=edges,
|
|
330
|
+
missing_variables=all_missing,
|
|
331
|
+
iterations_run=iterations_run,
|
|
332
|
+
confidence_scores=confidence_scores,
|
|
333
|
+
documents_retrieved=docs_retrieved,
|
|
334
|
+
)
|