eif-engine 4.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eif/__init__.py +23 -0
- eif/auto.py +199 -0
- eif/calibrate/__init__.py +15 -0
- eif/calibrate/bayesian.py +53 -0
- eif/calibrate/conformal.py +57 -0
- eif/calibrate/domain_constraints.py +169 -0
- eif/calibrate/ece.py +154 -0
- eif/calibrate/prior_strategy.py +38 -0
- eif/calibrate/trust.py +120 -0
- eif/causal_gate/__init__.py +31 -0
- eif/causal_gate/confound.py +196 -0
- eif/causal_gate/direction.py +28 -0
- eif/causal_gate/eif_causal_gate_v4_intent.yaml +265 -0
- eif/causal_gate/evidence_probe.py +280 -0
- eif/causal_gate/intervention.py +55 -0
- eif/causal_gate/iris_pipeline.py +334 -0
- eif/causal_gate/verdict.py +47 -0
- eif/challenge/__init__.py +17 -0
- eif/challenge/diversity.py +58 -0
- eif/challenge/eif_multi_critic_intent.yaml +198 -0
- eif/challenge/multi_critic.py +270 -0
- eif/challenge/protocol.py +86 -0
- eif/challenge/replicator.py +379 -0
- eif/cost_model/__init__.py +5 -0
- eif/cost_model/catch_rate_empirical.py +275 -0
- eif/cost_model/cost_ledger.py +107 -0
- eif/cost_model/eif_catchrate_intent.yaml +86 -0
- eif/declare/__init__.py +6 -0
- eif/declare/extractor.py +308 -0
- eif/declare/harking_guard.py +110 -0
- eif/declare/registry.py +146 -0
- eif/eif_engine_intent.yaml +296 -0
- eif/eif_v2_intent.yaml +779 -0
- eif/eif_v3_intent.yaml +360 -0
- eif/explain/__init__.py +7 -0
- eif/explain/artifact.py +78 -0
- eif/explain/eif_explain_coldstart_intent.yaml +216 -0
- eif/explain/hard_to_vary.py +168 -0
- eif/explain/reach.py +32 -0
- eif/falsify/__init__.py +7 -0
- eif/falsify/condition.py +58 -0
- eif/falsify/eif_evidence_collection_intent.yaml +310 -0
- eif/falsify/eif_native_tools_intent.yaml +128 -0
- eif/falsify/eif_v5_evidence_trust_weighting_intent.yaml +273 -0
- eif/falsify/evidence_collector.py +1304 -0
- eif/falsify/hard_to_vary.py +171 -0
- eif/falsify/native_tools.py +221 -0
- eif/falsify/sprt.py +84 -0
- eif/falsify/trivial_check.py +46 -0
- eif/hypothesis_agenda/__init__.py +27 -0
- eif/hypothesis_agenda/agenda.py +176 -0
- eif/hypothesis_agenda/eif_hypothesis_agenda_intent.yaml +168 -0
- eif/hypothesis_agenda/fdr.py +92 -0
- eif/hypothesis_agenda/scorer.py +132 -0
- eif/input_guard/__init__.py +13 -0
- eif/input_guard/detector.py +259 -0
- eif/input_guard/eif_input_guard_intent.yaml +181 -0
- eif/integration/__init__.py +34 -0
- eif/integration/eif_auto.py +228 -0
- eif/integration/interceptor.py +218 -0
- eif/mcp_server/__init__.py +1 -0
- eif/mcp_server/auth.py +81 -0
- eif/mcp_server/http_server.py +460 -0
- eif/mcp_server/resources.py +113 -0
- eif/mcp_server/server.py +3485 -0
- eif/memory/__init__.py +5 -0
- eif/memory/strategy.py +246 -0
- eif/programme/__init__.py +7 -0
- eif/programme/monitor.py +58 -0
- eif/programme/principle_memory.py +177 -0
- eif/programme/principle_revision.py +412 -0
- eif/programme/signals.py +131 -0
- eif/programme/status.py +56 -0
- eif/py.typed +0 -0
- eif/record/__init__.py +7 -0
- eif/record/chain.py +33 -0
- eif/record/compliance.py +62 -0
- eif/record/isc_disclosure.py +257 -0
- eif/record/outcome_store.py +177 -0
- eif/record/provenance.py +148 -0
- eif/record/research_object.py +412 -0
- eif/replicate/__init__.py +6 -0
- eif/replicate/divergence.py +51 -0
- eif/replicate/eif_independent_replication_intent.yaml +109 -0
- eif/replicate/independent.py +185 -0
- eif/replicate/protocol.py +68 -0
- eif/reporting/__init__.py +1 -0
- eif/reporting/pdf_renderer.py +169 -0
- eif/schemas.py +787 -0
- eif/sdk/__init__.py +36 -0
- eif/sdk/exceptions.py +87 -0
- eif/sdk/interceptor.py +316 -0
- eif/session.py +40 -0
- eif/sycophancy/__init__.py +24 -0
- eif/sycophancy/cost_model.py +215 -0
- eif/sycophancy/detector.py +270 -0
- eif/sycophancy/drift.py +306 -0
- eif/sycophancy/eif_sycophancy_intent.yaml +309 -0
- eif/sycophancy/faithfulness.py +106 -0
- eif/sycophancy/framing.py +191 -0
- eif/update/__init__.py +14 -0
- eif/update/eig.py +29 -0
- eif/update/paradigm.py +121 -0
- eif/update/posterior.py +21 -0
- eif/update/stopping.py +42 -0
- eif_engine-4.1.0.dist-info/METADATA +607 -0
- eif_engine-4.1.0.dist-info/RECORD +111 -0
- eif_engine-4.1.0.dist-info/WHEEL +4 -0
- eif_engine-4.1.0.dist-info/entry_points.txt +3 -0
- eif_engine-4.1.0.dist-info/licenses/LICENSE +21 -0
- eif_engine-4.1.0.dist-info/licenses/NOTICE +17 -0
eif/declare/registry.py
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
"""Build an AssumptionRegistry from a list of ClaimInput objects."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
from datetime import datetime
|
|
7
|
+
|
|
8
|
+
from eif.schemas import (
|
|
9
|
+
EVIDENCE_STALENESS_DAYS,
|
|
10
|
+
AssumptionRegistry,
|
|
11
|
+
Claim,
|
|
12
|
+
ClaimInput,
|
|
13
|
+
ClaimMode,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
_log = logging.getLogger(__name__)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def build_registry(
|
|
20
|
+
session_id: str,
|
|
21
|
+
decision: str,
|
|
22
|
+
claims: list[ClaimInput],
|
|
23
|
+
decision_timestamp: datetime | None = None,
|
|
24
|
+
prior_overrides: dict[str, float] | None = None,
|
|
25
|
+
) -> tuple[AssumptionRegistry, list[str]]:
|
|
26
|
+
"""Convert ClaimInput list into an AssumptionRegistry.
|
|
27
|
+
|
|
28
|
+
Args:
|
|
29
|
+
prior_overrides: optional dict from InputGuardResult.prior_overrides.
|
|
30
|
+
Maps claim_text -> degraded float prior. Claims whose text matches
|
|
31
|
+
a key in prior_overrides are reclassified:
|
|
32
|
+
prior >= 0.70 → KNOWN
|
|
33
|
+
prior >= 0.35 → ASSUMED
|
|
34
|
+
prior < 0.35 → GUESSED
|
|
35
|
+
The sentinel key "__override_turn__" forces all claims to GUESSED.
|
|
36
|
+
|
|
37
|
+
Returns:
|
|
38
|
+
(registry, stale_warnings) where stale_warnings is a list of claim
|
|
39
|
+
texts whose retrieved_at is older than EVIDENCE_STALENESS_DAYS.
|
|
40
|
+
|
|
41
|
+
Exploratory/confirmatory (Lakens 2021; F9 from EIF v2 intent):
|
|
42
|
+
EXPLORATORY claims generate hypotheses; CONFIRMATORY claims test them.
|
|
43
|
+
When a session contains ONLY EXPLORATORY claims, HALT verdicts are
|
|
44
|
+
advisory - confirmatory replication is required before acting.
|
|
45
|
+
build_registry() itself does not produce HALT verdicts, but it sets
|
|
46
|
+
exploration_flag so the downstream caller (eif_declare, eif_verify) can
|
|
47
|
+
surface the warning. Returned via stale_warnings as a distinguished
|
|
48
|
+
"EXPLORATION_ONLY:" prefix to avoid a schema change at this layer.
|
|
49
|
+
"""
|
|
50
|
+
if not decision:
|
|
51
|
+
raise ValueError("decision string cannot be empty")
|
|
52
|
+
|
|
53
|
+
prior_overrides = prior_overrides or {}
|
|
54
|
+
_force_guessed = "__override_turn__" in prior_overrides
|
|
55
|
+
|
|
56
|
+
def _override_type(text: str, original_type: str) -> str:
|
|
57
|
+
if _force_guessed:
|
|
58
|
+
return "GUESSED"
|
|
59
|
+
if text in prior_overrides:
|
|
60
|
+
p = prior_overrides[text]
|
|
61
|
+
if p >= 0.70:
|
|
62
|
+
return "KNOWN"
|
|
63
|
+
if p >= 0.35:
|
|
64
|
+
return "ASSUMED"
|
|
65
|
+
return "GUESSED"
|
|
66
|
+
return original_type
|
|
67
|
+
|
|
68
|
+
known: list[Claim] = []
|
|
69
|
+
assumed: list[Claim] = []
|
|
70
|
+
guessed: list[Claim] = []
|
|
71
|
+
stale_warnings: list[str] = []
|
|
72
|
+
|
|
73
|
+
now = datetime.utcnow()
|
|
74
|
+
|
|
75
|
+
for ci in claims:
|
|
76
|
+
claim = Claim(
|
|
77
|
+
text=ci.text,
|
|
78
|
+
claim_type=ci.claim_type,
|
|
79
|
+
basis=ci.basis,
|
|
80
|
+
evidence_source=ci.evidence_source,
|
|
81
|
+
falsification_condition=ci.falsification_condition,
|
|
82
|
+
consequence_of_wrong=ci.consequence_of_wrong,
|
|
83
|
+
retrieved_at=ci.retrieved_at,
|
|
84
|
+
claim_mode=ci.claim_mode,
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
if claim.claim_type == "KNOWN" and not claim.evidence_source:
|
|
88
|
+
raise ValueError(f"KNOWN claims must have an evidence_source: {claim.text!r}")
|
|
89
|
+
|
|
90
|
+
if claim.retrieved_at is not None:
|
|
91
|
+
age_days = (now - claim.retrieved_at).days
|
|
92
|
+
if age_days > EVIDENCE_STALENESS_DAYS:
|
|
93
|
+
stale_warnings.append(
|
|
94
|
+
f"Claim {claim.text!r} evidence retrieved {age_days} days ago "
|
|
95
|
+
f"(> {EVIDENCE_STALENESS_DAYS})"
|
|
96
|
+
)
|
|
97
|
+
_log.warning("STALE_EVIDENCE: %s", stale_warnings[-1])
|
|
98
|
+
|
|
99
|
+
effective_type = _override_type(claim.text, claim.claim_type)
|
|
100
|
+
if effective_type == "KNOWN":
|
|
101
|
+
known.append(claim)
|
|
102
|
+
elif effective_type == "ASSUMED":
|
|
103
|
+
assumed.append(claim)
|
|
104
|
+
else:
|
|
105
|
+
guessed.append(claim)
|
|
106
|
+
|
|
107
|
+
registry = AssumptionRegistry(
|
|
108
|
+
session_id=session_id,
|
|
109
|
+
decision=decision,
|
|
110
|
+
known=known,
|
|
111
|
+
assumed=assumed,
|
|
112
|
+
guessed=guessed,
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
# Exploratory-only detection (Lakens 2021 / EIF v2 F9).
|
|
116
|
+
all_claims = known + assumed + guessed
|
|
117
|
+
modes: set[ClaimMode] = {c.claim_mode for c in all_claims}
|
|
118
|
+
if modes and modes == {"EXPLORATORY"}:
|
|
119
|
+
stale_warnings.append(
|
|
120
|
+
"EXPLORATION_ONLY: All claims are EXPLORATORY. HALT verdicts are advisory - "
|
|
121
|
+
"confirmatory replication is required before acting on these findings."
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
# F9C2: HARKing risk - any CONFIRMATORY claim without a falsification_condition
|
|
125
|
+
# is a Hypothesizing After Results Known risk (Lakens 2021). Flag it regardless
|
|
126
|
+
# of timestamp so the pipeline can surface the warning.
|
|
127
|
+
harking_flag = any(
|
|
128
|
+
c.claim_mode == "CONFIRMATORY" and not c.falsification_condition
|
|
129
|
+
for c in all_claims
|
|
130
|
+
)
|
|
131
|
+
if harking_flag:
|
|
132
|
+
registry = AssumptionRegistry(
|
|
133
|
+
session_id=registry.session_id,
|
|
134
|
+
decision=registry.decision,
|
|
135
|
+
known=registry.known,
|
|
136
|
+
assumed=registry.assumed,
|
|
137
|
+
guessed=registry.guessed,
|
|
138
|
+
harking_flag=True,
|
|
139
|
+
)
|
|
140
|
+
_log.warning(
|
|
141
|
+
"HARKING_RISK: session=%s has CONFIRMATORY claim(s) without a "
|
|
142
|
+
"pre-registered falsification_condition (F9C2).",
|
|
143
|
+
session_id,
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
return registry, stale_warnings
|
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
# EIF Engine - IVD Intent Artifact
|
|
2
|
+
# ==============================================================================
|
|
3
|
+
# Purpose: Canonical intent for the Epistemic Integrity Framework v1 engine.
|
|
4
|
+
# All implementation decisions must satisfy these constraints.
|
|
5
|
+
# Use this artifact for post-implementation verification (IVD Rule 2)
|
|
6
|
+
# and future change proposals (IVD Rule 1).
|
|
7
|
+
#
|
|
8
|
+
# Audience: AI agents (IVD compliance), human engineers, code reviewers.
|
|
9
|
+
# Origin: EIF design and research corpus (validated, 208 claims, 11 rounds)
|
|
10
|
+
# ==============================================================================
|
|
11
|
+
|
|
12
|
+
id: eif-engine-intent
|
|
13
|
+
version: "1.0"
|
|
14
|
+
created: "2026-05-07"
|
|
15
|
+
|
|
16
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
17
|
+
# PROBLEM STATEMENT
|
|
18
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
19
|
+
|
|
20
|
+
problem:
|
|
21
|
+
title: "AI agent knowledge gaps and unvalidated assumptions produce compounding errors"
|
|
22
|
+
market_cost: "$67.4B annually (PromptFluent 2026)"
|
|
23
|
+
failure_rate: "88% of AI agent projects fail before production"
|
|
24
|
+
root_cause: >
|
|
25
|
+
AI agents treat assumptions as facts, skip falsification, accumulate
|
|
26
|
+
compound errors across multi-step decisions, and produce no auditable
|
|
27
|
+
record of their epistemic state. There is no systematic mechanism to
|
|
28
|
+
apply the scientific method - conjecture, falsification, calibration,
|
|
29
|
+
explanation - to agent reasoning.
|
|
30
|
+
forcing_events:
|
|
31
|
+
- "EU AI Act Article 9/12/13/14 enforcement - August 2026"
|
|
32
|
+
- "Colorado AI Act - June 2026"
|
|
33
|
+
- "LLM sycophancy bias (Perez et al. 2022) measurably degrades output quality"
|
|
34
|
+
|
|
35
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
36
|
+
# SOLUTION SUMMARY
|
|
37
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
38
|
+
|
|
39
|
+
solution:
|
|
40
|
+
name: "Epistemic Integrity Framework (EIF)"
|
|
41
|
+
layer_1: "Cursor rules (.cursor/rules/eif.mdc) - always-on agent rules"
|
|
42
|
+
layer_2: "Python engine (eif/ package) - DECLARE→FALSIFY→CAUSAL_GATE→CALIBRATE→CHALLENGE→UPDATE→EXPLAIN→RECORD (+INPUT_GUARD, +SYCOPHANCY_GATE, +HYPOTHESIS_AGENDA, +REPLICATE, +PROGRAMME)"
|
|
43
|
+
layer_3: "MCP server (eif.mcp_server) - 24 tools + 5 resources; FastMCP"
|
|
44
|
+
scientific_method_mapping:
|
|
45
|
+
DECLARE: "Operationalize: name every assumption before acting"
|
|
46
|
+
FALSIFY: "Falsifiability (Popper): define what observation disproves each claim; SPRT"
|
|
47
|
+
CAUSAL_GATE: "Causal validity (Pearl): reject association-only causal claims"
|
|
48
|
+
CALIBRATE: "Bayesian inference: update belief from evidence; ECE; conformal coverage"
|
|
49
|
+
CHALLENGE: "Adversarial criticism: independent critic challenges surviving claims"
|
|
50
|
+
UPDATE: "Sequential updating: EIG-gated Bayesian posterior; stopping rules"
|
|
51
|
+
EXPLAIN: "Hard-to-vary explanations (Deutsch): every detail must be load-bearing"
|
|
52
|
+
RECORD: "Epistemic provenance (DeepTRACE): immutable audit chain"
|
|
53
|
+
REPLICATE: "Replication crisis reforms (Nosek 2018): pre-registered replication protocols"
|
|
54
|
+
PROGRAMME: "Lakatos programme health: PROGRESSIVE / STABLE / DEGENERATIVE"
|
|
55
|
+
|
|
56
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
57
|
+
# CONSTRAINTS (each has a test - IVD Rule 3)
|
|
58
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
59
|
+
|
|
60
|
+
constraints:
|
|
61
|
+
|
|
62
|
+
# ── Claim classification ────────────────────────────────────────────────────
|
|
63
|
+
|
|
64
|
+
- id: C1
|
|
65
|
+
description: >
|
|
66
|
+
Every claim must be classified as KNOWN, ASSUMED, or GUESSED.
|
|
67
|
+
KNOWN claims must have an evidence_source.
|
|
68
|
+
rationale: "Operationalization principle - Nosek et al. 2018"
|
|
69
|
+
test: >
|
|
70
|
+
Create a Claim with claim_type=KNOWN and no evidence_source.
|
|
71
|
+
Pydantic must raise ValidationError.
|
|
72
|
+
PASS: ValidationError raised. FAIL: silent acceptance.
|
|
73
|
+
|
|
74
|
+
- id: C2
|
|
75
|
+
description: >
|
|
76
|
+
ASSUMED and GUESSED claims with no falsification_condition must be
|
|
77
|
+
returned in eif_declare's falsification_required list.
|
|
78
|
+
rationale: "Popper: an unfalsifiable claim is an assumption, not knowledge"
|
|
79
|
+
test: >
|
|
80
|
+
Call eif_declare with one ASSUMED claim that has no
|
|
81
|
+
falsification_condition. Assert falsification_required contains
|
|
82
|
+
that claim's text.
|
|
83
|
+
|
|
84
|
+
# ── SPRT orientation ────────────────────────────────────────────────────────
|
|
85
|
+
|
|
86
|
+
- id: C3
|
|
87
|
+
description: >
|
|
88
|
+
SPRT uses POPPER orientation: H₀ = claim IS TRUE.
|
|
89
|
+
REJECT H₀ means the claim is FALSIFIED.
|
|
90
|
+
Likelihood ratio Λₖ ≥ B (reject_boundary) triggers REJECT.
|
|
91
|
+
Λₖ ≤ A (accept_boundary) triggers ACCEPT.
|
|
92
|
+
rationale: "Wald 1945; POPPER (arXiv:2502.09858) - falsification-first hypothesis testing"
|
|
93
|
+
test: >
|
|
94
|
+
Call run_sprt with observations=[False, False, False, False, False]
|
|
95
|
+
(every sample refutes H₀ - i.e. each observation is disconfirmatory,
|
|
96
|
+
Bernoulli P(refute)=1). Assert SPRT decision=="REJECT" (claim falsified).
|
|
97
|
+
Call with observations=[True]*10 (all confirmatory). Assert ACCEPT.
|
|
98
|
+
Call with alternating [True, False]*3. Assert CONTINUE (inconclusive).
|
|
99
|
+
With effect_size=0.2, alpha=0.05, beta=0.10 and >= 30 observations,
|
|
100
|
+
decision must be REJECT.
|
|
101
|
+
|
|
102
|
+
# ── Bayesian calibration ────────────────────────────────────────────────────
|
|
103
|
+
|
|
104
|
+
- id: C4
|
|
105
|
+
description: >
|
|
106
|
+
Posterior must be computed using Bayes' theorem: P(H|E) ∝ P(E|H)·P(H).
|
|
107
|
+
Result must be in (0, 1) exclusive.
|
|
108
|
+
rationale: "Bayesian inference; Gelman et al. 2013"
|
|
109
|
+
test: >
|
|
110
|
+
Call compute_posterior(prior=0.5, evidence_supports=True, likelihood_estimate=0.8).
|
|
111
|
+
Assert 0 < posterior < 1 and posterior > 0.5 (positive evidence shifts upward).
|
|
112
|
+
|
|
113
|
+
- id: C5
|
|
114
|
+
description: >
|
|
115
|
+
Prior strategy is auto-selected: if calibration_history < PRIOR_EMPIRICAL_MIN
|
|
116
|
+
(10) use max_entropy; if >= 10 use empirical_bayes; if user supplies prior
|
|
117
|
+
use domain.
|
|
118
|
+
rationale: "Efron & Morris 1977; Casella 1985"
|
|
119
|
+
test: >
|
|
120
|
+
Call select_prior_strategy with history_size=5.
|
|
121
|
+
Assert strategy == "max_entropy".
|
|
122
|
+
Call with history_size=15. Assert strategy == "empirical_bayes".
|
|
123
|
+
|
|
124
|
+
# ── EIG stopping ────────────────────────────────────────────────────────────
|
|
125
|
+
|
|
126
|
+
- id: C6
|
|
127
|
+
description: >
|
|
128
|
+
EIG = KL(posterior ∥ prior). If EIG < EIG_THRESHOLD (0.01 nats),
|
|
129
|
+
stopping_rule_triggered must be True and stopping_reason must be
|
|
130
|
+
EIG_BELOW_THRESHOLD.
|
|
131
|
+
rationale: "Bayesian experimental design; KL divergence stopping (Lindley 1956); LAPD (arXiv:2503.02983)"
|
|
132
|
+
test: >
|
|
133
|
+
Call compute_eig(prior=0.5, posterior=0.501). Assert result < 0.01.
|
|
134
|
+
Call evaluate_stopping with eig=0.001.
|
|
135
|
+
Assert stopping_rule_triggered=True, stopping_reason=EIG_BELOW_THRESHOLD.
|
|
136
|
+
|
|
137
|
+
# ── Hard-to-vary explanations ───────────────────────────────────────────────
|
|
138
|
+
|
|
139
|
+
- id: C7
|
|
140
|
+
description: >
|
|
141
|
+
An ExplanationArtifact with any ExplanationDetail whose prediction_impact
|
|
142
|
+
is empty, generic, or a paraphrase of detail_text must receive
|
|
143
|
+
hard_to_vary_verdict=FAIL.
|
|
144
|
+
rationale: "Deutsch 2011 - hard-to-vary criterion; arXiv:2601.15808"
|
|
145
|
+
test: >
|
|
146
|
+
Build ExplanationArtifact with one detail where prediction_impact=
|
|
147
|
+
"changes the explanation" (generic phrase).
|
|
148
|
+
Assert check_hard_to_vary returns FAIL.
|
|
149
|
+
|
|
150
|
+
# ── Provenance record ───────────────────────────────────────────────────────
|
|
151
|
+
|
|
152
|
+
- id: C8
|
|
153
|
+
description: >
|
|
154
|
+
Every completed decision loop must produce a ProvenanceRecord with a
|
|
155
|
+
unique record_id (UUID4), session_id, timestamp, and registry.
|
|
156
|
+
rationale: "DeepTRACE epistemic provenance; arXiv:2509.04499"
|
|
157
|
+
test: >
|
|
158
|
+
Call eif_record with a valid session, registry, and decision.
|
|
159
|
+
Assert record_id is a valid UUID4 and record appears in
|
|
160
|
+
session provenance_chain.
|
|
161
|
+
|
|
162
|
+
- id: C9
|
|
163
|
+
description: >
|
|
164
|
+
contrary_evidence_considered is True in ProvenanceRecord when EITHER:
|
|
165
|
+
(a) the CHALLENGE phase ran with an independent critic (critic_independence != NONE) -
|
|
166
|
+
the independent critic process itself constitutes considering contrary evidence,
|
|
167
|
+
even when the critic finds no counter-evidence and returns an empty list; OR
|
|
168
|
+
(b) explicit counter-evidence strings were supplied (len(challenge.counter_evidence) > 0) -
|
|
169
|
+
the caller actively gathered opposing evidence, regardless of critic independence.
|
|
170
|
+
False only when challenge is None, or when challenge.critic_independence == NONE AND
|
|
171
|
+
challenge.counter_evidence is empty (no independent process and no gathered opposition).
|
|
172
|
+
rationale: >
|
|
173
|
+
Both pathways satisfy the coverage completeness intent: the agent did not merely
|
|
174
|
+
self-validate. DeepTRACE dimension 7; CONSENSAGENT ACL 2025 §3.
|
|
175
|
+
OR-logic preserves coverage for minimal pipelines where the critic finds nothing
|
|
176
|
+
(a valid result) and for callers who supply their own counter-evidence directly.
|
|
177
|
+
test: >
|
|
178
|
+
Build ProvenanceRecord where challenge.critic_independence=NONE and counter_evidence=[].
|
|
179
|
+
Assert contrary_evidence_considered=False (neither condition met).
|
|
180
|
+
Build one where critic_independence=DIFFERENT_FAMILY and counter_evidence=[].
|
|
181
|
+
Assert contrary_evidence_considered=True (independent critic ran - condition a).
|
|
182
|
+
Build one where critic_independence=NONE and counter_evidence=["evidence A"].
|
|
183
|
+
Assert contrary_evidence_considered=True (counter-evidence supplied - condition b).
|
|
184
|
+
Build one where critic_independence=DIFFERENT_FAMILY and counter_evidence=["evidence A"].
|
|
185
|
+
Assert contrary_evidence_considered=True (both conditions satisfied).
|
|
186
|
+
|
|
187
|
+
# ── Replication ─────────────────────────────────────────────────────────────
|
|
188
|
+
|
|
189
|
+
- id: C10
|
|
190
|
+
description: >
|
|
191
|
+
agreement_rate >= REPLICATE_AGREE_THRESHOLD (0.80) → REPLICATED.
|
|
192
|
+
agreement_rate < REPLICATE_FAIL_THRESHOLD (0.50) → DIVERGED.
|
|
193
|
+
Otherwise → INCONCLUSIVE.
|
|
194
|
+
rationale: "Nosek et al. 2018; arXiv:2602.21368"
|
|
195
|
+
test: >
|
|
196
|
+
Call evaluate_replication(claim_text="X", original_inputs={},
|
|
197
|
+
replication_results=["X"]*4+["Y"], n_replicates=5, replication_criterion="EXACT")
|
|
198
|
+
to produce agreement_rate=0.80 → assert verdict=="REPLICATED".
|
|
199
|
+
Build inputs yielding agreement_rate=0.45 → assert verdict=="DIVERGED".
|
|
200
|
+
Build inputs yielding agreement_rate=0.65 → assert verdict=="INCONCLUSIVE".
|
|
201
|
+
(Implementation is evaluate_replication() in replicate/divergence.py; no
|
|
202
|
+
determine_replication_verdict() exists.)
|
|
203
|
+
|
|
204
|
+
# ── Lakatos programme health ────────────────────────────────────────────────
|
|
205
|
+
|
|
206
|
+
- id: C11
|
|
207
|
+
description: >
|
|
208
|
+
ProgrammeStatus is PROGRESSIVE when novel_prediction_rate > 0.3 and
|
|
209
|
+
confirmed_prediction_rate > 0.3 and patch_rate < 0.4 and oscillation_count <= 1.
|
|
210
|
+
DEGENERATIVE when patch_rate >= 0.6 or oscillation_count >= 3 or
|
|
211
|
+
confirmed_prediction_rate < 0.1 (many predictions, none confirmed).
|
|
212
|
+
STABLE otherwise.
|
|
213
|
+
rationale: >
|
|
214
|
+
Lakatos 1978 - research programme health.
|
|
215
|
+
Thresholds deliberately conservative: patch_rate >= 0.6 (not 0.4) avoids
|
|
216
|
+
misclassifying early-stage sessions with limited history as DEGENERATIVE.
|
|
217
|
+
confirmed_prediction_rate > 0.3 sets a meaningful minimum for PROGRESSIVE.
|
|
218
|
+
test: >
|
|
219
|
+
Build ProgrammeSignals with novel_prediction_rate=0.5,
|
|
220
|
+
confirmed_prediction_rate=0.4, patch_rate=0.1, oscillation_count=0.
|
|
221
|
+
Assert compute_status returns PROGRESSIVE.
|
|
222
|
+
Build with novel_prediction_rate=0.31, confirmed_prediction_rate=0.31,
|
|
223
|
+
patch_rate=0.39, oscillation_count=1. Assert returns PROGRESSIVE (boundary).
|
|
224
|
+
Build with patch_rate=0.6. Assert returns DEGENERATIVE.
|
|
225
|
+
Build with oscillation_count=3. Assert returns DEGENERATIVE.
|
|
226
|
+
Build with confirmed_prediction_rate=0.05. Assert returns DEGENERATIVE.
|
|
227
|
+
|
|
228
|
+
# ── Causal gate ──────────────────────────────────────────────────────────────
|
|
229
|
+
|
|
230
|
+
- id: C12
|
|
231
|
+
description: >
|
|
232
|
+
CausalGateResult must detect disjunctive_bias_flag=True when the
|
|
233
|
+
hypothesis contains " and " conjunctions under ASSOCIATION-level evidence.
|
|
234
|
+
LLMs systematically treat conjunctive causes (A and B) as disjunctive
|
|
235
|
+
(A or B) when evidence is only observational (arXiv:2505.09614).
|
|
236
|
+
rationale: "Pearl 2009; arXiv:2505.09614 - LLM disjunctive bias"
|
|
237
|
+
test: >
|
|
238
|
+
Call check_intervention with hypothesis "A and B causes C" and
|
|
239
|
+
evidence_level="ASSOCIATION".
|
|
240
|
+
Assert disjunctive_bias_flag=True.
|
|
241
|
+
Call with hypothesis "A causes C" (no and).
|
|
242
|
+
Assert disjunctive_bias_flag=False.
|
|
243
|
+
|
|
244
|
+
# ── HARKing guard ────────────────────────────────────────────────────────────
|
|
245
|
+
|
|
246
|
+
- id: C13
|
|
247
|
+
description: >
|
|
248
|
+
harking_flag=True must be set when a claim's falsification_condition
|
|
249
|
+
was registered_at a timestamp after the claim's verified_at timestamp.
|
|
250
|
+
rationale: "HARKing (Kerr 1998) - Hypothesizing After Results are Known"
|
|
251
|
+
test: >
|
|
252
|
+
Build a Claim with verified_at=T1 and a FalsificationCondition with
|
|
253
|
+
registered_at=T2 where T2 > T1. Run detect_harking.
|
|
254
|
+
Assert harking_flag=True on the returned registry.
|
|
255
|
+
|
|
256
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
257
|
+
# POST-IMPLEMENTATION VERIFICATION CHECKLIST (IVD Rule 2)
|
|
258
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
259
|
+
|
|
260
|
+
verification_checklist:
|
|
261
|
+
- "Run: pytest tests/ - all tests pass with 0 failures"
|
|
262
|
+
- "C1: Pydantic validation test for KNOWN claim without evidence_source"
|
|
263
|
+
- "C2: eif_declare returns falsification_required list for untested ASSUMED claims"
|
|
264
|
+
- "C3: SPRT all-false observations → REJECT decision (tested in falsify/sprt.py, not registry.py)"
|
|
265
|
+
- "C4: Bayesian posterior in (0, 1) and directionally correct"
|
|
266
|
+
- "C5: Prior strategy auto-selection: below PRIOR_EMPIRICAL_MIN (10) → max_entropy; at or above → empirical_bayes (examples: history=5 → max_entropy, history=15 → empirical_bayes)"
|
|
267
|
+
- "C6: EIG < 0.01 triggers stopping_rule_triggered=True"
|
|
268
|
+
- "C7: Generic prediction_impact phrase → FAIL hard-to-vary verdict"
|
|
269
|
+
- "C8: ProvenanceRecord has UUID4 record_id and appears in session chain"
|
|
270
|
+
- "C9: contrary_evidence_considered=True when independent critic OR counter_evidence supplied; False when neither"
|
|
271
|
+
- "C10: Replication verdict thresholds correct (0.80→REPLICATED, 0.50→DIVERGED - per REPLICATE_FAIL_THRESHOLD)"
|
|
272
|
+
- "C11: DEGENERATIVE when patch_rate≥0.6 OR oscillation_count≥3 OR confirmed_prediction_rate<0.1 (disjunctive); PROGRESSIVE requires novel>0.3 AND confirmed>0.3 AND patch<0.4 AND oscillation_count≤1 (conjunctive - all four must hold)"
|
|
273
|
+
- "C12: Disjunctive bias detection fires on 'and' hypothesis under ASSOCIATION evidence; absent on simple causal claim"
|
|
274
|
+
- "C13: HARKing flag fires when registered_at > verified_at"
|
|
275
|
+
|
|
276
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
277
|
+
# SCOPE EXCLUSIONS (v1)
|
|
278
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
279
|
+
|
|
280
|
+
out_of_scope_v1:
|
|
281
|
+
- "Active IVD MCP integration (write IVD intent/feedback YAML only)"
|
|
282
|
+
- "External session correlation (linked_session_id field reserved but not wired)"
|
|
283
|
+
- "sentence-transformers semantic diversity (cost_model/compound_error.py stubs it)"
|
|
284
|
+
- "CLI entrypoint beyond basic skeleton"
|
|
285
|
+
- "Production deployment configuration"
|
|
286
|
+
- "Label-grounded ECE (v1 limitation - superseded by V3): ece.py and conformal.py originally used confidence_tier proxies as outcome approximations. EIF V3 introduced eif_record_outcome, OutcomeRecord, and the cross-session outcome_store, which close this gap. Label-grounded ECE now activates at >= 30 labeled outcome records. The v1 approximation note is retained here for historical reference only."
|
|
287
|
+
|
|
288
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
289
|
+
# RELATED ARTIFACTS
|
|
290
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
291
|
+
|
|
292
|
+
related_artifacts:
|
|
293
|
+
architecture: "docs/architecture.md"
|
|
294
|
+
research: "docs/research_foundation.md"
|
|
295
|
+
cursor_rules: ".cursor/rules/eif.mdc"
|
|
296
|
+
ivd_feedback: "the maintainer feedback ledger (private)"
|