eif-engine 4.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. eif/__init__.py +23 -0
  2. eif/auto.py +199 -0
  3. eif/calibrate/__init__.py +15 -0
  4. eif/calibrate/bayesian.py +53 -0
  5. eif/calibrate/conformal.py +57 -0
  6. eif/calibrate/domain_constraints.py +169 -0
  7. eif/calibrate/ece.py +154 -0
  8. eif/calibrate/prior_strategy.py +38 -0
  9. eif/calibrate/trust.py +120 -0
  10. eif/causal_gate/__init__.py +31 -0
  11. eif/causal_gate/confound.py +196 -0
  12. eif/causal_gate/direction.py +28 -0
  13. eif/causal_gate/eif_causal_gate_v4_intent.yaml +265 -0
  14. eif/causal_gate/evidence_probe.py +280 -0
  15. eif/causal_gate/intervention.py +55 -0
  16. eif/causal_gate/iris_pipeline.py +334 -0
  17. eif/causal_gate/verdict.py +47 -0
  18. eif/challenge/__init__.py +17 -0
  19. eif/challenge/diversity.py +58 -0
  20. eif/challenge/eif_multi_critic_intent.yaml +198 -0
  21. eif/challenge/multi_critic.py +270 -0
  22. eif/challenge/protocol.py +86 -0
  23. eif/challenge/replicator.py +379 -0
  24. eif/cost_model/__init__.py +5 -0
  25. eif/cost_model/catch_rate_empirical.py +275 -0
  26. eif/cost_model/cost_ledger.py +107 -0
  27. eif/cost_model/eif_catchrate_intent.yaml +86 -0
  28. eif/declare/__init__.py +6 -0
  29. eif/declare/extractor.py +308 -0
  30. eif/declare/harking_guard.py +110 -0
  31. eif/declare/registry.py +146 -0
  32. eif/eif_engine_intent.yaml +296 -0
  33. eif/eif_v2_intent.yaml +779 -0
  34. eif/eif_v3_intent.yaml +360 -0
  35. eif/explain/__init__.py +7 -0
  36. eif/explain/artifact.py +78 -0
  37. eif/explain/eif_explain_coldstart_intent.yaml +216 -0
  38. eif/explain/hard_to_vary.py +168 -0
  39. eif/explain/reach.py +32 -0
  40. eif/falsify/__init__.py +7 -0
  41. eif/falsify/condition.py +58 -0
  42. eif/falsify/eif_evidence_collection_intent.yaml +310 -0
  43. eif/falsify/eif_native_tools_intent.yaml +128 -0
  44. eif/falsify/eif_v5_evidence_trust_weighting_intent.yaml +273 -0
  45. eif/falsify/evidence_collector.py +1304 -0
  46. eif/falsify/hard_to_vary.py +171 -0
  47. eif/falsify/native_tools.py +221 -0
  48. eif/falsify/sprt.py +84 -0
  49. eif/falsify/trivial_check.py +46 -0
  50. eif/hypothesis_agenda/__init__.py +27 -0
  51. eif/hypothesis_agenda/agenda.py +176 -0
  52. eif/hypothesis_agenda/eif_hypothesis_agenda_intent.yaml +168 -0
  53. eif/hypothesis_agenda/fdr.py +92 -0
  54. eif/hypothesis_agenda/scorer.py +132 -0
  55. eif/input_guard/__init__.py +13 -0
  56. eif/input_guard/detector.py +259 -0
  57. eif/input_guard/eif_input_guard_intent.yaml +181 -0
  58. eif/integration/__init__.py +34 -0
  59. eif/integration/eif_auto.py +228 -0
  60. eif/integration/interceptor.py +218 -0
  61. eif/mcp_server/__init__.py +1 -0
  62. eif/mcp_server/auth.py +81 -0
  63. eif/mcp_server/http_server.py +460 -0
  64. eif/mcp_server/resources.py +113 -0
  65. eif/mcp_server/server.py +3485 -0
  66. eif/memory/__init__.py +5 -0
  67. eif/memory/strategy.py +246 -0
  68. eif/programme/__init__.py +7 -0
  69. eif/programme/monitor.py +58 -0
  70. eif/programme/principle_memory.py +177 -0
  71. eif/programme/principle_revision.py +412 -0
  72. eif/programme/signals.py +131 -0
  73. eif/programme/status.py +56 -0
  74. eif/py.typed +0 -0
  75. eif/record/__init__.py +7 -0
  76. eif/record/chain.py +33 -0
  77. eif/record/compliance.py +62 -0
  78. eif/record/isc_disclosure.py +257 -0
  79. eif/record/outcome_store.py +177 -0
  80. eif/record/provenance.py +148 -0
  81. eif/record/research_object.py +412 -0
  82. eif/replicate/__init__.py +6 -0
  83. eif/replicate/divergence.py +51 -0
  84. eif/replicate/eif_independent_replication_intent.yaml +109 -0
  85. eif/replicate/independent.py +185 -0
  86. eif/replicate/protocol.py +68 -0
  87. eif/reporting/__init__.py +1 -0
  88. eif/reporting/pdf_renderer.py +169 -0
  89. eif/schemas.py +787 -0
  90. eif/sdk/__init__.py +36 -0
  91. eif/sdk/exceptions.py +87 -0
  92. eif/sdk/interceptor.py +316 -0
  93. eif/session.py +40 -0
  94. eif/sycophancy/__init__.py +24 -0
  95. eif/sycophancy/cost_model.py +215 -0
  96. eif/sycophancy/detector.py +270 -0
  97. eif/sycophancy/drift.py +306 -0
  98. eif/sycophancy/eif_sycophancy_intent.yaml +309 -0
  99. eif/sycophancy/faithfulness.py +106 -0
  100. eif/sycophancy/framing.py +191 -0
  101. eif/update/__init__.py +14 -0
  102. eif/update/eig.py +29 -0
  103. eif/update/paradigm.py +121 -0
  104. eif/update/posterior.py +21 -0
  105. eif/update/stopping.py +42 -0
  106. eif_engine-4.1.0.dist-info/METADATA +607 -0
  107. eif_engine-4.1.0.dist-info/RECORD +111 -0
  108. eif_engine-4.1.0.dist-info/WHEEL +4 -0
  109. eif_engine-4.1.0.dist-info/entry_points.txt +3 -0
  110. eif_engine-4.1.0.dist-info/licenses/LICENSE +21 -0
  111. eif_engine-4.1.0.dist-info/licenses/NOTICE +17 -0
eif/__init__.py ADDED
@@ -0,0 +1,23 @@
1
+ """Epistemic Integrity Framework (EIF).
2
+
3
+ Apply the scientific method to every load-bearing assumption an AI agent
4
+ makes: declare, falsify against independent evidence, calibrate a Bayesian
5
+ posterior, challenge adversarially, and route the result to ACT, REVISE,
6
+ or HALT.
7
+
8
+ Public surfaces:
9
+ eif.session - async session store
10
+ eif.schemas - Pydantic v2 models (dependency root)
11
+ eif.mcp_server.server - MCP server exposing the 24 pipeline tools
12
+ eif.sdk - EIFInterceptor / EIFCallbackHandler / EIFHaltError
13
+ eif.auto - zero-config import hook (import eif.auto)
14
+ eif.integration - in-process interceptor (no HTTP server needed)
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ from eif.schemas import ENGINE_VERSION
20
+
21
+ __version__ = ENGINE_VERSION
22
+
23
+ __all__ = ["ENGINE_VERSION", "__version__"]
eif/auto.py ADDED
@@ -0,0 +1,199 @@
1
+ """eif.auto - zero-config EIF interceptor import hook.
2
+
3
+ Add a single line to your agent's entry point::
4
+
5
+ import eif.auto
6
+
7
+ EIF will automatically intercept every OpenAI (and optionally Anthropic) LLM
8
+ response and run eif_verify locally - before the response reaches your code.
9
+
10
+ Pattern inspired by ``import ddtrace.auto`` (Datadog APM zero-config hook).
11
+ Reference: eif_local_interceptor_intent.yaml §solution.B
12
+
13
+ Security model (constraint I1): interception runs entirely on the developer's
14
+ machine. LLM responses never leave their infrastructure. The only outbound
15
+ call is the evidence query (P3 web search, P0 host tool call) - read-only.
16
+
17
+ Configuration via environment variables::
18
+
19
+ EIF_API_KEY Required. Your EIF API key.
20
+ EIF_SERVER_URL Optional. Default: http://localhost:8080
21
+ EIF_HALT_MODE Optional. "raise" (default) | "log" | "callback"
22
+ EIF_LOG_LEVEL Optional. Default: WARNING
23
+
24
+ For more control, use EIFInterceptor or EIFCallbackHandler directly:
25
+ from eif.sdk import EIFInterceptor, EIFCallbackHandler
26
+ """
27
+
28
+ from __future__ import annotations
29
+
30
+ import logging
31
+ import os
32
+ import sys
33
+
34
+ logger = logging.getLogger("eif.auto")
35
+
36
+ _api_key = os.environ.get("EIF_API_KEY", "")
37
+ _server_url = os.environ.get("EIF_SERVER_URL", "http://localhost:8080")
38
+ _halt_mode = os.environ.get("EIF_HALT_MODE", "raise")
39
+
40
+
41
+ def _make_on_halt(mode: str):
42
+ """Build the on_halt callback based on EIF_HALT_MODE."""
43
+ from eif.sdk.exceptions import HaltRecord # noqa: PLC0415
44
+
45
+ if mode == "log":
46
+ def _log_halt(record: HaltRecord) -> None:
47
+ logger.warning("EIF HALT (log mode - not raising): %s", record)
48
+ return _log_halt
49
+
50
+ if mode == "callback":
51
+ # For future extension: users can register a custom callback at
52
+ # eif.auto.on_halt = my_function after importing eif.auto.
53
+ return getattr(sys.modules[__name__], "on_halt", None)
54
+
55
+ return None # "raise" mode: pass None → EIFHaltError raised by default
56
+
57
+
58
+ def _patch_openai() -> bool:
59
+ """Monkey-patch the OpenAI SDK to route all chat completions through EIF.
60
+
61
+ Uses attribute delegation wrapping on the OpenAI class - no private method
62
+ access. Works with openai >= 1.0.
63
+
64
+ Returns True if patching succeeded, False if openai is not installed.
65
+ """
66
+ try:
67
+ import openai # noqa: PLC0415
68
+ except ImportError:
69
+ return False
70
+
71
+ from eif.sdk.interceptor import EIFInterceptor, _EIFWrappedChat # noqa: PLC0415
72
+
73
+ if getattr(openai.OpenAI, "_eif_patched", False):
74
+ return True
75
+ openai.OpenAI._eif_patched = True # type: ignore[attr-defined]
76
+ _original_init = openai.OpenAI.__init__
77
+
78
+ def _patched_init(self, *args, **kwargs): # type: ignore[override]
79
+ _original_init(self, *args, **kwargs)
80
+ # Wrap the chat property with EIF interception
81
+ _interceptor = EIFInterceptor(
82
+ api_key=_api_key,
83
+ client=self,
84
+ eif_server_url=_server_url,
85
+ on_halt=_make_on_halt(_halt_mode),
86
+ )
87
+ # Replace the .chat property with an EIF-wrapped version
88
+ object.__setattr__(self, "_eif_chat_wrapper", _EIFWrappedChat(self.chat, _interceptor))
89
+ _original_chat_property = type(self).chat
90
+
91
+ def _eif_chat_getter(inner_self): # type: ignore[misc]
92
+ return getattr(inner_self, "_eif_chat_wrapper", None) or _original_chat_property.__get__(inner_self)
93
+
94
+ type(self).chat = property(_eif_chat_getter) # type: ignore[assignment]
95
+
96
+ openai.OpenAI.__init__ = _patched_init # type: ignore[method-assign]
97
+ logger.debug("eif.auto: OpenAI SDK patched.")
98
+ return True
99
+
100
+
101
+ def _patch_anthropic() -> bool:
102
+ """Best-effort monkey-patch for the Anthropic SDK.
103
+
104
+ Anthropic does not have a native interceptor API (as of 2026-05).
105
+ This wraps messages.create() via attribute delegation.
106
+ Labeled 'best-effort' per eif_local_interceptor_intent.yaml §OI2:
107
+ not guaranteed across Anthropic SDK version updates.
108
+ Returns True if patching succeeded, False if anthropic is not installed.
109
+ """
110
+ try:
111
+ import anthropic # noqa: PLC0415
112
+ except ImportError:
113
+ return False
114
+
115
+ from eif.sdk.exceptions import EIFHaltError # noqa: PLC0415
116
+ from eif.sdk.interceptor import _make_halt_record, _run_eif_locally # noqa: PLC0415
117
+
118
+ # Anthropic client: .messages.create() is the main entry point
119
+ class _EIFAnthropicMessages:
120
+ def __init__(self, original):
121
+ self._original = original
122
+
123
+ def __getattr__(self, name):
124
+ return getattr(self._original, name)
125
+
126
+ def create(self, *args, **kwargs):
127
+ response = self._original.create(*args, **kwargs)
128
+ text = ""
129
+ try:
130
+ for block in response.content:
131
+ if hasattr(block, "text"):
132
+ text += block.text
133
+ except Exception: # noqa: BLE001
134
+ return response
135
+
136
+ if not text:
137
+ return response
138
+
139
+ eif_result = _run_eif_locally(text, _api_key, _server_url)
140
+ from eif.sdk.interceptor import _EIF_UNAVAILABLE # noqa: PLC0415
141
+ if eif_result is _EIF_UNAVAILABLE:
142
+ logger.warning("EIF local server unreachable - Anthropic response unverified.")
143
+ return response
144
+
145
+ if eif_result.get("verdict") == "HALT":
146
+ record = _make_halt_record(text, eif_result, response)
147
+ on_halt = _make_on_halt(_halt_mode)
148
+ if on_halt:
149
+ on_halt(record)
150
+ else:
151
+ raise EIFHaltError(record)
152
+ return response
153
+
154
+ if getattr(anthropic.Anthropic, "_eif_patched", False):
155
+ return True
156
+ anthropic.Anthropic._eif_patched = True # type: ignore[attr-defined]
157
+ _original_anthropic_init = anthropic.Anthropic.__init__
158
+
159
+ def _patched_anthropic_init(self, *args, **kwargs): # type: ignore[override]
160
+ _original_anthropic_init(self, *args, **kwargs)
161
+ self._eif_messages_wrapper = _EIFAnthropicMessages(self.messages)
162
+ type(self).messages = property(lambda s: s._eif_messages_wrapper) # type: ignore[assignment]
163
+
164
+ anthropic.Anthropic.__init__ = _patched_anthropic_init # type: ignore[method-assign]
165
+ logger.debug("eif.auto: Anthropic SDK patched (best-effort).")
166
+ return True
167
+
168
+
169
+ # ─────────────────────────────────────────────────────────────────────────────
170
+ # Apply patches at import time
171
+ # ─────────────────────────────────────────────────────────────────────────────
172
+
173
+ if not _api_key:
174
+ logger.warning(
175
+ "eif.auto imported but EIF_API_KEY is not set. "
176
+ "Set EIF_API_KEY to match your self-hosted server's key "
177
+ "(or leave the server in development mode and set any value)."
178
+ )
179
+
180
+ _openai_patched = _patch_openai()
181
+ _anthropic_patched = _patch_anthropic()
182
+
183
+ if _openai_patched or _anthropic_patched:
184
+ patched = []
185
+ if _openai_patched:
186
+ patched.append("openai")
187
+ if _anthropic_patched:
188
+ patched.append("anthropic (best-effort)")
189
+ logger.info(
190
+ "eif.auto active - EIF intercepting %s locally. "
191
+ "Server: %s. Halt mode: %s.",
192
+ " + ".join(patched),
193
+ _server_url,
194
+ _halt_mode,
195
+ )
196
+
197
+ # Public hook: override at runtime after `import eif.auto`
198
+ # Example: eif.auto.on_halt = lambda r: print(f"HALT: {r}")
199
+ on_halt = None
@@ -0,0 +1,15 @@
1
+ """EIF Phase 3 - CALIBRATE: Bayesian posterior computation."""
2
+
3
+ from eif.calibrate.bayesian import compute_posterior
4
+ from eif.calibrate.conformal import compute_conformal_coverage
5
+ from eif.calibrate.ece import compute_ece
6
+ from eif.calibrate.prior_strategy import select_prior_strategy
7
+ from eif.calibrate.trust import tier_confidence_likelihood
8
+
9
+ __all__ = [
10
+ "compute_posterior",
11
+ "compute_ece",
12
+ "compute_conformal_coverage",
13
+ "select_prior_strategy",
14
+ "tier_confidence_likelihood",
15
+ ]
@@ -0,0 +1,53 @@
1
+ """Bayesian posterior computation: P(H|E) = P(E|H) * P(H) / P(E).
2
+
3
+ Fallibilism constraint (Deutsch): posteriors are bounded to (ε, 1-ε) so that
4
+ no claim is ever treated as absolutely certain or absolutely impossible.
5
+ Absolute certainty would mean no further evidence could change the belief -
6
+ a violation of scientific fallibilism.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ # Minimum distance from 0 and 1 - Deutsch fallibilism bound
12
+ # A claim with posterior 0.9999 is "very likely" not "proven"; 0.0001 is
13
+ # "very unlikely" not "impossible". Both remain open to revision.
14
+ _FALLIBILISM_EPS: float = 1e-6
15
+
16
+
17
+ def compute_posterior(
18
+ prior: float,
19
+ evidence_supports: bool,
20
+ likelihood_estimate: float | None = None,
21
+ ) -> tuple[float, float]:
22
+ """Compute Bayesian posterior given prior and evidence.
23
+
24
+ P(H|E) = P(E|H) * P(H) / P(E)
25
+
26
+ P(E) is computed via total probability:
27
+ P(E) = P(E|H) * P(H) + P(E|¬H) * P(¬H)
28
+
29
+ Args:
30
+ prior: P(H) - prior probability of the hypothesis
31
+ evidence_supports: True if evidence supports H, False if it contradicts
32
+ likelihood_estimate: P(E|H) - override if known; defaults to 0.8/0.2
33
+
34
+ Returns:
35
+ (posterior, likelihood) tuple
36
+ """
37
+ if likelihood_estimate is not None:
38
+ p_e_given_h = likelihood_estimate
39
+ else:
40
+ p_e_given_h = 0.8 if evidence_supports else 0.2
41
+
42
+ p_e_given_not_h = 1.0 - p_e_given_h
43
+
44
+ p_e = p_e_given_h * prior + p_e_given_not_h * (1.0 - prior)
45
+
46
+ if p_e == 0:
47
+ return prior, p_e_given_h
48
+
49
+ posterior = (p_e_given_h * prior) / p_e
50
+ # Fallibilism bound: never allow exact 0 or 1 (C4: posterior in open interval (0,1))
51
+ posterior = max(_FALLIBILISM_EPS, min(1.0 - _FALLIBILISM_EPS, posterior))
52
+
53
+ return posterior, p_e_given_h
@@ -0,0 +1,57 @@
1
+ """Conformal prediction coverage computation - V2 label-grounded.
2
+
3
+ V1 used confidence_tier (HIGH/MEDIUM/LOW) as a proxy for actual outcomes.
4
+ V2 uses real binary outcomes (outcome_history) when available (F1C1 analogy).
5
+
6
+ Research:
7
+ Conformal prediction arXiv:2107.07511 (Angelopoulos & Bates) -
8
+ P(y_true ∈ C(x)) ≥ 1 − α; distribution-free coverage guarantee.
9
+ Black-box agent conformal arXiv:2602.21368 (2026) -
10
+ self-consistency sampling + conformal calibration for AI agents.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import math
16
+
17
+ from eif.schemas import CONFORMAL_MIN_HISTORY, CalibrationResult
18
+
19
+
20
+ def compute_conformal_coverage(
21
+ history: list[CalibrationResult],
22
+ outcome_history: list[bool | None] | None = None,
23
+ alpha: float = 0.05,
24
+ ) -> float | None:
25
+ """Compute conformal prediction coverage.
26
+
27
+ Algorithm:
28
+ 1. Collect nonconformity scores: score_i = |posterior_i - actual_outcome_i|
29
+ When outcome_history contains non-None values, real binary labels are used;
30
+ otherwise the tier-proxy fallback applies.
31
+ 2. Compute quantile q = sorted_scores[ceil((n+1)(1-alpha))]
32
+ 3. conformal_coverage = 1 - (# scores above q) / n
33
+
34
+ Returns None when fewer than CONFORMAL_MIN_HISTORY calibration points exist.
35
+ """
36
+ if len(history) < CONFORMAL_MIN_HISTORY:
37
+ return None
38
+
39
+ tier_to_outcome = {"HIGH": 1.0, "MEDIUM": 0.5, "LOW": 0.0}
40
+
41
+ scores: list[float] = []
42
+ for i, r in enumerate(history):
43
+ if outcome_history and i < len(outcome_history) and outcome_history[i] is not None:
44
+ actual = 1.0 if outcome_history[i] else 0.0
45
+ else:
46
+ actual = tier_to_outcome.get(r.confidence_tier, 0.5)
47
+ scores.append(abs(r.posterior - actual))
48
+
49
+ n = len(scores)
50
+ sorted_scores = sorted(scores)
51
+
52
+ idx = math.ceil((n + 1) * (1 - alpha)) - 1
53
+ idx = min(idx, n - 1)
54
+ q = sorted_scores[idx]
55
+
56
+ above_q = sum(1 for s in scores if s > q)
57
+ return 1.0 - above_q / n
@@ -0,0 +1,169 @@
1
+ """Domain-specific posterior ceiling table - Physics-Informed Metacognition (F1C3).
2
+
3
+ Implements the Physics-Informed Metacognition principle from
4
+ OpenReview LF4RSTZUtA (2025): embedding domain constraints into the
5
+ metacognitive process yields a 37.2% ECE reduction vs unconstrained models.
6
+
7
+ IMPORTANT - Sourcing methodology
8
+ ─────────────────────────────────
9
+ The ceiling values below are CONSERVATIVE REGULATORY PROXIES, not literal
10
+ threshold numbers quoted from the standards cited. Regulatory frameworks
11
+ constrain acceptable failure rates and require independent validation before
12
+ deployment; these ceilings translate that principle into a Bayesian posterior
13
+ bound:
14
+
15
+ "An AI claim in a high-stakes regulated domain cannot express high
16
+ confidence from parametric evidence alone - independent validation
17
+ evidence is required before reaching high posterior."
18
+
19
+ The approach is grounded in Physics-Informed Metacognition (Roy et al. 2025):
20
+ domain constraints embedded as hard architectural limits outperform
21
+ post-hoc confidence calibration by anchoring the posterior to what is
22
+ epistemically achievable in each domain.
23
+
24
+ A future DomainConstraintTable v2 should source literal performance
25
+ thresholds from regulatory guidance (e.g., FDA SaMD sensitivity/specificity
26
+ requirements, ISO 26262 ASIL D residual risk targets) and map them to
27
+ probability bounds via ROC-AUC or equivalent - see OI2 resolution in
28
+ eif_v2_intent.yaml.
29
+
30
+ Research:
31
+ Roy, M. et al. (2025). Physics-Informed Metacognition: Improving LLMs
32
+ Self-Knowledge via Physical Constraints. OpenReview LF4RSTZUtA.
33
+ Findings: 37.2% ECE reduction from domain-constraint embedding.
34
+ """
35
+
36
+ from __future__ import annotations
37
+
38
+ from dataclasses import dataclass
39
+
40
+
41
+ @dataclass(frozen=True)
42
+ class DomainConstraintEntry:
43
+ """A single domain's posterior ceiling with regulatory provenance."""
44
+
45
+ domain: str
46
+ ceiling: float
47
+ source: str
48
+ note: str
49
+
50
+
51
+ # ─────────────────────────────────────────────────────────────────────────────
52
+ # Domain entries - one per constrained domain
53
+ #
54
+ # ceiling: maximum posterior an AI claim may reach using parametric (P4)
55
+ # evidence alone, before independent validation evidence is present.
56
+ # source: regulatory framework informing the ceiling value.
57
+ # note: rationale connecting the regulatory standard to the ceiling choice.
58
+ # ─────────────────────────────────────────────────────────────────────────────
59
+
60
+ _DOMAIN_ENTRIES: list[DomainConstraintEntry] = [
61
+ DomainConstraintEntry(
62
+ domain="healthcare",
63
+ ceiling=0.95,
64
+ source=(
65
+ "FDA. Artificial Intelligence/Machine Learning (AI/ML)-Based "
66
+ "Software as a Medical Device (SaMD) Action Plan. Jan 2021. "
67
+ "https://www.fda.gov/media/145022/download - "
68
+ "FDA Clinical Decision Support guidance (2022) classifies "
69
+ "high-risk clinical AI as requiring premarket review."
70
+ ),
71
+ note=(
72
+ "Independent clinical validation is mandatory before an AI/ML SaMD "
73
+ "reaches 'substantial evidence' standard. Ceiling 0.95 encodes: "
74
+ "parametric AI evidence alone cannot satisfy FDA's clinical "
75
+ "validation requirement. Any claim with posterior > 0.95 from P4 "
76
+ "alone implies unfounded confidence in a regulated context."
77
+ ),
78
+ ),
79
+ DomainConstraintEntry(
80
+ domain="medical",
81
+ ceiling=0.95,
82
+ source=(
83
+ "FDA AI/ML-Based SaMD Action Plan (Jan 2021) - alias for 'healthcare'."
84
+ ),
85
+ note="Alias for 'healthcare'; same regulatory basis and ceiling apply.",
86
+ ),
87
+ DomainConstraintEntry(
88
+ domain="engineering",
89
+ ceiling=0.92,
90
+ source=(
91
+ "ISO 26262-1:2018 Road Vehicle Functional Safety, §5 (ASIL determination); "
92
+ "ISO 26262-5:2018 Part 5 §8 (hardware target values for ASIL D: "
93
+ "probabilistic metric for random hardware failures PMHF < 1E-8/hr). "
94
+ "IEC 61508:2010 (Functional Safety of E/E/PE Systems) SIL 4: "
95
+ "residual risk < 1E-4 per hour demand rate."
96
+ ),
97
+ note=(
98
+ "ASIL D (highest automotive safety integrity level) and SIL 4 both "
99
+ "require test-validated evidence at every development stage. "
100
+ "Ceiling 0.92 encodes: safety-critical engineering claims require "
101
+ "independent test validation before high posterior is warranted. "
102
+ "The gap from 0.92 to 1.0 is reserved for claims backed by "
103
+ "physical test results (P1/P2 evidence tiers)."
104
+ ),
105
+ ),
106
+ DomainConstraintEntry(
107
+ domain="aviation",
108
+ ceiling=0.90,
109
+ source=(
110
+ "RTCA DO-178C (2011): Software Considerations in Airborne Systems and "
111
+ "Equipment Certification, §6.3 (Independence) and §11.4 "
112
+ "(Software Level A criteria - catastrophic failure condition). "
113
+ "FAA AC 20-115D (2017): Airborne Software Assurance."
114
+ ),
115
+ note=(
116
+ "DO-178C Level A requires independence at every development lifecycle "
117
+ "activity (planning, development, verification). No single-source "
118
+ "claim can achieve full confidence by design. Ceiling 0.90 encodes: "
119
+ "multi-source, independently verified evidence is required before "
120
+ "any aviation safety claim approaches certainty."
121
+ ),
122
+ ),
123
+ DomainConstraintEntry(
124
+ domain="nuclear",
125
+ ceiling=0.90,
126
+ source=(
127
+ "IEC 61513:2011: Nuclear Power Plants - Instrumentation and Control "
128
+ "Important to Safety - General Requirements for Systems, Clause 8 "
129
+ "(Safety I&C system requirements). "
130
+ "IAEA Safety Standards Series SSG-39 (2016): Design of Instrumentation "
131
+ "and Control Systems for Nuclear Power Plants."
132
+ ),
133
+ note=(
134
+ "IEC 61513 Category A qualification requires diverse and redundant "
135
+ "channels with independent verification at each stage. Conservative "
136
+ "treatment mirrors the nuclear safety instrumentation requirement "
137
+ "that no single channel or source constitutes sufficient evidence. "
138
+ "Ceiling 0.90 is shared with aviation - both reflect the highest "
139
+ "available safety integrity category in their respective standards."
140
+ ),
141
+ ),
142
+ ]
143
+
144
+ # ─────────────────────────────────────────────────────────────────────────────
145
+ # Public interfaces
146
+ # ─────────────────────────────────────────────────────────────────────────────
147
+
148
+ DOMAIN_CONSTRAINT_TABLE: dict[str, DomainConstraintEntry] = {
149
+ e.domain.lower(): e for e in _DOMAIN_ENTRIES
150
+ }
151
+
152
+ # Flat ceiling lookup - backward-compatible with ece.py DOMAIN_POSTERIOR_CEILINGS
153
+ DOMAIN_POSTERIOR_CEILINGS: dict[str, float] = {
154
+ e.domain: e.ceiling for e in _DOMAIN_ENTRIES
155
+ }
156
+
157
+
158
+ def get_domain_ceiling(domain: str) -> DomainConstraintEntry | None:
159
+ """Return the DomainConstraintEntry for a domain, or None if unconstrained.
160
+
161
+ Case-insensitive. Returns None for domains not in the constraint table
162
+ (no ceiling applies - general-purpose reasoning domain).
163
+ """
164
+ return DOMAIN_CONSTRAINT_TABLE.get(domain.lower())
165
+
166
+
167
+ def list_constrained_domains() -> list[str]:
168
+ """Return all domain names that have a posterior ceiling defined."""
169
+ return list(DOMAIN_CONSTRAINT_TABLE.keys())
eif/calibrate/ece.py ADDED
@@ -0,0 +1,154 @@
1
+ """Expected Calibration Error (ECE) computation - V2 label-grounded.
2
+
3
+ V1 used confidence_tier (HIGH/MEDIUM/LOW) as a proxy for observed outcomes.
4
+ V2 replaces that proxy with real binary outcomes when ≥ 30 labeled (posterior, outcome)
5
+ pairs are available for the session (F1C1). Below that threshold the tier-proxy is
6
+ retained with an explicit UNCALIBRATED warning (F1C2).
7
+
8
+ Domain-specific posterior ceilings (F1C3 - Physics-Informed Metacognition) are
9
+ defined in domain_constraints.py with full regulatory provenance. Import
10
+ DOMAIN_POSTERIOR_CEILINGS from there rather than duplicating values here.
11
+
12
+ Research:
13
+ HALoGEN arXiv:2501.08292 (ACL 2025 Outstanding Paper) - up to 86% hallucination;
14
+ high confidence does NOT reliably correlate with accuracy without label evidence.
15
+ Conformal prediction arXiv:2107.07511 - standard ECE formula.
16
+ Physics-Informed Metacognition OpenReview LF4RSTZUtA (2025) - 37.2% ECE
17
+ reduction from domain-constraint embedding; basis for apply_domain_ceiling().
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ from eif.calibrate.domain_constraints import DOMAIN_POSTERIOR_CEILINGS
23
+ from eif.schemas import CalibrationResult, ECEResult
24
+
25
+ # Minimum labeled-outcome history to switch from tier-proxy to real ECE (F1C2)
26
+ ECE_LABEL_MIN: int = 30
27
+
28
+
29
+ def _tier_to_outcome(tier: str) -> float:
30
+ return {"HIGH": 1.0, "MEDIUM": 0.5, "LOW": 0.0}.get(tier, 0.5)
31
+
32
+
33
+ def _ece_from_bins(
34
+ posteriors: list[float],
35
+ outcomes: list[float],
36
+ n_bins: int = 10,
37
+ ) -> float:
38
+ """Standard ECE = Σ_b (|B_b|/n) * |acc(B_b) - conf(B_b)|."""
39
+ n = len(posteriors)
40
+ if n == 0:
41
+ return 0.0
42
+
43
+ bin_size = 1.0 / n_bins
44
+ ece = 0.0
45
+ for b in range(n_bins):
46
+ lo, hi = b * bin_size, (b + 1) * bin_size
47
+ indices = [i for i, p in enumerate(posteriors) if lo <= p < hi]
48
+ if not indices:
49
+ continue
50
+ avg_conf = sum(posteriors[i] for i in indices) / len(indices)
51
+ avg_acc = sum(outcomes[i] for i in indices) / len(indices)
52
+ ece += (len(indices) / n) * abs(avg_acc - avg_conf)
53
+
54
+ return round(ece, 6)
55
+
56
+
57
+ def compute_ece(
58
+ history: list[CalibrationResult],
59
+ outcome_history: list[bool | None] | None = None,
60
+ n_bins: int = 10,
61
+ ) -> ECEResult:
62
+ """Compute ECE using real outcomes when available (F1C1/F1C2).
63
+
64
+ Args:
65
+ history: CalibrationResult objects from the session.
66
+ outcome_history: Optional parallel list of bool | None. True = correct,
67
+ False = incorrect, None = not yet observed. When provided
68
+ and ≥ ECE_LABEL_MIN values are non-None, label-grounded
69
+ ECE is computed. Otherwise falls back to tier proxy.
70
+ n_bins: Number of equal-width calibration bins.
71
+
72
+ Returns:
73
+ ECEResult with label_grounded flag and optional calibration_warning.
74
+ """
75
+ if not history:
76
+ return ECEResult(ece=None, label_grounded=False, sessions_used=0)
77
+
78
+ # Collect labeled pairs
79
+ labeled: list[tuple[float, float]] = []
80
+ if outcome_history:
81
+ for r, obs in zip(history, outcome_history, strict=False):
82
+ if obs is not None:
83
+ labeled.append((r.posterior, 1.0 if obs else 0.0))
84
+
85
+ if len(labeled) >= ECE_LABEL_MIN:
86
+ # F1C1 - label-grounded path
87
+ posteriors = [p for p, _ in labeled]
88
+ outcomes = [o for _, o in labeled]
89
+ ece_val = _ece_from_bins(posteriors, outcomes, n_bins)
90
+ return ECEResult(
91
+ ece=ece_val,
92
+ label_grounded=True,
93
+ sessions_used=len(labeled),
94
+ )
95
+
96
+ # F1C2 - tier-proxy fallback with UNCALIBRATED warning
97
+ # Use all calibration history rows for the tier-proxy computation but
98
+ # report sessions_used=len(labeled) so the field consistently reflects
99
+ # "how many real (posterior, outcome) pairs were available" in both modes.
100
+ n_history = len(history)
101
+ posteriors = [r.posterior for r in history]
102
+ outcomes = [_tier_to_outcome(r.confidence_tier) for r in history]
103
+ ece_val = _ece_from_bins(posteriors, outcomes, n_bins)
104
+ warning = (
105
+ "UNCALIBRATED: ECE is a confidence-tier proxy "
106
+ f"({n_history} calibration row(s) used; tier proxy active). "
107
+ "Provide outcome feedback via eif_record_outcome to enable label-grounded calibration "
108
+ f"({len(labeled)}/{ECE_LABEL_MIN} labeled (posterior, outcome) pairs available)."
109
+ )
110
+ return ECEResult(
111
+ ece=ece_val,
112
+ label_grounded=False,
113
+ sessions_used=len(labeled), # R11-07: consistent semantics - labeled pairs, not history rows
114
+ calibration_warning=warning,
115
+ )
116
+
117
+
118
+ def apply_domain_ceiling(
119
+ posterior: float,
120
+ domain: str,
121
+ evidence_tier: str | None = None,
122
+ ) -> tuple[float, bool]:
123
+ """F1C3: Physics-Informed Metacognition - clamp posteriors by domain.
124
+
125
+ Healthcare/engineering domains enforce posterior ceilings unless the
126
+ evidence tier is a DIRECT-OBSERVATION tier - P1 (native code execution) or
127
+ P2 (experiment / RCT-category). Parametric (P4) and web-search (P3) sources
128
+ alone cannot push a regulated-domain claim above its ceiling.
129
+
130
+ TW6 (eif_v5_evidence_trust_weighting): P3 web evidence is now SUBJECT to the
131
+ ceiling. Web text is not direct observation - it is the corpus an attacker can
132
+ poison (PoisonedRAG arXiv:2402.07867) and the noisy channel that degrades RAG
133
+ accuracy (The Power of Noise arXiv:2401.14887). Only direct-observation tiers
134
+ (P1/P2) - and authoritative host tools (P0), which are not regulated-domain
135
+ proxies - may exceed a regulated-domain ceiling. This narrows the prior F1C3
136
+ bypass tuple from (P1, P2, P3) to (P1, P2).
137
+
138
+ Returns:
139
+ (clamped_posterior, domain_clamp_applied)
140
+ """
141
+ ceiling = DOMAIN_POSTERIOR_CEILINGS.get(domain.lower())
142
+ if ceiling is None:
143
+ return posterior, False
144
+
145
+ # Only direct-observation tiers P1 (native code) and P2 (rct/meta-analysis
146
+ # category) may reach the ceiling. P3 (web search) and P4 (parametric LLM)
147
+ # cannot exceed it - TW6.
148
+ if evidence_tier in ("P1_NATIVE_CODE", "P2_EXPERIMENTAL"):
149
+ return posterior, False
150
+
151
+ if posterior > ceiling:
152
+ return ceiling, True
153
+
154
+ return posterior, False