xrtm-eval 0.1.0__py3-none-any.whl

xrtm/eval/__init__.py

@@ -0,0 +1,32 @@
+# coding=utf-8
+# Copyright 2026 XRTM Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from xrtm.eval.core.eval.definitions import (
+    EvaluationReport,
+    EvaluationResult,
+    Evaluator,
+)
+from xrtm.eval.kit.eval.metrics import (
+    BrierScoreEvaluator,
+    ExpectedCalibrationErrorEvaluator,
+)
+
+__all__ = [
+    "Evaluator",
+    "EvaluationResult",
+    "EvaluationReport",
+    "BrierScoreEvaluator",
+    "ExpectedCalibrationErrorEvaluator",
+]

xrtm/eval/core/__init__.py

@@ -0,0 +1,14 @@
+# coding=utf-8
+# Copyright 2026 XRTM Team. All rights reserved.
+
+from .epistemics import IntegrityGuardian, SourceTrustEntry, SourceTrustRegistry
+from .eval import EvaluationReport, EvaluationResult, Evaluator
+
+__all__ = [
+    "Evaluator",
+    "EvaluationResult",
+    "EvaluationReport",
+    "IntegrityGuardian",
+    "SourceTrustRegistry",
+    "SourceTrustEntry",
+]

xrtm/eval/core/epistemics.py

@@ -0,0 +1,156 @@
+# coding=utf-8
+# Copyright 2026 XRTM Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from typing import Dict, List, Optional, Set
+
+from pydantic import BaseModel, Field
+
+logger = logging.getLogger(__name__)
+
+__all__ = ["SourceTrustEntry", "SourceTrustRegistry", "IntegrityGuardian"]
+
+
+class SourceTrustEntry(BaseModel):
+    r"""Represents the trust profile for a specific information source or domain."""
+
+    domain: str
+    trust_score: float = Field(ge=0.0, le=1.0, default=0.5)
+    reliability_history: List[float] = Field(default_factory=list)
+    tags: Set[str] = Field(default_factory=set)
+
+
+class SourceTrustRegistry:
+    r"""
+    A registry of verified news and data domains with associated trust scores.
+    Enables the 'Epistemic Security' layer of the platform.
+    """
+
+    def __init__(self, default_trust: float = 0.5):
+        r"""
+        Initializes the registry with a global default trust score.
+
+        Args:
+            default_trust (`float`, *optional*, defaults to `0.5`):
+                The trust score to return for unknown domains.
+
+        Example:
+            ```python
+            >>> registry = SourceTrustRegistry(default_trust=0.2)
+            ```
+        """
+        self._registry: Dict[str, SourceTrustEntry] = {}
+        self.default_trust = default_trust
+
+    def register_source(self, domain: str, trust_score: float, tags: Optional[List[str]] = None):
+        r"""
+        Adds or updates a source in the registry.
+
+        Args:
+            domain (`str`):
+                The domain name to register (e.g., "reputable.org").
+            trust_score (`float`):
+                Trust score between 0.0 and 1.0.
+            tags (`List[str]`, *optional*):
+                Metadata tags for the source.
+
+        Example:
+            ```python
+            >>> registry.register_source("reputable.org", 0.9, tags=["verified"])
+            ```
+        """
+        entry = SourceTrustEntry(domain=domain, trust_score=trust_score, tags=set(tags or []))
+        self._registry[domain] = entry
+        logger.info(f"[TRUST] Registered {domain} with score {trust_score}")
+
+    def get_trust_score(self, domain: str) -> float:
+        r"""
+        Returns the trust score for a domain, or the default if unknown.
+
+        Args:
+            domain (`str`):
+                The domain name to query.
+
+        Returns:
+            `float`: The trust score.
+
+        Example:
+            ```python
+            >>> score = registry.get_trust_score("reputable.org")
+            >>> print(score)
+            0.9
+            ```
+        """
+        # Simple domain matching for now
+        if domain in self._registry:
+            return self._registry[domain].trust_score
+        return self.default_trust
+
+
+class IntegrityGuardian:
+    r"""
+    An automated stage for filtering or flagging data based on source trust.
+    Part of the 'Institutional Epistemics' layer.
+    """
+
+    def __init__(self, registry: SourceTrustRegistry, threshold: float = 0.3):
+        r"""
+        Initializes the guardian with a registry and blocking threshold.
+
+        Args:
+            registry (`SourceTrustRegistry`):
+                The source trust registry to use for domain lookups.
+            threshold (`float`, *optional*, defaults to `0.3`):
+                Any domain with a trust score below this value is blocked.
+
+        Example:
+            ```python
+            >>> guardian = IntegrityGuardian(registry, threshold=0.4)
+            ```
+        """
+        self.registry = registry
+        self.threshold = threshold
+
+    async def validate_data_sources(self, sources: List[str]) -> Dict[str, List[str]]:
+        r"""
+        Checks a list of sources against the trust registry.
+
+        Args:
+            sources (`List[str]`):
+                A list of domain names to validate.
+
+        Returns:
+            `Dict[str, List[str]]`:
+                A dictionary categorizing sources into "passed", "flagged", and "blocked".
+
+        Example:
+            ```python
+            >>> results = await guardian.validate_data_sources(["reputable.org", "shady.com"])
+            >>> print(results["blocked"])
+            ['shady.com']
+            ```
+        """
+        results: Dict[str, List[str]] = {"passed": [], "flagged": [], "blocked": []}
+
+        for src in sources:
+            score = self.registry.get_trust_score(src)
+            if score < self.threshold:
+                results["blocked"].append(src)
+            elif score < 0.5:
+                results["flagged"].append(src)
+            else:
+                results["passed"].append(src)
+
+        return results

xrtm/eval/core/eval/__init__.py

@@ -0,0 +1,10 @@
+# coding=utf-8
+# Copyright 2026 XRTM Team. All rights reserved.
+
+from .definitions import EvaluationReport, EvaluationResult, Evaluator
+
+__all__ = [
+    "Evaluator",
+    "EvaluationResult",
+    "EvaluationReport",
+]

xrtm/eval/core/eval/aggregation.py

@@ -0,0 +1,52 @@
+# coding=utf-8
+# Copyright 2026 XRTM Team. All rights reserved.
+
+from typing import List, Tuple
+
+# From xrtm-data
+from xrtm.data.schemas.forecast import ForecastOutput
+
+
+def inverse_variance_weighting(
+    predictions: List[ForecastOutput], default_variance: float = 0.05
+) -> Tuple[float, float]:
+    if not predictions:
+        return 0.5, 1.0
+
+    values = []
+    weights = []
+
+    for p in predictions:
+        val = p.confidence
+        if p.uncertainty is not None:
+            variance = p.uncertainty
+        else:
+            dist = abs(val - 0.5) * 2
+            variance = 0.25 * (1.0 - dist) + 0.01
+
+        weight = 1.0 / max(variance, 0.01)
+        values.append(val)
+        weights.append(weight)
+
+    sum_weights = sum(weights)
+    if sum_weights == 0:
+        return 0.5, 1.0
+
+    weighted_mean = sum(v * w for v, w in zip(values, weights)) / sum_weights
+    combined_variance = 1.0 / sum_weights
+
+    return weighted_mean, combined_variance
+
+
+def robustness_check_mad(values: List[float], threat_level: float = 2.0) -> List[float]:
+    if len(values) < 3:
+        return values
+
+    median = sorted(values)[len(values) // 2]
+    deviations = [abs(x - median) for x in values]
+    mad = sorted(deviations)[len(deviations) // 2]
+
+    if mad == 0:
+        return values
+
+    return [x for x in values if abs(x - median) <= threat_level * mad]

xrtm/eval/core/eval/bayesian.py

@@ -0,0 +1,29 @@
+# coding=utf-8
+# Copyright 2026 XRTM Team. All rights reserved.
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def probability_to_odds(probability: float) -> float:
+    if probability >= 1.0:
+        return float("inf")
+    if probability <= 0.0:
+        return 0.0
+    return probability / (1.0 - probability)
+
+
+def odds_to_probability(odds: float) -> float:
+    if odds == float("inf"):
+        return 1.0
+    return odds / (1.0 + odds)
+
+
+def bayesian_update(prior_probability: float, bayes_factor: float) -> float:
+    prior_odds = probability_to_odds(prior_probability)
+    posterior_odds = prior_odds * bayes_factor
+    return odds_to_probability(posterior_odds)
+
+
+__all__ = ["probability_to_odds", "odds_to_probability", "bayesian_update"]

xrtm/eval/core/eval/definitions.py

@@ -0,0 +1,63 @@
+# coding=utf-8
+# Copyright 2026 XRTM Team. All rights reserved.
+
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Protocol, Union
+
+from pydantic import BaseModel, Field
+
+
+class EvaluationResult(BaseModel):
+    subject_id: str
+    score: float
+    ground_truth: Any
+    prediction: Any
+    metadata: Dict[str, Any] = Field(default_factory=dict)
+
+
+class ReliabilityBin(BaseModel):
+    bin_center: float
+    mean_prediction: float
+    mean_ground_truth: float
+    count: int
+
+
+class BrierDecomposition(BaseModel):
+    reliability: float
+    resolution: float
+    uncertainty: float
+    score: float
+
+
+class Evaluator(Protocol):
+    def score(self, prediction: Any, ground_truth: Any) -> float:
+        ...
+
+    def evaluate(self, prediction: Any, ground_truth: Any, subject_id: str) -> EvaluationResult:
+        ...
+
+
+class EvaluationReport(BaseModel):
+    metric_name: str
+    mean_score: float
+    total_evaluations: int
+    results: List[EvaluationResult] = Field(default_factory=list)
+    summary_statistics: Dict[str, float] = Field(default_factory=dict)
+    reliability_bins: Optional[List[ReliabilityBin]] = None
+    slices: Optional[Dict[str, "EvaluationReport"]] = Field(
+        default=None, description="Sub-reports grouped by metadata tags"
+    )
+
+    def to_json(self, path: Union[str, Path]) -> None:
+        with open(path, "w") as f:
+            f.write(self.model_dump_json(indent=2))
+
+    def to_pandas(self) -> Any:
+        try:
+            import pandas as pd
+            return pd.DataFrame([r.model_dump() for r in self.results])
+        except ImportError:
+            raise ImportError("Pandas is required for to_pandas(). Install it with `pip install pandas`.")
+
+
+__all__ = ["EvaluationResult", "Evaluator", "EvaluationReport", "ReliabilityBin", "BrierDecomposition"]

xrtm/eval/kit/eval/__init__.py

@@ -0,0 +1,26 @@
+# coding=utf-8
+# Copyright 2026 XRTM Team. All rights reserved.
+
+from xrtm.eval.core.eval.definitions import EvaluationReport, EvaluationResult, Evaluator
+from xrtm.eval.kit.eval.analytics import SliceAnalytics
+from xrtm.eval.kit.eval.bias import BiasInterceptor
+from xrtm.eval.kit.eval.epistemic_evaluator import EpistemicEvaluator
+from xrtm.eval.kit.eval.intervention import InterventionEngine
+from xrtm.eval.kit.eval.metrics import BrierScoreEvaluator
+from xrtm.eval.kit.eval.resilience import AdversarialInjector, FakeNewsItem, GullibilityReport
+from xrtm.eval.kit.eval.viz import ReliabilityDiagram
+
+__all__ = [
+    "Evaluator",
+    "EvaluationResult",
+    "EvaluationReport",
+    "BrierScoreEvaluator",
+    "EpistemicEvaluator",
+    "SliceAnalytics",
+    "BiasInterceptor",
+    "ReliabilityDiagram",
+    "InterventionEngine",
+    "AdversarialInjector",
+    "GullibilityReport",
+    "FakeNewsItem",
+]

xrtm/eval/kit/eval/analytics.py

@@ -0,0 +1,51 @@
+# coding=utf-8
+# Copyright 2026 XRTM Team. All rights reserved.
+
+import logging
+from typing import Dict, List
+
+from xrtm.eval.core.eval.definitions import EvaluationReport, EvaluationResult
+from xrtm.eval.kit.eval.metrics import ExpectedCalibrationErrorEvaluator
+
+__all__ = ["SliceAnalytics"]
+
+logger = logging.getLogger(__name__)
+
+
+class SliceAnalytics:
+    @staticmethod
+    def compute_slices(results: List[EvaluationResult]) -> Dict[str, EvaluationReport]:
+        slices: Dict[str, List[EvaluationResult]] = {}
+        for res in results:
+            tags = res.metadata.get("tags", [])
+            if not tags:
+                continue
+            for tag in tags:
+                key = f"tag:{tag}"
+                if key not in slices:
+                    slices[key] = []
+                slices[key].append(res)
+
+        reports: Dict[str, EvaluationReport] = {}
+        for slice_key, slice_results in slices.items():
+            count = len(slice_results)
+            if count == 0:
+                continue
+            total_score = sum(r.score for r in slice_results)
+            mean_score = total_score / count
+            ece_evaluator = ExpectedCalibrationErrorEvaluator()
+            try:
+                ece_score, bins = ece_evaluator.compute_calibration_data(slice_results)
+            except Exception as e:
+                logger.warning(f"Failed to compute ECE for slice {slice_key}: {e}")
+                ece_score = 0.0
+                bins = None
+            reports[slice_key] = EvaluationReport(
+                metric_name="Slice Brier",
+                mean_score=mean_score,
+                total_evaluations=count,
+                results=slice_results,
+                reliability_bins=bins,
+                summary_statistics={"ece": ece_score},
+            )
+        return reports

xrtm/eval/kit/eval/bias.py

@@ -0,0 +1,49 @@
+# coding=utf-8
+# Copyright 2026 XRTM Team. All rights reserved.
+
+from typing import Any, Dict
+
+from xrtm.eval.core.eval.definitions import EvaluationResult, Evaluator
+
+
+class BiasInterceptor(Evaluator):
+    COGNITIVE_BIASES = [
+        "Base-Rate Neglect", "Overconfidence", "Availability Heuristic",
+        "Confirmation Bias", "Anchoring Bias", "Sunk Cost Fallacy",
+        "Hindsight Bias", "Optimism Bias", "Pessimism Bias",
+        "Status Quo Bias", "Framing Effect", "Recency Bias",
+    ]
+
+    def __init__(self, model: Any):
+        self.model = model
+
+    def score(self, prediction: Any, ground_truth: Any) -> float:
+        return 0.0
+
+    async def evaluate_reasoning(self, reasoning: str) -> Dict[str, Any]:
+        prompt = f"""
+        You are a Cognitive Bias Auditor specialized in probabilistic forecasting.
+        Analyze the following reasoning trace for indicators of any of these biases:
+        {", ".join(self.COGNITIVE_BIASES)}
+
+        Reasoning:
+        "{reasoning}"
+
+        Return a JSON object with:
+        - "detected_biases": [list of bias names]
+        - "severity": [0-10]
+        - "explanation": "Brief rationale"
+        """
+        response = await self.model.generate(prompt)
+        return {"raw_audit": response.text}
+
+    def evaluate(self, prediction: Any, ground_truth: Any, subject_id: str) -> EvaluationResult:
+        return EvaluationResult(
+            subject_id=subject_id,
+            score=0.0,
+            ground_truth=ground_truth,
+            prediction=prediction,
+            metadata={"type": "bias_audit"},
+        )
+
+__all__ = ["BiasInterceptor"]

xrtm/eval/kit/eval/epistemic_evaluator.py

@@ -0,0 +1,31 @@
+# coding=utf-8
+# Copyright 2026 XRTM Team. All rights reserved.
+
+import logging
+from typing import Any, Dict, Optional
+
+# From xrtm-data
+from xrtm.data.schemas.forecast import ForecastOutput
+
+# From xrtm-eval (local)
+from xrtm.eval.core.epistemics import IntegrityGuardian, SourceTrustRegistry
+
+logger = logging.getLogger(__name__)
+
+class EpistemicEvaluator:
+    def __init__(self, registry: Optional[SourceTrustRegistry] = None):
+        self.registry = registry or SourceTrustRegistry()
+        self.guardian = IntegrityGuardian(self.registry)
+
+    async def evaluate_forecast_integrity(self, output: ForecastOutput) -> Dict[str, Any]:
+        sources = output.metadata.get("sources", [])
+        validation = await self.guardian.validate_data_sources(sources)
+        scores = [self.registry.get_trust_score(s) for s in sources]
+        avg_trust = sum(scores) / len(scores) if scores else 0.5
+        return {
+            "aggregate_trust_score": avg_trust,
+            "source_validation": validation,
+            "integrity_level": "HIGH" if avg_trust > 0.8 else "MEDIUM" if avg_trust >= 0.5 else "LOW",
+        }
+
+__all__ = ["EpistemicEvaluator"]

xrtm/eval/kit/eval/intervention.py

@@ -0,0 +1,40 @@
+# coding=utf-8
+# Copyright 2026 XRTM Team. All rights reserved.
+
+import logging
+
+import networkx as nx
+from xrtm.data.schemas.forecast import ForecastOutput
+
+logger = logging.getLogger(__name__)
+
+class InterventionEngine:
+    @staticmethod
+    def apply_intervention(output: ForecastOutput, node_id: str, new_probability: float) -> ForecastOutput:
+        new_output = output.model_copy(deep=True)
+        dg = new_output.to_networkx()
+        if node_id not in dg:
+            raise ValueError(f"Node ID '{node_id}' not found in the causal graph.")
+        for node in new_output.logical_trace:
+            if node.node_id == node_id:
+                node.probability = new_probability
+                break
+        else:
+            raise ValueError(f"Node ID '{node_id}' not found in logical_trace.")
+        nodes_ordered = list(nx.topological_sort(dg))
+        start_index = nodes_ordered.index(node_id)
+        for current_id in nodes_ordered[start_index:]:
+            current_node = next(n for n in new_output.logical_trace if n.node_id == current_id)
+            for _, target_id, data in dg.out_edges(current_id, data=True):
+                weight = data.get("weight", 1.0)
+                target_node = next(n for n in new_output.logical_trace if n.node_id == target_id)
+                old_target_prob = target_node.probability or 0.5
+                normalized_delta = (current_node.probability - (dg.nodes[current_id].get("probability") or 0.5)) * weight
+                target_node.probability = max(0.0, min(1.0, old_target_prob + normalized_delta))
+        leaf_nodes = [n for n in dg.nodes() if dg.out_degree(n) == 0]
+        if leaf_nodes:
+            avg_leaf_prob = sum(next(n.probability for n in new_output.logical_trace if n.node_id == leaf_id) or 0.0 for leaf_id in leaf_nodes) / len(leaf_nodes)
+            new_output.confidence = avg_leaf_prob
+        return new_output
+
+__all__ = ["InterventionEngine"]

xrtm/eval/kit/eval/metrics.py

@@ -0,0 +1,117 @@
+# coding=utf-8
+# Copyright 2026 XRTM Team. All rights reserved.
+
+from typing import Any, List, Tuple, Union
+
+from xrtm.eval.core.eval.definitions import BrierDecomposition, EvaluationResult, Evaluator, ReliabilityBin
+
+
+class BrierScoreEvaluator(Evaluator):
+    def score(self, prediction: Union[float, Any], ground_truth: Union[int, bool, str, Any]) -> float:
+        try:
+            f = float(prediction)
+        except (ValueError, TypeError):
+            raise ValueError(f"Prediction must be convertible to float. Got {prediction}")
+
+        if isinstance(ground_truth, str):
+            o = 1.0 if ground_truth.lower() in ["yes", "1", "true", "won", "pass"] else 0.0
+        else:
+            o = 1.0 if ground_truth else 0.0
+
+        return (f - o) ** 2
+
+    def evaluate(self, prediction: Any, ground_truth: Any, subject_id: str) -> EvaluationResult:
+        s = self.score(prediction, ground_truth)
+        return EvaluationResult(
+            subject_id=subject_id,
+            score=s,
+            ground_truth=ground_truth,
+            prediction=prediction,
+            metadata={"metric": "Brier Score"},
+        )
+
+    def compute_decomposition(self, results: List[EvaluationResult], num_bins: int = 10) -> BrierDecomposition:
+        ece_eval = ExpectedCalibrationErrorEvaluator(num_bins=num_bins)
+        _, bins = ece_eval.compute_calibration_data(results)
+
+        total_count = len(results)
+        if total_count == 0:
+            return BrierDecomposition(reliability=0.0, resolution=0.0, uncertainty=0.0, score=0.0)
+
+        all_outcomes = []
+        for r in results:
+            if isinstance(r.ground_truth, str):
+                o = 1.0 if r.ground_truth.lower() in ["yes", "1", "true", "won", "pass"] else 0.0
+            else:
+                o = 1.0 if r.ground_truth else 0.0
+            all_outcomes.append(o)
+
+        o_bar = sum(all_outcomes) / total_count
+        uncertainty = o_bar * (1.0 - o_bar)
+
+        reliability = 0.0
+        resolution = 0.0
+
+        for b in bins:
+            w_k = b.count / total_count
+            reliability += w_k * (b.mean_prediction - b.mean_ground_truth) ** 2
+            resolution += w_k * (b.mean_ground_truth - o_bar) ** 2
+
+        score = reliability - resolution + uncertainty
+        return BrierDecomposition(reliability=reliability, resolution=resolution, uncertainty=uncertainty, score=score)
+
+
+class ExpectedCalibrationErrorEvaluator(Evaluator):
+    def __init__(self, num_bins: int = 10):
+        self.num_bins = num_bins
+
+    def score(self, prediction: Any, ground_truth: Any) -> float:
+        return BrierScoreEvaluator().score(prediction, ground_truth)
+
+    def evaluate(self, prediction: Any, ground_truth: Any, subject_id: str) -> EvaluationResult:
+        return BrierScoreEvaluator().evaluate(prediction, ground_truth, subject_id)
+
+    def compute_calibration_data(self, results: List[EvaluationResult]) -> Tuple[float, List[ReliabilityBin]]:
+        bin_size = 1.0 / self.num_bins
+        bins: List[List[EvaluationResult]] = [[] for _ in range(self.num_bins)]
+
+        for res in results:
+            try:
+                conf = min(max(float(res.prediction), 0.0), 1.0)
+                idx = int(conf / bin_size)
+                if idx == self.num_bins:
+                    idx -= 1
+                bins[idx].append(res)
+            except (ValueError, TypeError):
+                continue
+
+        total_count = len(results)
+        ece = 0.0
+        reliability_data = []
+
+        for i, bin_items in enumerate(bins):
+            n_b = len(bin_items)
+            bin_center = (i + 0.5) * bin_size
+
+            if n_b > 0:
+                mean_conf = sum(float(x.prediction) for x in bin_items) / n_b
+                accuracies = []
+                for x in bin_items:
+                    gt = x.ground_truth
+                    normalized_gt = 1.0 if (gt.lower() in ["yes", "1", "true", "won", "pass"] if isinstance(gt, str) else gt) else 0.0
+                    accuracies.append(normalized_gt)
+
+                mean_acc = sum(accuracies) / n_b
+                ece += (n_b / total_count) * abs(mean_acc - mean_conf)
+                reliability_data.append(
+                    ReliabilityBin(bin_center=bin_center, mean_prediction=mean_conf, mean_ground_truth=mean_acc, count=n_b)
+                )
+            else:
+                reliability_data.append(
+                    ReliabilityBin(bin_center=bin_center, mean_prediction=0.0, mean_ground_truth=0.0, count=0)
+                )
+
+        return ece, reliability_data
+
+
+__all__ = ["BrierScoreEvaluator", "ExpectedCalibrationErrorEvaluator"]

xrtm/eval/kit/eval/resilience.py

@@ -0,0 +1,32 @@
+# coding=utf-8
+# Copyright 2026 XRTM Team. All rights reserved.
+
+from pydantic import BaseModel, Field
+
+
+class FakeNewsItem(BaseModel):
+    headline: str
+    source_domain: str = "unverified-rumor.xyz"
+    content: str
+    trust_score: float = 0.1
+    intended_bias: str = Field(..., description="e.g. 'Bearish', 'Bullish'")
+
+class GullibilityReport(BaseModel):
+    initial_confidence: float
+    post_injection_confidence: float
+    delta: float
+    resilience_score: float
+
+class AdversarialInjector:
+    def __init__(self, intensity: float = 0.5):
+        self.intensity = intensity
+    def generate_attack(self, subject: str, direction: str) -> FakeNewsItem:
+        if direction.lower() == "bearish":
+            return FakeNewsItem(headline=f"BREAKING: {subject} CEO Under Investigation", content="...", intended_bias="Bearish")
+        return FakeNewsItem(headline=f"{subject} Secures Massive Contract", content="...", intended_bias="Bullish")
+    def measure_resilience(self, initial_confidence: float, post_injection_confidence: float) -> GullibilityReport:
+        delta = post_injection_confidence - initial_confidence
+        score = max(0.0, 1.0 - abs(delta))
+        return GullibilityReport(initial_confidence=initial_confidence, post_injection_confidence=post_injection_confidence, delta=delta, resilience_score=score)
+
+__all__ = ["FakeNewsItem", "GullibilityReport", "AdversarialInjector"]

xrtm/eval/kit/eval/viz.py

@@ -0,0 +1,73 @@
+# coding=utf-8
+# Copyright 2026 XRTM Team. All rights reserved.
+
+import logging
+from dataclasses import dataclass
+from typing import Any, List, Optional
+
+import numpy as np
+
+logger = logging.getLogger(__name__)
+
+@dataclass
+class ReliabilityCurveData:
+    prob_pred: np.ndarray
+    prob_true: np.ndarray
+    ece: float
+
+def compute_calibration_curve(y_true: List[int], y_prob: List[float], n_bins: int = 10) -> ReliabilityCurveData:
+    y_true_arr = np.array(y_true)
+    y_prob_arr = np.array(y_prob)
+    if len(y_prob_arr) == 0:
+        return ReliabilityCurveData(np.array([]), np.array([]), 0.0)
+    bins = np.linspace(0.0, 1.0, n_bins + 1)
+    binids = np.digitize(y_prob_arr, bins) - 1
+    binids = np.clip(binids, 0, n_bins - 1)
+    bin_true, bin_pred, bin_total = [], [], []
+    ece = 0.0
+    total_samples = len(y_prob_arr)
+    for i in range(n_bins):
+        mask = binids == i
+        if not np.any(mask):
+            continue
+        count = np.sum(mask)
+        fraction_true = np.mean(y_true_arr[mask])
+        mean_prob = np.mean(y_prob_arr[mask])
+        bin_true.append(fraction_true)
+        bin_pred.append(mean_prob)
+        bin_total.append(count)
+        ece += (count / total_samples) * np.abs(fraction_true - mean_prob)
+    return ReliabilityCurveData(prob_pred=np.array(bin_pred), prob_true=np.array(bin_true), ece=ece)
+
+def plot_reliability_diagram(data: ReliabilityCurveData, title: str = "Reliability Diagram", save_path: Optional[str] = None) -> Any:
+    try:
+        import matplotlib.pyplot as plt
+        import seaborn as sns
+    except ImportError:
+        logger.error("Visualization libraries not installed.")
+        return None
+    sns.set_theme(style="whitegrid")
+    fig, ax = plt.subplots(figsize=(6, 6))
+    ax.plot([0, 1], [0, 1], "k:", label="Perfectly Calibrated")
+    ax.plot(data.prob_pred, data.prob_true, "s-", label=f"Model (ECE={data.ece:.3f})")
+    ax.set_ylabel("Fraction of Positives")
+    ax.set_xlabel("Mean Predicted Probability")
+    ax.set_ylim((-0.05, 1.05))
+    ax.set_xlim((-0.05, 1.05))
+    ax.set_title(title)
+    ax.legend(loc="lower right")
+    plt.tight_layout()
+    if save_path:
+        plt.savefig(save_path)
+    return fig
+
+class ReliabilityDiagram:
+    def __init__(self, n_bins: int = 10):
+        self.n_bins = n_bins
+    def compute(self, y_true: List[int], y_prob: List[float]) -> ReliabilityCurveData:
+        return compute_calibration_curve(y_true, y_prob, self.n_bins)
+    def plot(self, y_true: List[int], y_prob: List[float], save_path: Optional[str] = None) -> Any:
+        data = self.compute(y_true, y_prob)
+        return plot_reliability_diagram(data, save_path=save_path)
+
+__all__ = ["ReliabilityCurveData", "compute_calibration_curve", "plot_reliability_diagram", "ReliabilityDiagram"]

xrtm/eval/schemas/__init__.py

@@ -0,0 +1,3 @@
+from .forecast import ForecastResolution
+
+__all__ = ["ForecastResolution"]

xrtm/eval/schemas/forecast.py

@@ -0,0 +1,21 @@
+# coding=utf-8
+# Copyright 2026 XRTM Team. All rights reserved.
+
+from datetime import datetime, timezone
+from typing import Any, Dict
+
+from pydantic import BaseModel, Field
+
+
+class ForecastResolution(BaseModel):
+    r"""
+    The ground-truth outcome used to evaluate forecast accuracy.
+    """
+
+    question_id: str
+    outcome: str = Field(..., description="The final winning outcome or value")
+    resolved_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    metadata: Dict[str, Any] = Field(default_factory=dict, description="Source info, verification method")
+
+
+__all__ = ["ForecastResolution"]

xrtm_eval-0.1.0.dist-info/METADATA

@@ -0,0 +1,68 @@
+Metadata-Version: 2.4
+Name: xrtm-eval
+Version: 0.1.0
+Summary: The Judge/Scoring engine for XRTM.
+Author-email: XRTM Team <moy@xrtm.org>
+License: Apache-2.0
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: numpy>=1.24.0
+Requires-Dist: networkx>=3.0
+Requires-Dist: xrtm-data
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+Dynamic: license-file
+
+# xrtm-eval
+
+[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
+[![Python](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
+
+**The Judge for XRTM.**
+
+`xrtm-eval` is the rigorous scoring engine used to grade probabilistic forecasts. It operates independently of the inference engine to ensure objective evaluation.
+
+## Installation
+
+```bash
+uv pip install xrtm-eval
+```
+
+## Core Primitives
+
+### 1. Brier Score Breakdown
+We do not use simple accuracy. We use the **Brier Score**, decomposed into its three component terms:
+
+*   **Reliability**: How well do the predicted probabilities match observed frequencies?
+*   **Resolution**: How well does the forecast distinguish between events that happen and those that don't?
+*   **Uncertainty**: The inherent difficulty of the problem.
+
+```python
+from xrtm.eval import BrierScoreEvaluator
+
+evaluator = BrierScoreEvaluator()
+score = evaluator.score(prediction=0.7, ground_truth=1)
+# score = (0.7 - 1.0)^2 = 0.09
+```
+
+### 2. Expected Calibration Error (ECE)
+Use the `ExpectedCalibrationErrorEvaluator` to measure the gap between confidence and accuracy across bin buckets.
+
+## Development
+
+Prerequisites:
+- [uv](https://github.com/astral-sh/uv)
+
+```bash
+# Install dependencies
+uv sync
+
+# Run tests
+uv run pytest
+```

xrtm_eval-0.1.0.dist-info/RECORD

@@ -0,0 +1,22 @@
+xrtm/eval/__init__.py,sha256=4DLMyE6iTtJmIhlns76iKvBu3NCarF4wyUF-mBkgCes,973
+xrtm/eval/core/__init__.py,sha256=ypetAyWpTtLvQK5sXk5sSdlPKad5zpBwhymUoyoxVl4,366
+xrtm/eval/core/epistemics.py,sha256=3luGGiyWQBbULPkgHANSYBfQhmaJpfv08tJK1hsv30I,5144
+xrtm/eval/core/eval/__init__.py,sha256=LEHGg2YJ8V-ZFZG6C3Ld0PEAFiIBcnsKK8Qgys3krrk,216
+xrtm/eval/core/eval/aggregation.py,sha256=3RHpQ8ruXl6cFjbX_odHAWpLZSbOt7E5tqwvPR9ytRw,1359
+xrtm/eval/core/eval/bayesian.py,sha256=nSlWxcwEGclm4SjCqpzKuTY56DXHbTqV6Aylc4hZRS8,755
+xrtm/eval/core/eval/definitions.py,sha256=p7Sf5JjOlypjglOg_4oOw-1y48s-d-ud8U3Ll6MEE4I,1813
+xrtm/eval/kit/eval/__init__.py,sha256=F4tByHG13YysLVYJshG9gL5zoo5G-tdcy2jsNTGxMpk,906
+xrtm/eval/kit/eval/analytics.py,sha256=ahaGE7l_Lb8LBRWeCyC8oaE-7K8XShT0hwqQwT4CHXY,1818
+xrtm/eval/kit/eval/bias.py,sha256=OQvJzSd_beBzVai-n9fZyvXPwLo78Y794HOkC8eyEiU,1613
+xrtm/eval/kit/eval/epistemic_evaluator.py,sha256=KYqgfxg_MK8qaeZEr6o7sGbP3MBXu5FSbfsLrr-warQ,1154
+xrtm/eval/kit/eval/intervention.py,sha256=ghC9dw8I1DcEf9M0JqO1oxLiTUZd4AOsM8vlX36XlVI,1931
+xrtm/eval/kit/eval/metrics.py,sha256=VKMRwIt9UGJ23uqSUhq_vUSO0LhSe4e2H7S6eAablw4,4596
+xrtm/eval/kit/eval/resilience.py,sha256=q95enu-mglq-S5MSIiO9kDN2Kn9898LvuxNeqo90mKc,1406
+xrtm/eval/kit/eval/viz.py,sha256=LGss9nuGpubJdBbXRlQU2KiGqouTNnWCDyXvKAct3pE,2773
+xrtm/eval/schemas/__init__.py,sha256=6nwuE6LpwosNhFR-xRPPWcGzP1ZxoXhN-vWrzK4XfbQ,75
+xrtm/eval/schemas/forecast.py,sha256=vA5V5RTo7k2ZV3THs2h3fxmdusS06Vw92H5WKJmXVzA,624
+xrtm_eval-0.1.0.dist-info/licenses/LICENSE,sha256=BexUTTsX5WlzyJ0Tqajo1h_LFYfCtfFgWdRaGltpm5I,11328
+xrtm_eval-0.1.0.dist-info/METADATA,sha256=1eUelsA8-xB9ijhPxR2wI_6syH8J7jzyCQyFXISYOck,1996
+xrtm_eval-0.1.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+xrtm_eval-0.1.0.dist-info/top_level.txt,sha256=Jz-i0a9P8GVrIR9KJTT-9wT95E1brww6U5o2QViAt20,5
+xrtm_eval-0.1.0.dist-info/RECORD,,

xrtm_eval-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.10.2)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

xrtm_eval-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that you changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tour (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2026 XRTM Team
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

xrtm_eval-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+xrtm