ioc-scope-mce-core 2.0.0__py3-none-any.whl

ioc_scope_mce_core-2.0.0.dist-info/METADATA

@@ -0,0 +1,47 @@
+Metadata-Version: 2.4
+Name: ioc-scope-mce-core
+Version: 2.0.0
+Summary: Metrics Computation Engine core
+Author-email: Outshift Open <oss-github@cisco.com>
+License-Expression: Apache-2.0
+Requires-Python: <3.15,>=3.13
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: click>=8.0.0
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: requests>=2.28.0
+Requires-Dist: networkx>=3.0.0
+Requires-Dist: packaging>=23.0
+Requires-Dist: pandas>=1.5.0
+Requires-Dist: pydantic-settings>=2.10.1
+Requires-Dist: typer>=0.9.0
+Requires-Dist: rdflib>=7.0.0
+Requires-Dist: pyyaml>=6.0.0
+Requires-Dist: ioc-scope-ontology>=1.1.0
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-asyncio; extra == "dev"
+Requires-Dist: black; extra == "dev"
+Requires-Dist: flake8; extra == "dev"
+Requires-Dist: mypy; extra == "dev"
+Requires-Dist: httpx; extra == "dev"
+Provides-Extra: neo4j
+Requires-Dist: neo4j>=6.1.0; extra == "neo4j"
+Provides-Extra: deepeval
+Requires-Dist: ioc-scope-mce-provider-deepeval>=2.0.0; extra == "deepeval"
+Provides-Extra: llm
+Requires-Dist: ioc-scope-mce-provider-native>=2.0.0; extra == "llm"
+Provides-Extra: opik
+Requires-Dist: ioc-scope-mce-provider-opik>=2.0.0; extra == "opik"
+Provides-Extra: ragas
+Requires-Dist: ioc-scope-mce-provider-ragas>=2.0.0; extra == "ragas"
+Provides-Extra: server
+Requires-Dist: fastapi>=0.111.0; extra == "server"
+Requires-Dist: uvicorn>=0.30.0; extra == "server"
+Provides-Extra: all
+Requires-Dist: ioc-scope-mce-provider-native>=2.0.0; extra == "all"
+Requires-Dist: ioc-scope-mce-provider-sdk>=2.0.0; extra == "all"
+Requires-Dist: ioc-scope-mce-provider-deepeval>=2.0.0; extra == "all"
+Requires-Dist: ioc-scope-mce-provider-opik>=2.0.0; extra == "all"
+Requires-Dist: ioc-scope-mce-provider-ragas>=2.0.0; extra == "all"
+Requires-Dist: fastapi>=0.111.0; extra == "all"
+Requires-Dist: uvicorn>=0.30.0; extra == "all"

ioc_scope_mce_core-2.0.0.dist-info/RECORD

@@ -0,0 +1,74 @@
+mce/__init__.py,sha256=wMRx_jlzot3wmSSQ9yR078H8XKg9UGEAwrJp5wSVQ9I,886
+mce/core/catalog.py,sha256=9BPycJRunRj6LCeeCiKgCsF3XuzcKowcXYWKhgARGQQ,8282
+mce/core/config.py,sha256=XNznNiy1WdQmXz-TZhyFRiE5sc1fQa-iJdefESUW814,1044
+mce/core/helpers.py,sha256=d2k7upydKMGJjsyHH6KCTkjGbWNTyER5TjV4CzSnFgo,6430
+mce/core/metadata.py,sha256=JVxS39FSGnVKul0CUOrSwKZHqCbI95dAwKkYPJY0wEE,5859
+mce/core/metric.py,sha256=G1l1WUn_-4IP43Yrw66wcUjGTbqokjStG_sAbcuJEhY,5659
+mce/core/provider.py,sha256=9rVSq1eXgvyFEB6QWsIPS63OxU8b7-h-wcHOtGbN_CA,1373
+mce/core/types.py,sha256=otm3X1gIhO3vF975LdpKSBAPf-qlvqR5c44aMe5kfYQ,3086
+mce/core/registry/__init__.py,sha256=o1S_LhJfgMhgxmWXaECJclMtgpVzaro28YQv9DgTaD0,2914
+mce/core/registry/discovery.py,sha256=zo6t4Xndg8GhSepkAVkYzfV_GJw57s1MfDn1--Q80e8,27387
+mce/core/registry/provider.py,sha256=5QIq93ghTYHRWGS6C9xcOc7wJJx7b1lq7EH9S6h_JRw,1261
+mce/core/registry/service.py,sha256=SBQgORDsg1qZkPI9J810tvRtp3_lVpkLPoGZ6cC6beQ,2888
+mce/core/specs/__init__.py,sha256=wxxV-Uv1ljibU1RrYFjZScqE2UVLXcRRe7NM1POWTqE,563
+mce/core/specs/_registry.py,sha256=jI3qCQ2g4hn-qXBgjgZMNG658kGIi1uFWUOkAS-1Ao8,8076
+mce/core/specs/agent_to_agent_interactions.py,sha256=JeQXwh9PQ_HyMapxspmss-Hye-ktVDTt0bpdGabH7-g,1638
+mce/core/specs/agent_to_tool_interactions.py,sha256=yWsi7UXJ5V6f-4Fk8T8Qlv0WOjtNUI_M7kYB8DdQyds,521
+mce/core/specs/answer_correctness.py,sha256=3dT-wmYvX9s_0pnAKkXpHifosVg9FxahH-FLy930vDE,712
+mce/core/specs/answer_relevancy.py,sha256=bYsPBQZLoVHcOSpmdg3Q_Eqkeu4xU4YHyzWl9Fp9YUY,718
+mce/core/specs/bias.py,sha256=zos8-G3L8o24SuztZ_n9dFUl7_ox3jjbMv6qLD4hNp0,706
+mce/core/specs/call_count.py,sha256=IZO66wbUWMUi79v62dipgtZ781iHN7p5u95OcrJDR5g,540
+mce/core/specs/coherence.py,sha256=WnAU5H5vR1bQEG-lGgg-KWJGcx4QAlA71lhF83e-qn8,698
+mce/core/specs/completion_rate.py,sha256=9yJQWobhsCorXep-gMNVT9477Y1CRUBbImYuZyIFDQ0,1856
+mce/core/specs/component_conflict_rate.py,sha256=GkBJINRW82fEaeUCVjEiwfa4yFJrO1ps46SBFUeV4QE,701
+mce/core/specs/consistency.py,sha256=3E_Tk5YTB6G5hwiwc3YoI_otHILJcRlRuitwH9Iu9s4,693
+mce/core/specs/context_preservation.py,sha256=HETT74CH7inEJuDwMDS_qDhELOMRh8ql7PdTeNvPmY0,701
+mce/core/specs/conversation_completeness.py,sha256=DQBGA88MQkUvkk4oBc9XVi8eyi8ywlzAajax8BMt_zk,728
+mce/core/specs/cost.py,sha256=rYeETaAUAszXW6nSyAUNSleqwoBrnzrrxBRpmEOMjXw,546
+mce/core/specs/cycles_count.py,sha256=0g-RvcYZonT3F_cm6SjaQJ0bTuzjHXMvJ8UZmswcVGw,511
+mce/core/specs/duration.py,sha256=8ck8rLwaLwc7s80ST0CucPcaW5AvnwVKuaprbUZHaW0,528
+mce/core/specs/general_structure_and_style.py,sha256=CkqEUnynn5x4zrfc80PQyJJUO88qlBhfTQzUI-4DYUs,741
+mce/core/specs/goal_success_rate.py,sha256=VPZ1XUCMq1M5WMg3h1SRdA3Vxi6D2Pblyy6FUoQkPjs,690
+mce/core/specs/graph_determinism_score.py,sha256=w-y5pyPexR0wbh5It2OHkC1DtTyOoIjPqdks3XSa3j8,528
+mce/core/specs/groundedness.py,sha256=5C9Ftmzl9Yev-cjNuX1ZMVpYhtbQ6zzFHBvEnscTjJM,689
+mce/core/specs/hallucination.py,sha256=TqPJeIS0kveq_f34vfL6cWGFSM2cPepqCDBvBQ5Dp-E,678
+mce/core/specs/information_retention.py,sha256=DDDNPI7YovEbMhtk8vblAHI5fIJIEDmxPQ1rT7DuiSU,708
+mce/core/specs/intent_recognition_accuracy.py,sha256=ECApsUgj2kQwOe4TxQd8Q2GO0tofWuR89aowuW1Z_BU,704
+mce/core/specs/llm_average_confidence.py,sha256=WQXJ1oOTun4dJg12Ind8TGsZdOdFRXzJRXtidJy91tc,527
+mce/core/specs/llm_error_rate.py,sha256=Ykn8n0Ogn9XwKT2M3S6M8nd44I8PZVV4kiey3EAypIk,2483
+mce/core/specs/llm_maximum_confidence.py,sha256=vtgP_fHeGVZC-QGEWdLMoBQMwDROmLAcp1-HMuUmN8Y,527
+mce/core/specs/llm_minimum_confidence.py,sha256=KoLJEVD83PQmIpo2sDAKwUdcaBuLuPl3ZAIIaIokorw,527
+mce/core/specs/response_completeness.py,sha256=3KxaYbJg8jwYIk7bVzo6A2HWXjqNyu44k_xPp_5kQYA,692
+mce/core/specs/role_adherence.py,sha256=X7oHPDYELByqXoS9Ts15550lzRlPZWi5zC36qPBUpfo,711
+mce/core/specs/sentiment.py,sha256=XnAg69SUTmRWtM-7MG-rZ-QrRB21jgwbT4BhIIIcveg,663
+mce/core/specs/task_completion.py,sha256=-2RXHd3eShvOxA7_PHSwA36-YlIKDqEDirTksZHWCJw,735
+mce/core/specs/task_delegation_accuracy.py,sha256=noHXyy7s5Ldcg0CjRF5kKOtMisp-OGS8NHqAxNJ7tSM,726
+mce/core/specs/token_count.py,sha256=D80Y4GkoQFpAsoRSDvYNmme9XXbYgX_CE_CVxvxkdyU,545
+mce/core/specs/tonality.py,sha256=_saGt5R6Vlc6dDX1Yve5B9Ln_2mW_3uhviEam-8X6P4,677
+mce/core/specs/tool_error.py,sha256=TJ2TZ9bGmG3-4wsh9qL4wxEybVW_m0HTqLhnjYoU9ws,518
+mce/core/specs/tool_error_rate.py,sha256=98-pNR_m4RZcB8DKxTzYV9EgSRJvw4J75HSqkl3EHcw,525
+mce/core/specs/tool_utilization_accuracy.py,sha256=5TBW_EIzxxVOeiQcgXMEvD8UI8W1Wi8v2IiHqhf1Lrw,3035
+mce/core/specs/topic_adherence.py,sha256=Xn9VnKPXif00AGGgRWCVHpglJ6QCCCm-vnteC4ye07w,700
+mce/core/specs/toxicity.py,sha256=AWqCtevXMVT4hIxHuwJMF2ntc7ODewZ1wNVS_la_cFI,710
+mce/core/specs/workflow_cohesion_index.py,sha256=y4LkBwO8kPHcZ37kBGVYF7YsvJREy0NTEjufrxzpARM,710
+mce/core/specs/workflow_efficiency.py,sha256=qpKNKKX4bE-Fz6OmNSnenpqmBQQj0vwArEe1At8ykLk,3047
+mce/engine/__init__.py,sha256=mfLW6UKKVo6maM5jD9r7r6eQSW8deEwiln4FuzZF1J0,807
+mce/engine/caching.py,sha256=oD83Bwnbe2YYuNSBEcaPRpqfOVCPoqh0WseO_WzTamM,24097
+mce/engine/context_builder.py,sha256=Weu03WEJGDM0ehABo4oBbuZe56D_5HC5Z4-dn3ubsMg,10868
+mce/engine/engine.py,sha256=mY6L8rG6COHe5Lozkh0LUXA_NEYqzgExSWCbZopYSlw,34610
+mce/engine/llm.py,sha256=Nksjh1lXNJXrEukgprL35phd1dzaZEG7QIQrHjOM27s,12350
+mce/engine/pricing.py,sha256=brdkvEZukuDedEh9j2R8px3s4eTL12x9uzn7V1LWFs8,5292
+mce/engine/scheduler.py,sha256=S7J2evwodA_CScSzGOzOPQ4hX59Bm2pR1Sy6KLrH9KA,2154
+mce/engine/strategies.py,sha256=qzQy3QjVEVxO_O5pBojUWIZ7aKbQlYdkk7EztY-q7LY,4813
+mce/engine/usage.py,sha256=WUVz50xClk4y7Kqjz0TJtcF4L7Vx3rKNDiG-0MIson0,3008
+mce/engine/worker.py,sha256=sCdotByhpVQ_0nY1KhuuPQiMCA8anLZ-rASjnHQybcI,526
+mce/engine/data/model_prices.json,sha256=NWKp4UqRt5gN-Ht468-v5c43asJurTKj1HYVh7v4m8w,322046
+mce/providers/__init__.py,sha256=U0m0dDHDeU7LiiGdBKmqddgbzR9l3kO4I-S7olR0p5w,361
+mce/providers/base_wrapper.py,sha256=uWhv3dG41qw4jmDPXJsj--U3-JYiPtK9oCODuFIhH6E,4852
+mce/providers/catalog.py,sha256=8nDFvC7D-2qCbMmBRtOoI8tF4Z4HAy1iW3jgIWCt1O8,799
+mce/providers/template/__init__.py,sha256=-7KPZOheq2Mevhzt7vt6VZgK8lBTBQ4HEXgJm-njqTE,177
+mce/providers/virtual/aggregators.py,sha256=Yv8bHtSzv5s-W6TsozdZAjjrJI1vhIruv_2OEuv49BU,5311
+ioc_scope_mce_core-2.0.0.dist-info/METADATA,sha256=GXuvWV0glctawmazwUY2IrZy4hr6FAx6kp1vkR2XzGE,1863
+ioc_scope_mce_core-2.0.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+ioc_scope_mce_core-2.0.0.dist-info/top_level.txt,sha256=Fxy5kcrJSiMg-a89I_3tPx7wdnlIQ5vM85AN-Krho4U,4
+ioc_scope_mce_core-2.0.0.dist-info/RECORD,,

ioc_scope_mce_core-2.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

ioc_scope_mce_core-2.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+mce

mce/__init__.py

@@ -0,0 +1,27 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+from pkgutil import extend_path
+
+import warnings
+
+# Allow multiple workspace packages to contribute to top-level `mce`.
+__path__ = extend_path(__path__, __name__)
+
+# Suppress Pydantic v1 compatibility warning from opik in Python 3.14+
+# Must be at the top level before any other imports
+warnings.filterwarnings(
+    "ignore",
+    message="Core Pydantic V1 functionality isn't compatible with Python 3.14 or greater",
+    category=UserWarning,
+)
+
+# Expose key public components
+try:
+    from .core.types import MetricResult as MetricResult
+    from .core.metric import Metric as Metric
+    from .engine.engine import MetricEngine as MetricEngine
+
+    __all__ = ["MetricResult", "Metric", "MetricEngine"]
+except ImportError:
+    __all__: list[str] = []

mce/core/catalog.py

@@ -0,0 +1,208 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+import logging
+import importlib
+import inspect
+import pkgutil
+import threading
+import statistics
+from mce.core.metric import Metric
+
+logger = logging.getLogger(__name__)
+
+# Lazy imports for heavy dependencies
+try:
+    from scope_ontology import OntologyService
+except ImportError:
+    OntologyService = None
+
+
+class MetricCatalogService:
+    """
+    Core Service: unified registry for Metric instances.
+    Supports lazy creation of templated aggregate (virtual) metrics.
+    """
+
+    _instance = None
+    _lock = threading.Lock()  # Guards singleton creation
+
+    def __init__(self):
+        self._contracts: dict[str, Metric | type[Metric]] = {}
+        self._unified_metrics: dict[str, list[Metric]] = {}
+
+    @classmethod
+    def get_instance(cls):
+        """Thread-safe singleton accessor."""
+        if cls._instance is None:
+            with cls._lock:
+                if cls._instance is None:  # double-checked locking
+                    cls._instance = MetricCatalogService()
+        return cls._instance
+
+    def register_metric(self, metric: Metric | type[Metric]) -> None:
+        """Register a unified Metric plugin or abstract Base."""
+        if isinstance(metric, type):
+            # Abstract Class registration
+            if not hasattr(metric, "metadata"):
+                return  # Skip if no metadata
+            mid = metric.metadata.name
+            self._contracts[mid] = metric
+            return
+
+        # Instance registration
+        mid = metric.metric_id
+        if mid not in self._contracts:
+            self._contracts[mid] = metric
+
+        if mid not in self._unified_metrics:
+            self._unified_metrics[mid] = []
+        self._unified_metrics[mid].append(metric)
+
+    def get_contract(self, metric_id: str) -> Metric | type[Metric] | None:
+        """Look up a registered contract (or unified metric) by ID."""
+        return self._contracts.get(metric_id)
+
+    def get_metric(self, metric_id: str) -> Metric | None:
+        """
+        Get a unified metric instance.
+
+        **Lazy Loading for Templated Aggregates** (Option C):
+        If metric_id matches pattern "operation_BaseMetric" (e.g., "avg_AnswerRelevancy"),
+        automatically create and cache an AggregateMetric instance.
+
+        Supported operations:
+        - avg_* → AverageMetric(base_metric)
+        - max_* → AggregateMetric(base_metric, "max", max)
+        - min_* → AggregateMetric(base_metric, "min", min)
+        - sum_* → AggregateMetric(base_metric, "sum", sum)
+        """
+        # Check if already registered
+        metrics = self._unified_metrics.get(metric_id, [])
+        if metrics:
+            return metrics[0]
+
+        # Lazy loading: detect templated aggregate pattern
+        operation_mapping = {
+            "avg": ("avg", "AverageMetric"),
+            "max": ("max", "max"),
+            "min": ("min", "min"),
+            "sum": ("sum", "sum"),
+        }
+
+        for prefix, (op_name, op_type) in operation_mapping.items():
+            if metric_id.startswith(f"{prefix}_"):
+                base_metric_id = metric_id[len(prefix) + 1 :]  # Remove "avg_" prefix
+
+                # Validate base metric exists
+                if base_metric_id not in self._contracts:
+                    return None  # Base metric not found, cannot create aggregate
+
+                # Import and instantiate appropriate aggregate metric
+                if op_type == "AverageMetric":
+                    from mce.providers.virtual.aggregators import AverageMetric
+
+                    aggregate_metric = AverageMetric(target_metric_id=base_metric_id)
+                else:
+                    # max / min / sum operations
+                    from mce.providers.virtual.aggregators import AggregateMetric
+
+                    # Fix #20: statistics imported at module level
+                    op_fn_map = {
+                        "max": max,
+                        "min": min,
+                        "sum": sum,
+                        "avg": statistics.mean,
+                    }
+                    aggregate_metric = AggregateMetric(
+                        target_metric_id=base_metric_id,
+                        operation_name=op_name,
+                        operation_fn=op_fn_map[op_name],
+                    )
+
+                # Resolve target_types for the new metric
+                self._resolve_target_types(aggregate_metric)
+
+                # Register the lazily created metric
+                self.register_metric(aggregate_metric)
+                return aggregate_metric
+
+        return None
+
+    def _resolve_target_types(self, metric: "Metric") -> None:
+        """Resolve target_types for a lazily created aggregate metric.
+
+        Fix #16: delegates to the single source of truth in discovery.py
+        (ensure_target_types) instead of duplicating the mapping logic here.
+        The lazy import avoids the circular dependency
+        (catalog → discovery → catalog).
+        """
+        from mce.core.registry.discovery import (
+            ensure_target_types,
+        )  # lazy — avoids circular import
+
+        ensure_target_types(metric)
+
+        # If ensure_target_types could not resolve (no scope/attachment_point),
+        # fall back to ontology_class-based mapping for aggregate metrics which
+        # inherit the base metric's class name rather than a MAS prefix.
+        md = metric.metadata
+        if not getattr(md, "target_types", None):
+            ap = md.attachment_point
+            if ap and OntologyService is not None:
+                try:
+                    onto = OntologyService.get_instance()
+                    md.target_types = onto.get_subclasses_of(ap)
+                    return
+                except Exception as e:
+                    logger.warning(
+                        "OntologyService.get_subclasses_of(%r) failed: %s — "
+                        "falling back to ontology_class mapping for '%s'.",
+                        ap,
+                        e,
+                        md.name,
+                    )
+            # Last resort: map known ontology_class strings.
+            _class_map = {
+                "Session": {"mas:Session"},
+                "LLMCall": {"mas:LLMCall"},
+                "mas:LLMCall": {"mas:LLMCall"},
+                "ToolCall": {"mas:ToolCall"},
+                "mas:ToolCall": {"mas:ToolCall"},
+            }
+            md.target_types = _class_map.get(md.ontology_class, set())
+
+    def get_all_contracts(self) -> list[Metric | type[Metric]]:
+        return list(self._contracts.values())
+
+    def discover_metrics(self, package_path: str = "mce.metrics"):
+        """Auto-discover concrete Metric subclasses in the specified package."""
+        try:
+            module = importlib.import_module(package_path)
+            path = getattr(module, "__path__", [])
+            # NOTE: use module_name (not `name`) so the inner member loop cannot
+            # shadow the outer iteration variable — that was a silent bug.
+            # Fix #5: use walk_packages (recursive) instead of iter_modules (single-level)
+            # so metrics in sub-packages are not silently dropped.
+            for _, module_name, _ in pkgutil.walk_packages(
+                path, prefix=f"{package_path}."
+            ):
+                sub_module = importlib.import_module(module_name)
+                for member_name, obj in inspect.getmembers(sub_module):
+                    if (
+                        inspect.isclass(obj)
+                        and issubclass(obj, Metric)
+                        and obj is not Metric
+                    ):
+                        if inspect.isabstract(obj):
+                            continue  # Skip abstract bases
+                        try:
+                            instance = obj()
+                            self.register_metric(instance)
+                        except Exception as e:
+                            logger.warning(
+                                "Failed to instantiate metric %s: %s", member_name, e
+                            )
+        except ImportError as e:
+            logger.warning("Could not import package %s: %s", package_path, e)

mce/core/config.py

@@ -0,0 +1,33 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+from pydantic_settings import BaseSettings, SettingsConfigDict
+from pydantic import Field
+
+
+class MceSettings(BaseSettings):
+    """
+    Configuration for the MCE library.
+    Settings can be overridden by environment variables with the prefix MCE_.
+
+    Provider availability is determined automatically: if a package is importable
+    (e.g. deepeval, ragas, opik) its metrics are discovered and registered.
+    Install the metapackage to get everything, or install individual provider
+    packages to pick only what you need.
+    """
+
+    strict_fail_fast: bool = Field(
+        default=False, description="Raise exceptions on provider initialization failure"
+    )
+
+    model_config = SettingsConfigDict(
+        env_prefix="MCE_",
+        env_file=".env",
+        env_file_encoding="utf-8",
+        case_sensitive=False,
+        extra="ignore",
+    )
+
+
+# Global settings instance
+settings = MceSettings()

mce/core/helpers.py

@@ -0,0 +1,184 @@
+#  Copyright (c) 2026 Cisco Systems, Inc. and its affiliates
+#  SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+from typing import Any
+import json
+import re
+from mce.core.metric import Metric
+
+
+def _safe_session(context: dict[str, Any]) -> Any:
+    """Return the session object from context if available."""
+    return context.get("session")
+
+
+def _conversation_data(context: dict[str, Any]) -> dict[str, Any]:
+    """Return conversation data from context or session."""
+    data = context.get("conversation_data")
+    if isinstance(data, dict):
+        return data
+    session = _safe_session(context)
+    if session is not None:
+        session_data = getattr(session, "conversation_data", None)
+        if isinstance(session_data, dict):
+            return session_data
+    return {}
+
+
+def _conversation_text(context: dict[str, Any]) -> str:
+    """Extract the conversation text from context or session."""
+    for key in ["conversation_text", "transcript", "conversation"]:
+        value = context.get(key)
+        if isinstance(value, str) and value:
+            return value
+    data = _conversation_data(context)
+    if "conversation" in data and isinstance(data["conversation"], str):
+        return data["conversation"]
+    if "elements" in data:
+        elements = data.get("elements")
+        if isinstance(elements, (list, dict)):
+            return json.dumps(elements, indent=2)
+        if isinstance(elements, str):
+            return elements
+    query, response = _query_response(context)
+    if query or response:
+        return f"User: {query}\nAssistant: {response}".strip()
+    return ""
+
+
+def _query_response(context: dict[str, Any]) -> tuple[str, str]:
+    """Extract the input query and final response from context or session."""
+    query = context.get("input_query") or context.get("input_text")
+    response = context.get("final_response") or context.get("output_text")
+    data = _conversation_data(context)
+    if not query and isinstance(data.get("query"), str):
+        query = data.get("query")
+    if not response and isinstance(data.get("response"), str):
+        response = data.get("response")
+    session = _safe_session(context)
+    if session is not None:
+        if not query and isinstance(getattr(session, "input_query", None), str):
+            query = getattr(session, "input_query")
+        if not response and isinstance(getattr(session, "final_response", None), str):
+            response = getattr(session, "final_response")
+
+    llm_spans = context.get("llm_spans") or []
+    if not llm_spans and session is not None and hasattr(session, "llm_spans"):
+        llm_spans = session.llm_spans or []
+
+    if not query:
+        for span in llm_spans:
+            query = _span_value(span, "inputContent", "prompt", "input_text")
+            if query:
+                break
+
+    if not response:
+        for span in reversed(llm_spans):
+            response = _span_value(span, "outputContent", "completion", "output_text")
+            if response:
+                break
+
+    return query or "", response or ""
+
+
+def _llm_binary_score(metric: Metric, prompt: str) -> tuple[float, str]:
+    """Get a binary score from an LLM response using the first numeric token."""
+    response = metric.call_llm(prompt).strip()
+    match = re.search(r"-?\d+(?:\.\d+)?", response)
+    score = float(match.group()) if match else 0.0
+    score = max(0.0, min(1.0, score))
+    return score, response
+
+
+def _span_dict(span: Any) -> dict[str, Any]:
+    """Normalize span to a dictionary for attribute access."""
+    if isinstance(span, dict):
+        return span
+    return getattr(span, "__dict__", {})
+
+
+def _span_attrs(span: Any) -> dict[str, Any]:
+    """Extract raw span attributes with multiple fallback keys."""
+    data = _span_dict(span)
+    if isinstance(data.get("attrs"), dict):
+        return data["attrs"]
+    raw_span = data.get("raw_span_data")
+    if isinstance(raw_span, dict):
+        for key in ["SpanAttributes", "attributes"]:
+            attrs = raw_span.get(key)
+            if isinstance(attrs, dict):
+                return attrs
+    return {}
+
+
+def _span_value(span: Any, *keys: str) -> str | None:
+    """Return the first available string value from span data or attributes."""
+    data = _span_dict(span)
+    attrs = _span_attrs(span)
+    for key in keys:
+        value = data.get(key)
+        if isinstance(value, str) and value:
+            return value
+        attr_value = attrs.get(key)
+        if isinstance(attr_value, str) and attr_value:
+            return attr_value
+    return None
+
+
+def _first_number(context: dict[str, Any], keys: list[str]) -> float:
+    for key in keys:
+        value = context.get(key)
+        if isinstance(value, (int, float)):
+            return float(value)
+    return 0.0
+
+
+def _find_key_value(data: Any, target_key: str) -> Any:
+    if isinstance(data, dict):
+        if target_key in data:
+            return data[target_key]
+        for value in data.values():
+            result = _find_key_value(value, target_key)
+            if result is not None:
+                return result
+    elif isinstance(data, list):
+        for item in data:
+            result = _find_key_value(item, target_key)
+            if result is not None:
+                return result
+    return None
+
+
+def _extract_logprobs(span: Any) -> list[float]:
+    span_dict = span if isinstance(span, dict) else getattr(span, "__dict__", {})
+    output_payload = span_dict.get("output_payload") or span_dict.get("output")
+    if output_payload is None:
+        return []
+
+    logprobs_block = _find_key_value(output_payload, "logprobs")
+    if not isinstance(logprobs_block, dict):
+        return []
+
+    content = logprobs_block.get("content")
+    if not isinstance(content, list):
+        return []
+
+    logprobs: list[float] = []
+    for token_dict in content:
+        if isinstance(token_dict, dict) and isinstance(
+            token_dict.get("logprob"), (int, float)
+        ):
+            logprobs.append(float(token_dict["logprob"]))
+    return logprobs
+
+
+def _session_logprobs(context: dict[str, Any]) -> list[float]:
+    spans = context.get("llm_spans") or []
+    session = context.get("session")
+    if not spans and session is not None and hasattr(session, "llm_spans"):
+        spans = session.llm_spans or []
+
+    logprobs: list[float] = []
+    for span in spans:
+        logprobs.extend(_extract_logprobs(span))
+    return logprobs