layaeval 0.1.0__py3-none-any.whl

layaeval/mcp/client.py

@@ -0,0 +1,310 @@
+"""MCP stdio client connecting to the Laya MCP server."""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+import sys
+import threading
+from collections.abc import Sequence
+from typing import Any
+
+
+class MCPClientError(Exception):
+    """Raised when an MCP client or server interaction fails."""
+
+
+class LayaMCPClient:
+    """Manages an MCP stdio subprocess connection to `laya-mcp-server`."""
+
+    def __init__(
+        self,
+        command: Sequence[str] | None = None,
+        timeout: float = 60.0,
+        env: dict[str, str] | None = None,
+    ):
+        """Initialize the client.
+
+        Args:
+            command: Command list to launch the MCP server. Defaults to
+                `["laya-mcp-server"]` or `[sys.executable, "-m", "laya.mcp.server"]`.
+            timeout: Subprocess read/write timeout in seconds.
+            env: Custom environment variables to pass to the subprocess.
+        """
+        self.command = list(command) if command else self._default_command()
+        self.timeout = timeout
+        self.env = env
+        self.process: subprocess.Popen | None = None
+        self._msg_id = 0
+        self._lock = threading.Lock()
+
+    @staticmethod
+    def _default_command() -> list[str]:
+        # Try binary on PATH first, otherwise fallback to python module
+        if shutil.which("laya-mcp-server"):
+            return ["laya-mcp-server"]
+        return [sys.executable, "-m", "laya.mcp.server"]
+
+    def start(self) -> None:
+        """Start the MCP server subprocess and perform the initialize handshake."""
+        if self.process is not None and self.process.poll() is None:
+            return  # Already running
+
+        cmd_env = dict(os.environ)
+        if self.env:
+            cmd_env.update(self.env)
+        # Ensure UTF-8 IO
+        cmd_env.setdefault("PYTHONIOENCODING", "utf-8")
+
+        try:
+            self.process = subprocess.Popen(
+                self.command,
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+                bufsize=1,
+                env=cmd_env,
+            )
+        except FileNotFoundError as exc:
+            raise MCPClientError(
+                f"Failed to execute command '{' '.join(self.command)}'. "
+                "Ensure `laya[mcp]` is installed (`pip install 'layaeval[laya]'` or "
+                "`pip install 'laya[mcp]'`)."
+            ) from exc
+
+        # Perform MCP initialize handshake
+        self._initialize()
+
+    def stop(self) -> None:
+        """Gracefully terminate the subprocess."""
+        if self.process is None:
+            return
+        try:
+            if self.process.stdin and not self.process.stdin.closed:
+                self.process.stdin.close()
+            self.process.terminate()
+            self.process.wait(timeout=2.0)
+        except Exception:
+            if self.process.poll() is None:
+                self.process.kill()
+        finally:
+            self.process = None
+
+    def __enter__(self) -> LayaMCPClient:
+        self.start()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
+        self.stop()
+
+    def _next_id(self) -> int:
+        self._msg_id += 1
+        return self._msg_id
+
+    def _send_request(self, method: str, params: dict[str, Any] | None = None) -> dict[str, Any]:
+        """Send JSON-RPC request and wait for the response."""
+        if self.process is None:
+            self.start()
+
+        if self.process is None or self.process.poll() is not None:
+            raise MCPClientError("MCP server process is not running.")
+
+        req_id = self._next_id()
+        msg: dict[str, Any] = {
+            "jsonrpc": "2.0",
+            "id": req_id,
+            "method": method,
+        }
+        if params is not None:
+            msg["params"] = params
+
+        payload = json.dumps(msg) + "\n"
+        with self._lock:
+            try:
+                assert self.process.stdin is not None
+                self.process.stdin.write(payload)
+                self.process.stdin.flush()
+            except (BrokenPipeError, OSError) as exc:
+                stderr_text = ""
+                if self.process.stderr:
+                    try:
+                        stderr_text = self.process.stderr.read()
+                    except Exception:
+                        pass
+                raise MCPClientError(
+                    f"Failed writing to MCP server stdin: {exc}. Server stderr: {stderr_text}"
+                ) from exc
+
+            # Read response lines until we get the matching ID
+            while True:
+                assert self.process.stdout is not None
+                line = self.process.stdout.readline()
+                if not line:
+                    stderr_msg = ""
+                    if self.process.stderr:
+                        try:
+                            stderr_msg = self.process.stderr.read()
+                        except Exception:
+                            pass
+                    raise MCPClientError(
+                        f"MCP server closed stdout unexpectedly. Stderr: {stderr_msg}"
+                    )
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    resp = json.loads(line)
+                except json.JSONDecodeError:
+                    continue
+
+                if resp.get("id") == req_id:
+                    if "error" in resp:
+                        raise MCPClientError(f"MCP server error: {resp['error']}")
+                    return resp.get("result", {})
+
+    def _send_notification(self, method: str, params: dict[str, Any] | None = None) -> None:
+        if self.process is None or self.process.poll() is not None:
+            return
+        msg: dict[str, Any] = {
+            "jsonrpc": "2.0",
+            "method": method,
+        }
+        if params is not None:
+            msg["params"] = params
+        payload = json.dumps(msg) + "\n"
+        with self._lock:
+            try:
+                assert self.process.stdin is not None
+                self.process.stdin.write(payload)
+                self.process.stdin.flush()
+            except Exception:
+                pass
+
+    def _initialize(self) -> None:
+        """Execute MCP initialize sequence."""
+        init_params = {
+            "protocolVersion": "2024-11-05",
+            "capabilities": {},
+            "clientInfo": {
+                "name": "layaeval",
+                "version": "0.1.0",
+            },
+        }
+        self._send_request("initialize", init_params)
+        self._send_notification("notifications/initialized")
+
+    def call_tool(self, name: str, arguments: dict[str, Any]) -> Any:
+        """Invoke an MCP tool by name and return its parsed output."""
+        params = {
+            "name": name,
+            "arguments": arguments,
+        }
+        result = self._send_request("tools/call", params)
+        if result.get("isError"):
+            content = result.get("content", [])
+            err_text = content[0].get("text", "Unknown tool error") if content else "Error"
+            raise MCPClientError(f"Tool {name} failed: {err_text}")
+
+        content = result.get("content", [])
+        if not content:
+            return result
+
+        first = content[0]
+        if first.get("type") == "text":
+            text = first.get("text", "")
+            try:
+                return json.loads(text)
+            except json.JSONDecodeError:
+                return text
+        return content
+
+    def predict(
+        self,
+        state: dict[str, Any],
+        questions: dict[str, Any],
+        model: str = "auto",
+        **kwargs,
+    ) -> dict[str, Any]:
+        """Convenience method calling the `laya_predict` MCP tool."""
+        args: dict[str, Any] = {
+            "state": state,
+            "questions": questions,
+            "model": model,
+        }
+        args.update(kwargs)
+        res = self.call_tool("laya_predict", args)
+        if isinstance(res, dict) and "answers" in res:
+            return res["answers"]
+        return res
+
+
+class DemoMCPClient:
+    """Lightweight simulated client for offline demonstrations, testing, and notebooks."""
+
+    def __init__(self, **kwargs: Any) -> None:
+        pass
+
+    def start(self) -> None:
+        """No-op for demo client."""
+        pass
+
+    def stop(self) -> None:
+        """No-op for demo client."""
+        pass
+
+    def __enter__(self) -> DemoMCPClient:
+        return self
+
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        pass
+
+    def predict(
+        self,
+        state: dict[str, Any],
+        questions: dict[str, Any],
+        model: str = "auto",
+        **kwargs: Any,
+    ) -> dict[str, Any]:
+        """Simulate realistic calibrated neural answers for demo execution."""
+        answers: dict[str, Any] = {}
+        resp = str(state.get("response", state.get("final_result", ""))).lower()
+        ctx = str(state.get("retrieved_context", state.get("agent_trace", ""))).lower()
+
+        # Simple semantic overlap heuristic to provide natural outputs
+        has_overlap = (
+            any(w in ctx for w in resp.replace(".", " ").replace(",", " ").split() if len(w) > 3)
+            if (resp and ctx)
+            else True
+        )
+
+        for q_key, q_spec in questions.items():
+            q_type = q_spec.get("type", "score")
+            is_risk = any(
+                k in q_key.lower()
+                for k in ("risk", "contradiction", "conflict", "injection", "hallucination")
+            )
+
+            if q_type == "score":
+                score = round(3.8 if has_overlap else 1.8, 2)
+                answers[q_key] = {
+                    "score": score,
+                    "confidence": 0.94,
+                    "answer_confidence": 0.91,
+                }
+            elif q_type == "noul":
+                noul = round(0.06 if is_risk else (0.94 if has_overlap else 0.25), 3)
+                answers[q_key] = {
+                    "noul": noul,
+                    "confidence": 0.95,
+                    "answer_confidence": 0.92,
+                }
+            elif q_type == "choice":
+                choices = q_spec.get("criteria", q_spec.get("choices", ["default"]))
+                answers[q_key] = {
+                    "choice": choices[0] if choices else "selected",
+                    "confidence": 0.96,
+                }
+        return answers

layaeval/metrics/__init__.py

@@ -0,0 +1,109 @@
+"""LayaEval metric classes — RAG and Agent evaluation metrics."""
+
+from .agent import (
+    AgentAbstention,
+    AgentAnswerCompleteness,
+    AgentCorrectness,
+    AgentGroundedness,
+    AgentHallucinationRisk,
+    AgentResponseRelevance,
+    GoalAchievement,
+    IndirectPromptInjectionDetection,
+    IntentResolution,
+    NavigationEfficiency,
+    PolicyCompliance,
+    PromptInjectionDetection,
+    TaskAdherence,
+    TaskCompletion,
+    ToolCallAccuracy,
+    ToolInputAccuracy,
+    ToolNecessity,
+    ToolOutputUtilization,
+    ToolRedundancy,
+    ToolSelection,
+    ToolSelectionCorrectness,
+    ToolSequenceCorrectness,
+)
+from .rag import (
+    AbstentionCorrectness,
+    Answerability,
+    CitationCompleteness,
+    CitationCorrectness,
+    CitationRelevance,
+    ClaimSupport,
+    ContextConflictDetection,
+    ContextCoverage,
+    ContextSufficiency,
+    ContextUtilization,
+    ContradictionDetection,
+    DocumentRelevance,
+    FactualCorrectness,
+    Groundedness,
+    HallucinationRisk,
+    ResponseCompleteness,
+    ResponseRelevance,
+    RetrievalCompleteness,
+    RetrievalRelevance,
+    StrictGroundedness,
+)
+from .registry import (
+    AGENT_METRICS,
+    ALL_METRICS,
+    RAG_METRICS,
+    get_metric,
+    list_metrics,
+    resolve_metrics,
+)
+
+__all__ = [
+    "get_metric",
+    "resolve_metrics",
+    "list_metrics",
+    "ALL_METRICS",
+    "RAG_METRICS",
+    "AGENT_METRICS",
+    # RAG
+    "RetrievalRelevance",
+    "DocumentRelevance",
+    "ContextSufficiency",
+    "ContextCoverage",
+    "RetrievalCompleteness",
+    "Groundedness",
+    "StrictGroundedness",
+    "HallucinationRisk",
+    "ResponseRelevance",
+    "ResponseCompleteness",
+    "FactualCorrectness",
+    "Answerability",
+    "AbstentionCorrectness",
+    "ContextUtilization",
+    "CitationCorrectness",
+    "CitationCompleteness",
+    "CitationRelevance",
+    "ClaimSupport",
+    "ContradictionDetection",
+    "ContextConflictDetection",
+    # Agent
+    "IntentResolution",
+    "TaskCompletion",
+    "TaskAdherence",
+    "GoalAchievement",
+    "ToolSelection",
+    "ToolSelectionCorrectness",
+    "ToolCallAccuracy",
+    "ToolInputAccuracy",
+    "ToolOutputUtilization",
+    "ToolNecessity",
+    "ToolRedundancy",
+    "ToolSequenceCorrectness",
+    "NavigationEfficiency",
+    "AgentGroundedness",
+    "AgentHallucinationRisk",
+    "AgentResponseRelevance",
+    "AgentAnswerCompleteness",
+    "AgentCorrectness",
+    "AgentAbstention",
+    "PolicyCompliance",
+    "PromptInjectionDetection",
+    "IndirectPromptInjectionDetection",
+]

layaeval/metrics/_base.py

@@ -0,0 +1,175 @@
+"""Base class and data containers for LayaEval metrics."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class EvalResult:
+    """The result of evaluating a single metric on an input state."""
+
+    metric_id: str
+    metric_name: str
+    score: float | None = None
+    noul: float | None = None
+    choice: str | None = None
+    confidence: float | None = None
+    answer_confidence: float | None = None
+    passed: bool = False
+    threshold: float = 0.0
+    reason: str | None = None
+    raw: dict[str, Any] = field(default_factory=dict)
+    state: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert the result to a JSON-serializable dictionary."""
+        return {
+            "metric_id": self.metric_id,
+            "metric_name": self.metric_name,
+            "score": self.score,
+            "noul": self.noul,
+            "choice": self.choice,
+            "confidence": self.confidence,
+            "answer_confidence": self.answer_confidence,
+            "passed": self.passed,
+            "threshold": self.threshold,
+            "reason": self.reason,
+            "raw": self.raw,
+        }
+
+
+class Metric:
+    """Abstract base class for all LayaEval evaluation metrics."""
+
+    metric_id: str = "GEN-000"
+    metric_name: str = "generic_metric"
+    question_type: str = "score"  # "score", "noul", "choice"
+    state_fields: tuple[str, ...] = ()
+    optional_fields: tuple[str, ...] = ()
+    _question_key: str = ""
+    _question_template: dict[str, Any] = {}
+
+    def __init__(self, threshold: float | None = None):
+        """Initialize metric with optional default threshold."""
+        if threshold is not None:
+            self.default_threshold = float(threshold)
+        elif self.question_type == "score":
+            self.default_threshold = 3.0  # default 3 on 0..4 rubric
+        elif self.question_type == "noul":
+            self.default_threshold = 0.5  # default P(true) >= 0.5
+        else:
+            self.default_threshold = 0.0
+
+    def __call__(
+        self,
+        evaluator: Any | None = None,
+        threshold: float | None = None,
+        **state_kwargs: Any,
+    ) -> EvalResult:
+        """Evaluate this metric directly as a callable.
+
+        Example:
+            groundedness = Groundedness(threshold=3.0)
+            res = groundedness(retrieved_context="...", response="...")
+        """
+        from ..evaluator import default_evaluator
+
+        ev = evaluator or default_evaluator()
+        return ev.evaluate(self, threshold=threshold, **state_kwargs)
+
+    def evaluate(
+        self,
+        evaluator: Any | None = None,
+        threshold: float | None = None,
+        **state_kwargs: Any,
+    ) -> EvalResult:
+        """Evaluate this metric against input state."""
+        return self(evaluator=evaluator, threshold=threshold, **state_kwargs)
+
+    @property
+    def question_key(self) -> str:
+        """The key used in the questions dict."""
+        return self._question_key or self.metric_name
+
+    def build_state(self, **kwargs) -> dict[str, Any]:
+        """Validate required state fields and return the state dict for LAYA."""
+        missing = [f for f in self.state_fields if f not in kwargs or kwargs[f] is None]
+        if missing:
+            raise ValueError(
+                f"Metric {self.metric_name} ({self.metric_id}) requires state fields: "
+                f"{', '.join(missing)}"
+            )
+        state: dict[str, Any] = {}
+        for key in self.state_fields:
+            state[key] = kwargs[key]
+        for key in self.optional_fields:
+            if key in kwargs and kwargs[key] is not None:
+                state[key] = kwargs[key]
+        return state
+
+    def build_questions(self) -> dict[str, Any]:
+        """Generate the typed question dictionary to send to LAYA."""
+        return dict(self._question_template)
+
+    def parse_result(
+        self,
+        answer: dict[str, Any],
+        threshold: float | None = None,
+        state: dict[str, Any] | None = None,
+    ) -> EvalResult:
+        """Parse raw LAYA answer output into a structured EvalResult."""
+        t = self.default_threshold if threshold is None else float(threshold)
+        raw_ans = answer.get(self.question_key, answer)
+
+        score_val: float | None = None
+        noul_val: float | None = None
+        choice_val: str | None = None
+        passed = False
+        reason = None
+
+        if self.question_type == "score":
+            if "score" in raw_ans and raw_ans["score"] is not None:
+                score_val = float(raw_ans["score"])
+                passed = score_val >= t
+                reason = f"score {score_val} {'≥' if passed else '<'} threshold {t}"
+        elif self.question_type == "noul":
+            if "noul" in raw_ans and raw_ans["noul"] is not None:
+                noul_val = float(raw_ans["noul"])
+                # For risk/hallucination/contradiction metrics, pass condition may be <= max
+                if (
+                    "risk" in self.metric_name
+                    or "contradiction" in self.metric_name
+                    or "conflict" in self.metric_name
+                    or "injection" in self.metric_name
+                    or "hallucination" in self.metric_name
+                    or "redundant" in self.metric_name
+                ):
+                    passed = noul_val <= t
+                    reason = f"risk {noul_val} {'≤' if passed else '>'} threshold {t}"
+                else:
+                    passed = noul_val >= t
+                    reason = f"noul P(true) {noul_val} {'≥' if passed else '<'} threshold {t}"
+        elif self.question_type == "choice":
+            choice_val = raw_ans.get("choice")
+            passed = bool(choice_val is not None)
+            reason = f"selected choice: {choice_val}"
+
+        conf = raw_ans.get("confidence")
+        ans_conf = raw_ans.get("answer_confidence")
+
+        return EvalResult(
+            metric_id=self.metric_id,
+            metric_name=self.metric_name,
+            score=score_val,
+            noul=noul_val,
+            choice=choice_val,
+            confidence=float(conf) if conf is not None else None,
+            answer_confidence=float(ans_conf) if ans_conf is not None else None,
+            passed=passed,
+            threshold=t,
+            reason=reason,
+            raw=raw_ans,
+            state=state or {},
+        )