mindzeval 0.1.0__py3-none-any.whl

mindzeval/__init__.py

@@ -0,0 +1,63 @@
+"""MindzEval: evaluation framework for SQL agents.
+
+Grades the data an agent returns rather than the SQL it writes, validates the
+reference answers before trusting them, and scores behaviour (answer / clarify /
+decline) in both directions.
+"""
+
+from mindzeval._version import __version__
+from mindzeval.db import Database, QueryResult
+from mindzeval.errors import (
+    ConfigError,
+    DatabaseError,
+    Issue,
+    MindzEvalError,
+    QueryTimeoutError,
+    SQLGuardError,
+)
+from mindzeval.loading import Project, find_config, load_cases, load_config, load_project
+from mindzeval.results import TurnResult, Verdict
+from mindzeval.runner import RunResult, run, run_project, select_cases
+from mindzeval.schema import (
+    AgentResponse,
+    Behavior,
+    Case,
+    CompareMode,
+    Config,
+    Expectation,
+    Tolerance,
+    ToleranceOverride,
+    Turn,
+)
+
+__all__ = [
+    "AgentResponse",
+    "Behavior",
+    "Case",
+    "CompareMode",
+    "Config",
+    "ConfigError",
+    "Database",
+    "DatabaseError",
+    "Expectation",
+    "Issue",
+    "MindzEvalError",
+    "Project",
+    "QueryResult",
+    "QueryTimeoutError",
+    "RunResult",
+    "SQLGuardError",
+    "Tolerance",
+    "ToleranceOverride",
+    "Turn",
+    "TurnResult",
+    "Verdict",
+    "__version__",
+    "find_config",
+    "load_cases",
+    "load_config",
+    "load_project",
+    "run",
+    "run_project",
+    "select_cases",
+]

mindzeval/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

mindzeval/adapters/__init__.py

@@ -0,0 +1,58 @@
+"""Agent adapters: Python function, HTTP endpoint, or pre-generated outputs."""
+
+from __future__ import annotations
+
+from mindzeval.adapters.base import (
+    Adapter,
+    AgentError,
+    AgentErrorKind,
+    AgentRequest,
+    Message,
+    coerce_response,
+)
+from mindzeval.adapters.http import HttpAdapter
+from mindzeval.adapters.outputs import OutputsAdapter
+from mindzeval.adapters.python import PythonAdapter, load_target
+from mindzeval.adapters.resilient import ResilientAdapter
+from mindzeval.schema.config import (
+    Config,
+    HttpAgentConfig,
+    OutputsAgentConfig,
+    PythonAgentConfig,
+)
+
+
+def build_adapter(config: Config) -> Adapter:
+    """Create the adapter described by ``config.agent``, with retries and rate limiting."""
+    agent = config.agent
+    if isinstance(agent, OutputsAgentConfig):
+        return OutputsAdapter(config.resolve_path(agent.path))
+    inner: Adapter
+    if isinstance(agent, PythonAgentConfig):
+        inner = PythonAdapter(load_target(agent.target, config.base_dir), name=agent.target)
+    elif isinstance(agent, HttpAgentConfig):
+        inner = HttpAdapter(agent)
+    else:  # pragma: no cover - the union is exhaustive
+        raise TypeError(f"unsupported agent config {type(agent).__name__}")
+    return ResilientAdapter(
+        inner,
+        timeout_s=agent.timeout_s,
+        retries=agent.retries,
+        requests_per_minute=agent.requests_per_minute,
+    )
+
+
+__all__ = [
+    "Adapter",
+    "AgentError",
+    "AgentErrorKind",
+    "AgentRequest",
+    "HttpAdapter",
+    "Message",
+    "OutputsAdapter",
+    "PythonAdapter",
+    "ResilientAdapter",
+    "build_adapter",
+    "coerce_response",
+    "load_target",
+]

mindzeval/adapters/base.py

@@ -0,0 +1,101 @@
+"""The adapter interface: how MindzEval calls an agent, whatever it is built with."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from enum import StrEnum
+from typing import Any
+
+from pydantic import ValidationError
+
+from mindzeval.schema.response import AgentResponse
+
+
+@dataclass(frozen=True)
+class Message:
+    """One message of the conversation so far."""
+
+    role: str  # "user" or "assistant"
+    content: str
+
+    def as_dict(self) -> dict[str, str]:
+        return {"role": self.role, "content": self.content}
+
+
+@dataclass(frozen=True)
+class AgentRequest:
+    """Everything an adapter passes to the agent for one turn."""
+
+    question: str
+    history: list[Message] = field(default_factory=list)
+    session_id: str = ""
+    """Stable across the turns of one case, unique per case and run."""
+    case_id: str = ""
+    turn: int = 0
+    params: dict[str, Any] = field(default_factory=dict)
+    """Config ``params`` merged with the current matrix combination."""
+
+
+class AgentErrorKind(StrEnum):
+    TIMEOUT = "timeout"
+    RATE_LIMITED = "rate_limited"
+    CONNECTION = "connection"
+    CRASH = "crash"
+    """The agent raised an exception or returned a server error."""
+    INVALID_RESPONSE = "invalid_response"
+    """The agent replied, but not with a valid result record."""
+    MISSING_OUTPUT = "missing_output"
+    """Pre-generated outputs have no entry for this turn."""
+
+
+RETRYABLE = frozenset(
+    {AgentErrorKind.TIMEOUT, AgentErrorKind.RATE_LIMITED, AgentErrorKind.CONNECTION}
+)
+
+
+class AgentError(Exception):
+    """The agent call failed. These are agent-side failures, never MindzEval bugs."""
+
+    def __init__(self, kind: AgentErrorKind, message: str) -> None:
+        super().__init__(message)
+        self.kind = kind
+        self.message = message
+
+    @property
+    def retryable(self) -> bool:
+        return self.kind in RETRYABLE
+
+
+class Adapter(ABC):
+    """Calls one agent. Implementations must be safe to call concurrently."""
+
+    name: str = "agent"
+
+    @abstractmethod
+    async def call(self, request: AgentRequest) -> AgentResponse:
+        """Return the agent's result for one turn, or raise :class:`AgentError`."""
+
+    async def aclose(self) -> None:  # noqa: B027 - optional hook
+        """Release resources (connections, clients)."""
+
+
+def coerce_response(value: Any) -> AgentResponse:
+    """Turn whatever an agent returned into an :class:`AgentResponse`, or raise AgentError."""
+    if isinstance(value, AgentResponse):
+        return value
+    if value is None:
+        raise AgentError(AgentErrorKind.INVALID_RESPONSE, "the agent returned nothing (None)")
+    if hasattr(value, "model_dump") and not isinstance(value, dict):
+        value = value.model_dump()
+    try:
+        return AgentResponse.model_validate(value)
+    except ValidationError as exc:
+        problems = "; ".join(
+            f"{'.'.join(str(p) for p in e['loc']) or 'response'}: "
+            f"{str(e['msg']).removeprefix('Value error, ')}"
+            for e in exc.errors(include_url=False)
+        )
+        raise AgentError(
+            AgentErrorKind.INVALID_RESPONSE, f"invalid result record: {problems}"
+        ) from None

mindzeval/adapters/http.py

@@ -0,0 +1,190 @@
+"""Call an agent over HTTP. Works for agents written in any language.
+
+Request body: by default ``{"question", "history", "session_id", "params"}`` as
+JSON. A custom ``body`` template is filled per call: a string that is exactly a
+placeholder (``"{history}"``) is replaced by the value itself, keeping its type;
+placeholders inside longer strings are substituted as text.
+
+Placeholders: ``{question}``, ``{history}``, ``{session_id}``, ``{case_id}``,
+``{turn}``, ``{params}`` and ``{params.<name>}``.
+
+Response: each result field is read from the JSON response with a dotted path
+(``data.rows``, ``choices.0.text``). Agent status values can be translated with
+``status_map``.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+import httpx
+
+from mindzeval.adapters.base import (
+    Adapter,
+    AgentError,
+    AgentErrorKind,
+    AgentRequest,
+    coerce_response,
+)
+from mindzeval.schema.config import HttpAgentConfig, ResponseMapping
+from mindzeval.schema.response import AgentResponse
+
+_PLACEHOLDER = re.compile(r"\{(question|history|session_id|case_id|turn|params(?:\.[\w.]+)?)\}")
+_MISSING = object()
+
+
+class HttpAdapter(Adapter):
+    def __init__(self, config: HttpAgentConfig, *, client: httpx.AsyncClient | None = None) -> None:
+        self._config = config
+        self.name = config.url
+        self._client = client or httpx.AsyncClient(timeout=config.timeout_s, follow_redirects=False)
+
+    async def call(self, request: AgentRequest) -> AgentResponse:
+        config = self._config
+        values = _template_values(request)
+        body = (
+            render_template(config.body, values)
+            if config.body is not None
+            else {
+                "question": request.question,
+                "history": values["history"],
+                "session_id": request.session_id,
+                "params": request.params,
+            }
+        )
+        headers = {"Accept": "application/json", **config.headers}
+        try:
+            if config.method == "GET":
+                params = body if isinstance(body, dict) else None
+                response = await self._client.get(
+                    config.url, params=_flat(params), headers=headers, timeout=config.timeout_s
+                )
+            else:
+                response = await self._client.request(
+                    config.method, config.url, json=body, headers=headers, timeout=config.timeout_s
+                )
+        except httpx.TimeoutException as exc:
+            raise AgentError(
+                AgentErrorKind.TIMEOUT, f"no response within {config.timeout_s:g}s"
+            ) from exc
+        except httpx.HTTPError as exc:
+            raise AgentError(AgentErrorKind.CONNECTION, f"{type(exc).__name__}: {exc}") from exc
+
+        if response.status_code == 429:
+            raise AgentError(AgentErrorKind.RATE_LIMITED, "the agent returned HTTP 429")
+        if response.status_code >= 400:
+            kind = (
+                AgentErrorKind.CRASH
+                if response.status_code >= 500
+                else AgentErrorKind.INVALID_RESPONSE
+            )
+            raise AgentError(kind, f"HTTP {response.status_code}: {response.text[:300]}")
+        try:
+            payload = response.json()
+        except (json.JSONDecodeError, ValueError):
+            raise AgentError(
+                AgentErrorKind.INVALID_RESPONSE,
+                f"the response is not JSON: {response.text[:200]!r}",
+            ) from None
+        return coerce_response(map_response(payload, config.response))
+
+    async def aclose(self) -> None:
+        await self._client.aclose()
+
+
+def map_response(payload: Any, mapping: ResponseMapping) -> dict[str, Any]:
+    """Pick each result field out of a JSON payload using the mapping's dotted paths."""
+    out: dict[str, Any] = {}
+    for field in (
+        "answer",
+        "sql",
+        "params",
+        "rows",
+        "columns",
+        "status",
+        "reason",
+        "tool_calls",
+        "trace",
+        "usage",
+    ):
+        path = getattr(mapping, field)
+        if not path:
+            continue
+        value = get_path(payload, path, _MISSING)
+        if value is _MISSING or value is None:
+            continue
+        out[field] = value
+
+    status = out.get("status")
+    if status is not None:
+        mapped = mapping.status_map.get(str(status))
+        if mapped is not None:
+            out["status"] = mapped.value
+        elif str(status).lower() in ("answer", "clarify", "decline"):
+            out["status"] = str(status).lower()
+        else:
+            raise AgentError(
+                AgentErrorKind.INVALID_RESPONSE,
+                f"unknown status {status!r}; map it with agent.response.status_map",
+            )
+    return out
+
+
+def get_path(data: Any, path: str, default: Any = None) -> Any:
+    """Follow a dotted path through dicts and lists (``a.b.0.c``)."""
+    current = data
+    for part in path.split("."):
+        if isinstance(current, dict) and part in current:
+            current = current[part]
+        elif (
+            isinstance(current, list)
+            and part.lstrip("-").isdigit()
+            and -len(current) <= int(part) < len(current)
+        ):
+            current = current[int(part)]
+        else:
+            return default
+    return current
+
+
+def render_template(template: Any, values: dict[str, Any]) -> Any:
+    if isinstance(template, dict):
+        return {k: render_template(v, values) for k, v in template.items()}
+    if isinstance(template, list):
+        return [render_template(v, values) for v in template]
+    if not isinstance(template, str):
+        return template
+    whole = _PLACEHOLDER.fullmatch(template)
+    if whole:
+        return _lookup(values, whole.group(1))
+
+    def text(match: re.Match[str]) -> str:
+        value = _lookup(values, match.group(1))
+        return value if isinstance(value, str) else json.dumps(value, default=str)
+
+    return _PLACEHOLDER.sub(text, template)
+
+
+def _template_values(request: AgentRequest) -> dict[str, Any]:
+    return {
+        "question": request.question,
+        "history": [m.as_dict() for m in request.history],
+        "session_id": request.session_id,
+        "case_id": request.case_id,
+        "turn": request.turn,
+        "params": dict(request.params),
+    }
+
+
+def _lookup(values: dict[str, Any], name: str) -> Any:
+    if name.startswith("params."):
+        return get_path(values["params"], name[len("params.") :])
+    return values[name]
+
+
+def _flat(params: dict[str, Any] | None) -> dict[str, str] | None:
+    if params is None:
+        return None
+    return {k: v if isinstance(v, str) else json.dumps(v, default=str) for k, v in params.items()}

mindzeval/adapters/outputs.py

@@ -0,0 +1,81 @@
+"""Score answers an agent produced earlier, without calling it.
+
+The file is JSONL (one record per line) or a JSON list. Each record names the
+case and turn, and holds the result record either under ``response`` or at the
+top level::
+
+    {"case_id": "customers_de", "turn": 0, "response": {"answer": "...", "sql": "..."}}
+    {"case_id": "order_count", "answer": "4", "sql": "SELECT count(*) FROM orders"}
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from mindzeval.adapters.base import (
+    Adapter,
+    AgentError,
+    AgentErrorKind,
+    AgentRequest,
+    coerce_response,
+)
+from mindzeval.errors import ConfigError, Issue
+from mindzeval.schema.response import AgentResponse
+
+
+class OutputsAdapter(Adapter):
+    def __init__(self, path: Path) -> None:
+        self.name = f"outputs:{path.name}"
+        self._records = _load(path)
+
+    async def call(self, request: AgentRequest) -> AgentResponse:
+        record = self._records.get((request.case_id, request.turn))
+        if record is None:
+            raise AgentError(
+                AgentErrorKind.MISSING_OUTPUT,
+                f"no saved output for case '{request.case_id}' turn {request.turn}",
+            )
+        return coerce_response(record)
+
+
+def _load(path: Path) -> dict[tuple[str, int], Any]:
+    source = str(path)
+    try:
+        text = path.read_text(encoding="utf-8")
+    except OSError as exc:
+        raise ConfigError([Issue(source, f"cannot read outputs file: {exc}")]) from None
+
+    items: list[tuple[int | None, Any]] = []
+    try:
+        if path.suffix.lower() == ".json":
+            loaded = json.loads(text)
+            if not isinstance(loaded, list):
+                raise ConfigError([Issue(source, "an outputs JSON file must contain a list")])
+            items = [(None, item) for item in loaded]
+        else:
+            items = [
+                (n, json.loads(line)) for n, line in enumerate(text.splitlines(), 1) if line.strip()
+            ]
+    except json.JSONDecodeError as exc:
+        raise ConfigError([Issue(source, f"invalid JSON: {exc.msg}", line=exc.lineno)]) from None
+
+    records: dict[tuple[str, int], Any] = {}
+    issues: list[Issue] = []
+    for line, item in items:
+        if not isinstance(item, dict) or "case_id" not in item:
+            issues.append(Issue(source, "each output needs a 'case_id'", line=line))
+            continue
+        key = (str(item["case_id"]), int(item.get("turn", 0)))
+        if key in records:
+            issues.append(
+                Issue(source, f"duplicate output for case '{key[0]}' turn {key[1]}", line=line)
+            )
+            continue
+        records[key] = item.get(
+            "response", {k: v for k, v in item.items() if k not in ("case_id", "turn")}
+        )
+    if issues:
+        raise ConfigError(issues)
+    return records

mindzeval/adapters/python.py

@@ -0,0 +1,140 @@
+"""Call an agent that is a Python function (LangChain, LangGraph, CrewAI, plain code...).
+
+The function can take any subset of these keyword arguments, and MindzEval
+passes only the ones it declares:
+
+    question: str            the question for this turn
+    history: list[dict]      earlier turns as {"role": "user"|"assistant", "content": str}
+    session_id: str          stable across the turns of one case
+    case_id: str, turn: int  identity of the turn being evaluated
+    params: dict             config params plus the current matrix combination
+
+Alternatively it can take a single ``request`` argument (an
+:class:`~mindzeval.adapters.AgentRequest`). It may be sync or async, and may
+return an :class:`~mindzeval.AgentResponse`, a dict, a pydantic model or a string.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import importlib
+import inspect
+import sys
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+from mindzeval.adapters.base import (
+    Adapter,
+    AgentError,
+    AgentErrorKind,
+    AgentRequest,
+    coerce_response,
+)
+from mindzeval.errors import ConfigError, Issue
+from mindzeval.schema.response import AgentResponse
+
+_ARGUMENTS = ("question", "history", "session_id", "case_id", "turn", "params")
+
+
+def load_target(target: str, base_dir: Path | None = None) -> Callable[..., Any]:
+    """Import ``module.path:attribute``. ``base_dir`` is put on ``sys.path`` first."""
+    module_name, _, attribute = target.partition(":")
+    if base_dir is not None and str(base_dir) not in sys.path:
+        sys.path.insert(0, str(base_dir))
+    try:
+        obj: Any = importlib.import_module(module_name)
+    except ImportError as exc:
+        raise ConfigError([Issue("agent.target", f"cannot import '{module_name}': {exc}")]) from exc
+    for part in attribute.split("."):
+        if not hasattr(obj, part):
+            raise ConfigError([Issue("agent.target", f"'{module_name}' has no '{attribute}'")])
+        obj = getattr(obj, part)
+    if not callable(obj):
+        raise ConfigError([Issue("agent.target", f"'{target}' is not callable")])
+    return obj  # type: ignore[no-any-return]
+
+
+class PythonAdapter(Adapter):
+    def __init__(self, func: Callable[..., Any], *, name: str | None = None) -> None:
+        self._func = func
+        self.name = name or str(getattr(func, "__qualname__", "python-agent"))
+        self._is_async = inspect.iscoroutinefunction(func) or inspect.iscoroutinefunction(
+            getattr(func, "__call__", None)  # noqa: B004 - callable objects
+        )
+        self._mode, self._accepted = _inspect_signature(func)
+
+    def _arguments(self, request: AgentRequest) -> dict[str, Any]:
+        if self._mode == "request":
+            return {"request": request}
+        values: dict[str, Any] = {
+            "question": request.question,
+            "history": [m.as_dict() for m in request.history],
+            "session_id": request.session_id,
+            "case_id": request.case_id,
+            "turn": request.turn,
+            "params": dict(request.params),
+        }
+        if self._mode == "kwargs":
+            return values
+        return {k: v for k, v in values.items() if k in self._accepted}
+
+    async def call(self, request: AgentRequest) -> AgentResponse:
+        kwargs = self._arguments(request)
+        try:
+            if self._is_async:
+                value = await self._func(**kwargs)
+            else:
+                value = await asyncio.to_thread(self._func, **kwargs)
+                if inspect.isawaitable(value):
+                    value = await value
+        except AgentError:
+            raise
+        except Exception as exc:  # the agent's own failure
+            raise AgentError(_classify(exc), f"{type(exc).__name__}: {exc}") from exc
+        return coerce_response(value)
+
+
+def _inspect_signature(func: Callable[..., Any]) -> tuple[str, frozenset[str]]:
+    try:
+        signature = inspect.signature(func)
+    except (TypeError, ValueError):
+        return "kwargs", frozenset()
+    params = signature.parameters
+    if any(p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values()):
+        return "kwargs", frozenset()
+    names = [n for n, p in params.items() if p.kind is not inspect.Parameter.VAR_POSITIONAL]
+    if names == ["request"]:
+        return "request", frozenset()
+    accepted = frozenset(n for n in names if n in _ARGUMENTS)
+    required = [
+        n
+        for n, p in params.items()
+        if p.default is inspect.Parameter.empty
+        and p.kind not in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD)
+    ]
+    unknown = [n for n in required if n not in _ARGUMENTS]
+    if unknown or "question" not in accepted:
+        raise ConfigError(
+            [
+                Issue(
+                    "agent.target",
+                    f"the agent function must accept 'question' (and optionally "
+                    f"{', '.join(_ARGUMENTS[1:])}); it has required parameters {required}",
+                )
+            ]
+        )
+    return "named", accepted
+
+
+def _classify(exc: BaseException) -> AgentErrorKind:
+    if isinstance(exc, TimeoutError):
+        return AgentErrorKind.TIMEOUT
+    status = getattr(exc, "status_code", None) or getattr(
+        getattr(exc, "response", None), "status_code", None
+    )
+    if status == 429 or "ratelimit" in type(exc).__name__.lower():
+        return AgentErrorKind.RATE_LIMITED
+    if isinstance(exc, ConnectionError):
+        return AgentErrorKind.CONNECTION
+    return AgentErrorKind.CRASH