graph-agents-cli 0.3.1__py3-none-any.whl

graph_agents_cli/eval/_common.py

@@ -0,0 +1,177 @@
+# Copyright 2026 graph-agents-cli contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Shared helpers for the eval commands: exit codes, hashing, project metadata."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import tomllib
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+import click
+import yaml
+
+# Exit codes of the evaluation gate (see gate.py).
+EXIT_OK = 0
+EXIT_GATE_FAILED = 1
+EXIT_INCOMPLETE = 2
+EXIT_CONFIG_ERROR = 3
+
+MANIFEST_FILENAME = "graph-agents-cli-manifest.yaml"
+
+
+class EvalConfigError(click.ClickException):
+    """A configuration error: unknown metric, unreachable judge, bad dataset (exit 3)."""
+
+    exit_code = EXIT_CONFIG_ERROR
+
+
+def worst_exit_code(*codes: int) -> int:
+    """The worse of several gate exit codes (3 > 2 > 1 > 0)."""
+    return max(codes) if codes else EXIT_OK
+
+
+def utc_now_iso() -> str:
+    return datetime.now(UTC).isoformat(timespec="seconds")
+
+
+def canonical_hash(obj: Any) -> str:
+    """sha256 of the canonical JSON encoding of ``obj`` (sorted keys, no whitespace)."""
+    encoded = json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+    return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
+
+
+def load_json_file(path: Path, what: str = "file") -> Any:
+    """Read a JSON file, raising :class:`EvalConfigError` when unreadable or invalid."""
+    try:
+        return json.loads(Path(path).read_text(encoding="utf-8"))
+    except FileNotFoundError as exc:
+        raise EvalConfigError(f"{what} not found: {path}") from exc
+    except (OSError, json.JSONDecodeError) as exc:
+        raise EvalConfigError(f"{what} is not valid JSON ({path}): {exc}") from exc
+
+
+def write_json_file(path: Path, data: Any) -> None:
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+
+
+def project_env(project_root: Path | None) -> dict[str, str]:
+    """The project's ``.env`` values overlaid with the process environment.
+
+    Only the keys present in one of the two sources are returned. The process
+    environment wins so ``JUDGE_MODEL_PROVIDER=fake graph-agents-cli eval grade``
+    behaves as expected.
+    """
+    values: dict[str, str] = {}
+    if project_root is not None:
+        env_file = Path(project_root) / ".env"
+        if env_file.is_file():
+            from dotenv import dotenv_values
+
+            for key, value in dotenv_values(env_file).items():
+                if value is not None:
+                    values[key] = value
+    for key, value in os.environ.items():
+        if key in values or key.startswith(("MODEL_", "JUDGE_", "TRACE_", "LANGSMITH_", "API_")):
+            values[key] = value
+    return values
+
+
+def read_manifest(project_root: Path | None) -> dict[str, Any]:
+    """The raw project manifest, or ``{}`` when absent or unreadable."""
+    if project_root is None:
+        return {}
+    path = Path(project_root) / MANIFEST_FILENAME
+    if not path.is_file():
+        return {}
+    try:
+        data = yaml.safe_load(path.read_text(encoding="utf-8"))
+    except (OSError, yaml.YAMLError):
+        return {}
+    return data if isinstance(data, dict) else {}
+
+
+def project_meta(project_root: Path | None) -> dict[str, Any]:
+    """Identity recorded in traces and results: name, agent directory, model, version.
+
+    Reads the manifest (``create_params.model_provider`` / ``model``), the
+    project's ``.env`` (``MODEL_PROVIDER`` / ``MODEL_NAME`` win when set) and
+    ``pyproject.toml`` (``project.version``). Every field degrades to ``None``
+    outside a project so ``eval grade --traces ... --output ...`` works anywhere.
+    """
+    manifest = read_manifest(project_root)
+    create_params = manifest.get("create_params") or {}
+    if not isinstance(create_params, dict):
+        create_params = {}
+    env = project_env(project_root)
+
+    provider = env.get("MODEL_PROVIDER") or create_params.get("model_provider")
+    model_name = env.get("MODEL_NAME") or create_params.get("model")
+    version = None
+    if project_root is not None:
+        pyproject = Path(project_root) / "pyproject.toml"
+        if pyproject.is_file():
+            try:
+                with open(pyproject, "rb") as f:
+                    version = (tomllib.load(f).get("project") or {}).get("version")
+            except (OSError, tomllib.TOMLDecodeError):
+                version = None
+
+    return {
+        "name": manifest.get("name"),
+        "agent_directory": manifest.get("agent_directory") or "app",
+        "runtime": create_params.get("runtime"),
+        "model_provider": provider,
+        "model_name": model_name,
+        "model": f"{provider}/{model_name}"
+        if provider and model_name
+        else (model_name or provider),
+        "agent_version": str(version) if version is not None else None,
+        "capture": env.get("TRACE_CAPTURE") or "metadata",
+    }
+
+
+def resolve_judge_identity(
+    project_root: Path | None,
+    *,
+    config_provider: str | None,
+    config_model: str | None,
+    flag_provider: str | None,
+    flag_model: str | None,
+) -> dict[str, str | None]:
+    """The judge provider/model to record and to hand the runner.
+
+    Precedence: CLI flag > ``eval_config.yaml`` ``judge:`` > ``JUDGE_MODEL_*``
+    in ``.env`` / environment > the agent's ``MODEL_PROVIDER`` / ``MODEL_NAME``.
+    ``None`` means "let ``get_judge_model()`` decide".
+    """
+    env = project_env(project_root)
+    provider = (
+        flag_provider
+        or config_provider
+        or env.get("JUDGE_MODEL_PROVIDER")
+        or env.get("MODEL_PROVIDER")
+        or None
+    )
+    model = (
+        flag_model or config_model or env.get("JUDGE_MODEL_NAME") or env.get("MODEL_NAME") or None
+    )
+    return {"provider": provider, "model": model}

graph_agents_cli/eval/_judge.py

@@ -0,0 +1,168 @@
+# Copyright 2026 graph-agents-cli contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Stage and run the judge runner inside the project's environment.
+
+The CLI never imports LangChain. ``_judge_runner.py`` is copied to
+``<project>/.graph-agents-cli/judge_runner.py`` and executed with
+``uv run python .graph-agents-cli/judge_runner.py <input.json> <output.json>``
+from the project root, so it resolves ``<agent_directory>.app_utils.model.get_judge_model()``
+and ``langchain_core`` from the project's own lock. Prompts are rendered here;
+the runner only invokes the model (or a custom metric callable) and parses
+scores.
+
+Input payload (``agent_directory`` is added from the manifest)::
+
+    {"judge": {"provider": null, "model": null}, "agent_directory": "app",
+     "items": [{"id": "greeting/response_quality", "kind": "judge",
+                "case_id": "greeting", "metric": "response_quality",
+                "prompt": "...", "scale": 5},
+               {"id": "greeting/my_metric", "kind": "custom", "case_id": "...",
+                "metric": "my_metric", "callable": "module:function",
+                "case": {...}, "trace": {...}},
+               {"id": "summary", "kind": "summarize", "prompt": "..."}]}
+
+Output payload::
+
+    {"provider": "...", "model": "...",
+     "results": [{"id": "...", "case_id": "...", "metric": "...",
+                  "score": 4, "reasoning": "...", "error": null}]}
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+from importlib import resources
+from pathlib import Path
+from typing import Any
+
+from graph_agents_cli.eval import _paths
+from graph_agents_cli.eval._common import (
+    EvalConfigError,
+    load_json_file,
+    read_manifest,
+    write_json_file,
+)
+
+RUNNER_SOURCE = "_judge_runner.py"
+RUNNER_STAGED_NAME = "judge_runner.py"
+DEFAULT_TIMEOUT = 600  # seconds
+
+
+def stage_runner(project_root: Path) -> Path:
+    """Copy the runner into ``<project>/.graph-agents-cli/judge_runner.py``.
+
+    Loaded through :mod:`importlib.resources` so it works from a wheel or an
+    editable install. Overwritten on every run; the file is not removed
+    afterwards (it lives in the run-state directory, which is gitignored by
+    the template).
+    """
+    dest_dir = _paths.stage_dir(project_root)
+    dest = dest_dir / RUNNER_STAGED_NAME
+    src = resources.files("graph_agents_cli.eval").joinpath(RUNNER_SOURCE)
+    with resources.as_file(src) as src_path:
+        shutil.copyfile(src_path, dest)
+    return dest
+
+
+def run_judge_runner(
+    project_root: Path,
+    payload: dict[str, Any],
+    *,
+    timeout: int = DEFAULT_TIMEOUT,
+) -> dict[str, Any]:
+    """Execute the runner over ``payload`` and return its parsed output.
+
+    A runner that cannot start, cannot build the judge model, or exits
+    non-zero is an *unreachable judge*: :class:`EvalConfigError` (exit 3).
+    Per-item failures come back in ``results[*].error`` and are the caller's
+    to map onto case statuses.
+    """
+    from graph_agents_cli._runner import run_resolved
+
+    project_root = Path(project_root)
+    # The runner imports the judge model from the project's agent package, which
+    # `create --agent-directory` names (`app` by default).
+    payload = {
+        **payload,
+        "agent_directory": read_manifest(project_root).get("agent_directory") or "app",
+    }
+    script = stage_runner(project_root)
+    stamp = _paths.timestamp()
+    stage = _paths.stage_dir(project_root)
+    input_path = stage / f"judge_input_{stamp}.json"
+    output_path = stage / f"judge_output_{stamp}.json"
+    write_json_file(input_path, payload)
+
+    rel_script = script.relative_to(project_root)
+    cmd = [
+        "uv",
+        "run",
+        "python",
+        str(rel_script),
+        str(input_path.relative_to(project_root)),
+        str(output_path.relative_to(project_root)),
+    ]
+    # The template's get_judge_model() reads JUDGE_MODEL_PROVIDER / JUDGE_MODEL_NAME;
+    # the runner sets them as well, but a child that loads .env at import must
+    # already see the resolved values.
+    env = dict(os.environ)
+    judge = payload.get("judge") or {}
+    if judge.get("provider"):
+        env["JUDGE_MODEL_PROVIDER"] = str(judge["provider"])
+    if judge.get("model"):
+        env["JUDGE_MODEL_NAME"] = str(judge["model"])
+    try:
+        result = run_resolved(
+            cmd,
+            cwd=str(project_root),
+            capture_output=True,
+            text=True,
+            encoding="utf-8",
+            errors="replace",
+            timeout=timeout,
+            env=env,
+        )
+    except subprocess.TimeoutExpired as exc:
+        raise EvalConfigError(
+            f"judge runner timed out after {timeout}s; the judge model may be unreachable"
+        ) from exc
+    except Exception as exc:  # ToolNotFoundError (uv missing), OSError
+        raise EvalConfigError(f"judge runner could not start: {exc}") from exc
+
+    if result.returncode != 0:
+        detail = "\n".join(
+            part.strip() for part in (result.stdout, result.stderr) if part and part.strip()
+        )
+        raise EvalConfigError(
+            f"judge runner failed (exit code {result.returncode}); the judge model is "
+            f"unreachable or misconfigured.\n{detail}".rstrip()
+        )
+    if not output_path.exists():
+        raise EvalConfigError("judge runner exited 0 but wrote no output file")
+    output = load_json_file(output_path, "judge runner output")
+    if not isinstance(output, dict) or not isinstance(output.get("results"), list):
+        raise EvalConfigError("judge runner output is malformed (no 'results' list)")
+    try:
+        input_path.unlink()
+        output_path.unlink()
+    except OSError:
+        pass
+    return output
+
+
+def results_by_id(output: dict[str, Any]) -> dict[str, dict[str, Any]]:
+    return {str(r.get("id")): r for r in output.get("results", []) if isinstance(r, dict)}

graph_agents_cli/eval/_judge_runner.py

@@ -0,0 +1,238 @@
+# Copyright 2026 graph-agents-cli contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Judge runner for ``graph-agents-cli eval grade`` and ``eval analyze --judge``.
+
+This file is staged into the user's project as ``.graph-agents-cli/judge_runner.py``
+and executed there with ``uv run python .graph-agents-cli/judge_runner.py
+<input.json> <output.json>`` so it runs inside the project's environment.
+Do not edit the staged copy; it is overwritten on every run.
+
+It imports ``<agent_directory>.app_utils.model.get_judge_model()`` (the
+payload's ``agent_directory``, the project's agent package, ``app`` by default;
+the template contract: no arguments, judge chosen from ``JUDGE_MODEL_PROVIDER`` /
+``JUDGE_MODEL_NAME``)
+and ``langchain_core`` lazily; under the ``fake`` provider it returns a
+deterministic score equal to the scale so tests and CI run without keys.
+It never imports graph_agents_cli.
+"""
+
+from __future__ import annotations
+
+import importlib
+import json
+import os
+import re
+import sys
+import traceback
+from pathlib import Path
+from typing import Any
+
+FAKE_PROVIDER = "fake"
+
+
+def _load_dotenv() -> None:
+    try:
+        from dotenv import load_dotenv  # the template depends on python-dotenv
+    except ImportError:
+        return
+    load_dotenv(Path.cwd() / ".env")
+
+
+def resolve_provider(payload: dict[str, Any]) -> str | None:
+    judge = payload.get("judge") or {}
+    return (
+        judge.get("provider")
+        or os.environ.get("JUDGE_MODEL_PROVIDER")
+        or os.environ.get("MODEL_PROVIDER")
+        or None
+    )
+
+
+def is_fake(provider: str | None) -> bool:
+    return (provider or "").lower() == FAKE_PROVIDER
+
+
+# A Python package name: what `create --agent-directory` accepts.
+_PACKAGE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+
+
+def agent_package(payload: dict[str, Any]) -> str:
+    """The project's agent package (the manifest's ``agent_directory``), ``app`` by default."""
+    name = payload.get("agent_directory")
+    return name if isinstance(name, str) and _PACKAGE.fullmatch(name) else "app"
+
+
+def build_model(provider: str | None, model: str | None, package: str = "app") -> Any:
+    """``<package>.app_utils.model.get_judge_model()`` from the project.
+
+    The template resolves the judge from ``JUDGE_MODEL_PROVIDER`` /
+    ``JUDGE_MODEL_NAME`` (falling back to ``MODEL_*``) when called, so the
+    overrides the CLI resolved travel through the environment, set after the
+    import so an import-time ``.env`` load cannot shadow them.
+    """
+    if str(Path.cwd()) not in sys.path:
+        sys.path.insert(0, str(Path.cwd()))
+    module = importlib.import_module(f"{package}.app_utils.model")
+    if provider:
+        os.environ["JUDGE_MODEL_PROVIDER"] = provider
+    if model:
+        os.environ["JUDGE_MODEL_NAME"] = model
+    return module.get_judge_model()
+
+
+def invoke_model(model: Any, prompt: str) -> str:
+    """Send ``prompt`` to a LangChain chat model and return the text of its reply."""
+    try:
+        from langchain_core.messages import HumanMessage
+
+        result = model.invoke([HumanMessage(content=prompt)])
+    except ImportError:
+        result = model.invoke(prompt)
+    content = getattr(result, "content", result)
+    if isinstance(content, list):
+        parts = []
+        for part in content:
+            if isinstance(part, dict):
+                parts.append(str(part.get("text", "")))
+            else:
+                parts.append(str(part))
+        content = "".join(parts)
+    return str(content)
+
+
+_JSON_OBJECT = re.compile(r"\{.*\}", re.DOTALL)
+_SCORE = re.compile(r"score\W{0,5}(-?\d+(?:\.\d+)?)", re.IGNORECASE)
+_NUMBER = re.compile(r"-?\d+(?:\.\d+)?")
+
+
+def parse_score(text: str) -> tuple[float, str]:
+    """Extract ``(score, reasoning)`` from a judge reply.
+
+    Accepts a JSON object (optionally fenced or embedded in prose) with a
+    ``score`` key, a ``score: N`` phrase, or, as a last resort, a bare number.
+    Raises ``ValueError`` when no score can be read.
+    """
+    stripped = text.strip()
+    candidates = [stripped]
+    match = _JSON_OBJECT.search(stripped)
+    if match:
+        candidates.append(match.group(0))
+    for candidate in candidates:
+        try:
+            data = json.loads(candidate)
+        except (json.JSONDecodeError, ValueError):
+            continue
+        if isinstance(data, dict) and "score" in data:
+            score = float(data["score"])
+            reasoning = data.get("reasoning") or data.get("explanation") or ""
+            return score, str(reasoning)
+        if isinstance(data, int | float) and not isinstance(data, bool):
+            return float(data), ""
+    match = _SCORE.search(stripped)
+    if match:
+        return float(match.group(1)), stripped
+    match = _NUMBER.fullmatch(stripped)
+    if match:
+        return float(match.group(0)), ""
+    raise ValueError(f"could not parse a score from judge reply: {stripped[:200]!r}")
+
+
+def run_custom(item: dict[str, Any]) -> tuple[float, str]:
+    """Import ``module:function`` from the project and call it with ``(case, trace)``."""
+    target = item["callable"]
+    module_name, _, func_name = target.partition(":")
+    if str(Path.cwd()) not in sys.path:
+        sys.path.insert(0, str(Path.cwd()))
+    module = importlib.import_module(module_name)
+    func = getattr(module, func_name)
+    result = func(item.get("case") or {}, item.get("trace") or {})
+    if isinstance(result, bool):
+        return (1.0 if result else 0.0), ""
+    if isinstance(result, int | float):
+        return float(result), ""
+    if isinstance(result, dict) and "score" in result:
+        reasoning = result.get("reasoning") or result.get("explanation") or ""
+        score = result["score"]
+        if isinstance(score, bool):
+            score = 1.0 if score else 0.0
+        return float(score), str(reasoning)
+    raise ValueError(
+        f"{target} returned {type(result).__name__}; expected bool, number, or {{'score': ...}}"
+    )
+
+
+def run_items(payload: dict[str, Any], model: Any = None, *, fake: bool = False) -> dict[str, Any]:
+    """Score every item; per-item failures become ``error`` entries, never exceptions."""
+    results: list[dict[str, Any]] = []
+    for item in payload.get("items", []):
+        entry: dict[str, Any] = {
+            "id": item.get("id"),
+            "case_id": item.get("case_id"),
+            "metric": item.get("metric"),
+            "score": None,
+            "reasoning": "",
+            "error": None,
+        }
+        kind = item.get("kind", "judge")
+        try:
+            if kind == "custom":
+                entry["score"], entry["reasoning"] = run_custom(item)
+            elif kind == "summarize":
+                if fake:
+                    entry["reasoning"] = "fake provider: no summary generated"
+                else:
+                    entry["reasoning"] = invoke_model(model, item["prompt"])
+            else:
+                scale = float(item.get("scale") or 5)
+                if fake:
+                    entry["score"] = scale
+                    entry["reasoning"] = "fake provider: deterministic score"
+                else:
+                    text = invoke_model(model, item["prompt"])
+                    entry["score"], entry["reasoning"] = parse_score(text)
+        except Exception as exc:
+            entry["error"] = f"{type(exc).__name__}: {exc}"
+        results.append(entry)
+    return {"results": results}
+
+
+def main(argv: list[str]) -> int:
+    if len(argv) != 3:
+        print("usage: judge_runner.py <input.json> <output.json>", file=sys.stderr)
+        return 2
+    input_path, output_path = Path(argv[1]), Path(argv[2])
+    payload = json.loads(input_path.read_text(encoding="utf-8"))
+    _load_dotenv()
+    provider = resolve_provider(payload)
+    model_name = (payload.get("judge") or {}).get("model") or os.environ.get("JUDGE_MODEL_NAME")
+    fake = is_fake(provider)
+    needs_model = any(i.get("kind", "judge") != "custom" for i in payload.get("items", []))
+    model = None
+    if needs_model and not fake:
+        try:
+            model = build_model(provider, model_name, agent_package(payload))
+        except Exception:
+            print("judge runner: could not build the judge model", file=sys.stderr)
+            traceback.print_exc()
+            return 1
+    output = run_items(payload, model, fake=fake)
+    output["provider"] = provider
+    output["model"] = model_name or (getattr(model, "model_name", None) if model else None)
+    output_path.write_text(json.dumps(output, indent=2), encoding="utf-8")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))