agentops-accelerator 0.3.0__py3-none-any.whl

agentops/mcp/server.py

@@ -0,0 +1,232 @@
+"""AgentOps MCP server (stdio transport).
+
+This module is imported lazily - the ``mcp`` extra is optional. Importing
+this file triggers ``import mcp.server.fastmcp`` which fails with a clear
+message if the extra is not installed.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+
+def _build_server() -> Any:
+    """Construct and return the FastMCP server.
+
+    Imports of ``mcp`` happen inside the function so that ``agentops --help``
+    keeps working when the optional extra is not installed.
+    """
+    try:
+        from mcp.server.fastmcp import FastMCP
+    except ImportError as exc:  # pragma: no cover - exercised only without extra
+        raise RuntimeError(
+            "The MCP server requires the 'mcp' extra. "
+            "Install it with: pip install agentops-accelerator[mcp]"
+        ) from exc
+
+    server = FastMCP("agentops")
+
+    @server.tool()
+    def agentops_init(directory: str = ".", force: bool = False) -> Dict[str, Any]:
+        """Initialise an AgentOps workspace at ``directory``.
+
+        Creates ``agentops.yaml`` at the project root and a tiny seed
+        dataset under ``.agentops/data/smoke.jsonl``. Set ``force=True``
+        to overwrite existing files.
+        """
+        from agentops.services.initializer import initialize_flat_workspace
+
+        result = initialize_flat_workspace(Path(directory), force=force)
+        return {
+            "workspace_dir": str(result.workspace_dir),
+            "created_files": [str(p) for p in result.created_files],
+            "overwritten_files": [str(p) for p in result.overwritten_files],
+            "skipped_files": [str(p) for p in result.skipped_files],
+        }
+
+    @server.tool()
+    def agentops_eval_run(
+        config_path: Optional[str] = None,
+        output_dir: Optional[str] = None,
+        baseline: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Run an AgentOps evaluation and return a summary.
+
+        ``config_path`` defaults to ``agentops.yaml`` in the current
+        directory. ``output_dir`` defaults to
+        ``.agentops/results/latest``. ``baseline`` is an optional path to
+        a previous ``results.json`` for comparison.
+        """
+        from agentops.core.config_loader import load_agentops_config
+        from agentops.pipeline.orchestrator import (
+            RunOptions,
+            exit_code_from,
+            run_evaluation,
+        )
+
+        config = Path(config_path) if config_path else Path("agentops.yaml")
+        if not config.exists():
+            return {
+                "ok": False,
+                "exit_code": 1,
+                "error": f"config not found at {config}",
+            }
+        try:
+            config_obj = load_agentops_config(config)
+        except Exception as exc:  # noqa: BLE001
+            return {
+                "ok": False,
+                "exit_code": 1,
+                "error": f"failed to load config: {exc}",
+            }
+
+        out_dir = (
+            Path(output_dir)
+            if output_dir
+            else config.parent / ".agentops" / "results" / "latest"
+        )
+        options = RunOptions(
+            config_path=config.resolve(),
+            output_dir=out_dir,
+            baseline_path=Path(baseline).resolve() if baseline else None,
+        )
+        try:
+            run = run_evaluation(config_obj, options=options)
+        except Exception as exc:  # noqa: BLE001
+            return {"ok": False, "exit_code": 1, "error": f"evaluation failed: {exc}"}
+
+        return {
+            "ok": True,
+            "exit_code": exit_code_from(run),
+            "output_dir": str(out_dir),
+            "results_json": str(out_dir / "results.json"),
+            "report_md": str(out_dir / "report.md"),
+            "passed": bool(run.summary.overall_passed),
+            "metrics": dict(run.aggregate_metrics or {}),
+        }
+
+    @server.tool()
+    def agentops_report_show(report_path: Optional[str] = None) -> Dict[str, Any]:
+        """Return the contents of ``report.md`` for a finished run.
+
+        Defaults to ``.agentops/results/latest/report.md``.
+        """
+        path = (
+            Path(report_path)
+            if report_path
+            else Path(".agentops/results/latest/report.md")
+        )
+        if not path.exists():
+            return {"ok": False, "error": f"report not found at {path}"}
+        return {
+            "ok": True,
+            "path": str(path),
+            "markdown": path.read_text(encoding="utf-8"),
+        }
+
+    @server.tool()
+    def agentops_results_summary(results_path: Optional[str] = None) -> Dict[str, Any]:
+        """Return a compact JSON summary extracted from ``results.json``.
+
+        Defaults to ``.agentops/results/latest/results.json``.
+        """
+        path = (
+            Path(results_path)
+            if results_path
+            else Path(".agentops/results/latest/results.json")
+        )
+        if not path.exists():
+            return {"ok": False, "error": f"results not found at {path}"}
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+        except Exception as exc:  # noqa: BLE001
+            return {"ok": False, "error": f"failed to parse results.json: {exc}"}
+
+        summary = data.get("summary") or {}
+        return {
+            "ok": True,
+            "path": str(path),
+            "version": data.get("version"),
+            "target": data.get("target"),
+            "rows": len(data.get("rows") or []),
+            "metrics": summary.get("metrics") or {},
+            "thresholds": summary.get("thresholds") or {},
+            "overall_passed": summary.get("overall_passed"),
+        }
+
+    @server.tool()
+    def agentops_dataset_add(
+        dataset_path: str,
+        rows: List[Dict[str, Any]],
+    ) -> Dict[str, Any]:
+        """Append JSONL rows to ``dataset_path``.
+
+        Creates the parent directory if needed. Each row must be a JSON
+        object - typical keys are ``input``, ``expected``, ``context``,
+        and ``tool_calls`` depending on the agent type.
+        """
+        if not isinstance(rows, list) or not all(isinstance(r, dict) for r in rows):
+            return {"ok": False, "error": "rows must be a list of JSON objects"}
+        path = Path(dataset_path)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        with path.open("a", encoding="utf-8") as fh:
+            for row in rows:
+                fh.write(json.dumps(row, ensure_ascii=False) + "\n")
+        return {"ok": True, "path": str(path), "appended": len(rows)}
+
+    @server.tool()
+    def agentops_list_runs(workspace_dir: str = ".") -> Dict[str, Any]:
+        """List historical runs under ``<workspace_dir>/.agentops/results/``."""
+        results_dir = Path(workspace_dir) / ".agentops" / "results"
+        if not results_dir.is_dir():
+            return {"ok": True, "runs": []}
+        runs: List[Dict[str, Any]] = []
+        for entry in sorted(results_dir.iterdir(), key=lambda p: p.name):
+            if not entry.is_dir():
+                continue
+            results_json = entry / "results.json"
+            run_info: Dict[str, Any] = {
+                "name": entry.name,
+                "path": str(entry),
+                "has_results": results_json.exists(),
+            }
+            if results_json.exists():
+                try:
+                    payload = json.loads(results_json.read_text(encoding="utf-8"))
+                    summary = payload.get("summary") or {}
+                    run_info["overall_passed"] = summary.get("overall_passed")
+                    run_info["metrics"] = summary.get("metrics") or {}
+                except Exception:  # noqa: BLE001
+                    pass
+            runs.append(run_info)
+        return {"ok": True, "runs": runs}
+
+    @server.tool()
+    def agentops_workflow_init(
+        directory: str = ".",
+        force: bool = False,
+    ) -> Dict[str, Any]:
+        """Generate GitHub Actions workflows for AgentOps evaluation."""
+        from agentops.services.cicd import generate_cicd_workflows
+
+        result = generate_cicd_workflows(directory=Path(directory), force=force)
+        return {
+            "ok": True,
+            "created_files": [str(p) for p in result.created_files],
+            "overwritten_files": [str(p) for p in result.overwritten_files],
+            "skipped_files": [str(p) for p in result.skipped_files],
+        }
+
+    return server
+
+
+def serve_stdio() -> None:
+    """Entry point for ``agentops mcp serve``.
+
+    Builds the FastMCP server and runs it on the stdio transport. This
+    function blocks until the client disconnects.
+    """
+    server = _build_server()
+    server.run()

agentops/pipeline/__init__.py

@@ -0,0 +1,8 @@
+"""AgentOps 1.0 evaluation pipeline.
+
+Public entry point: :func:`agentops.pipeline.orchestrator.run_evaluation`.
+"""
+
+from agentops.pipeline.orchestrator import run_evaluation
+
+__all__ = ["run_evaluation"]

agentops/pipeline/cloud_results.py

@@ -0,0 +1,189 @@
+"""Map Foundry cloud eval output items into AgentOps result shapes.
+
+When ``execution: cloud`` is used in ``agentops.yaml``, the agent and
+evaluators run server-side via the Foundry / OpenAI Evals API. We then
+download per-row ``output_items`` from Foundry and reshape them into the
+same :class:`RowResult` / :class:`RunResult` schema that local execution
+produces, so downstream consumers (``report.md``, ``--baseline`` diffing,
+CI gates) behave identically regardless of where the run executed.
+
+The cloud output schema is intentionally loose: we accept multiple field
+spellings (``output_text`` / ``output`` / ``message``; ``score`` /
+``value`` / ``passed``) and fall back gracefully when a field is absent.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any, Dict, List, Optional
+
+from agentops.core.results import RowMetric, RowResult
+
+
+def rows_from_cloud_output_items(
+    output_items: List[Dict[str, Any]],
+) -> List[RowResult]:
+    """Build a list of :class:`RowResult` from raw Foundry output items.
+
+    ``output_items`` is the list returned by
+    ``cloud_runner._list_output_items``. Each item is a dict with at
+    least ``datasource_item``, ``sample`` and ``results``; missing keys
+    yield blank fields rather than raising.
+    """
+    rows: List[RowResult] = []
+    for index, item in enumerate(output_items):
+        rows.append(_row_from_item(index, item))
+    return rows
+
+
+def _row_from_item(index: int, item: Dict[str, Any]) -> RowResult:
+    datasource = _as_dict(item.get("datasource_item")) or {}
+    sample = _as_dict(item.get("sample")) or {}
+    results = item.get("results") or []
+
+    metrics: List[RowMetric] = []
+    if isinstance(results, list):
+        for entry in results:
+            metric = _metric_from_result(entry)
+            if metric is not None:
+                metrics.append(metric)
+
+    return RowResult(
+        row_index=index,
+        input=_as_str(datasource.get("input")),
+        expected=_optional_str(datasource.get("expected")),
+        response=_extract_response_text(sample),
+        context=_optional_str(datasource.get("context")),
+        latency_seconds=None,  # Foundry-side latency is not client-perceived.
+        tool_calls=datasource.get("tool_calls") if isinstance(datasource.get("tool_calls"), list) else None,
+        metrics=metrics,
+        error=_extract_item_error(item),
+    )
+
+
+def _metric_from_result(entry: Any) -> Optional[RowMetric]:
+    if not isinstance(entry, dict):
+        return None
+    name = entry.get("name") or entry.get("metric")
+    if not isinstance(name, str) or not name:
+        return None
+    score = _coerce_float(
+        entry.get("score"),
+        entry.get("value"),
+        entry.get("result"),
+    )
+    if score is None and isinstance(entry.get("passed"), bool):
+        score = 1.0 if entry["passed"] else 0.0
+    reason = entry.get("reason") if isinstance(entry.get("reason"), str) else None
+    err = entry.get("error") if isinstance(entry.get("error"), str) else None
+    return RowMetric(name=name, value=score, error=err, reason=reason)
+
+
+def _extract_response_text(sample: Dict[str, Any]) -> str:
+    """Reach into a Foundry sample payload and pull a plain text response.
+
+    Foundry's sample shape varies: sometimes the response is a clean string
+    under ``output_text``, sometimes it's a list of output items under
+    ``output`` / ``output_items``, and occasionally ``output_text`` is set
+    to a JSON-encoded version of the structured output. Try structured
+    fields first (they're authoritative), and recurse into JSON-encoded
+    strings rather than passing them through as the response.
+    """
+    # 1. Structured fields are authoritative. Walk them first.
+    for key in ("output", "messages", "output_items"):
+        text = _text_from_structured(sample.get(key))
+        if text:
+            return text
+
+    # 2. Flat string fields. If the value looks like JSON, parse and recurse.
+    for key in ("output_text", "text", "content"):
+        value = sample.get(key)
+        if isinstance(value, str) and value:
+            stripped = value.strip()
+            if stripped.startswith("[") or stripped.startswith("{"):
+                try:
+                    parsed = json.loads(stripped)
+                except (ValueError, TypeError):
+                    return value
+                if isinstance(parsed, list):
+                    text = _text_from_structured(parsed)
+                    if text:
+                        return text
+                elif isinstance(parsed, dict):
+                    return _extract_response_text(parsed)
+                # Fall through to raw value if we couldn't extract.
+            return value
+        if isinstance(value, list):
+            text = _text_from_structured(value)
+            if text:
+                return text
+    return ""
+
+
+def _text_from_structured(value: Any) -> str:
+    """Walk a list-of-dicts (output / messages / output_items shape) and
+    return the first textual payload encountered, or ``""`` when none is
+    found. Iterates in reverse so the assistant's final message wins.
+    """
+    if not isinstance(value, list):
+        return ""
+    for entry in reversed(value):
+        if not isinstance(entry, dict):
+            continue
+        # Try flat text fields first.
+        for field in ("output_text", "text"):
+            candidate = entry.get(field)
+            if isinstance(candidate, str) and candidate:
+                return candidate
+        # Some shapes nest under "content" as either a string or a list of
+        # content blocks (OpenAI Responses API: content: [{type, text}, ...]).
+        nested = entry.get("content")
+        if isinstance(nested, str) and nested:
+            return nested
+        if isinstance(nested, list):
+            text = _text_from_structured(nested)
+            if text:
+                return text
+    return ""
+
+
+def _extract_item_error(item: Dict[str, Any]) -> Optional[str]:
+    err = item.get("error")
+    if isinstance(err, str) and err:
+        return err
+    if isinstance(err, dict):
+        msg = err.get("message") or err.get("error")
+        if isinstance(msg, str) and msg:
+            return msg
+    status = item.get("status")
+    if isinstance(status, str) and status.lower() in {"failed", "error"}:
+        return f"output item status: {status}"
+    return None
+
+
+def _as_dict(value: Any) -> Optional[Dict[str, Any]]:
+    return value if isinstance(value, dict) else None
+
+
+def _as_str(value: Any) -> str:
+    return value if isinstance(value, str) else ""
+
+
+def _optional_str(value: Any) -> Optional[str]:
+    return value if isinstance(value, str) and value else None
+
+
+def _coerce_float(*candidates: Any) -> Optional[float]:
+    for value in candidates:
+        if value is None:
+            continue
+        if isinstance(value, bool):
+            return 1.0 if value else 0.0
+        if isinstance(value, (int, float)):
+            return float(value)
+        if isinstance(value, str):
+            try:
+                return float(value)
+            except ValueError:
+                continue
+    return None