fred-deepeval-cli 0.1.0__tar.gz

fred_deepeval_cli-0.1.0/PKG-INFO

@@ -0,0 +1,70 @@
+Metadata-Version: 2.4
+Name: fred-deepeval-cli
+Version: 0.1.0
+Summary: External CLI for evaluating Fred agent turns via /agents/evaluate
+License: Apache-2.0
+Requires-Python: <3.13,>=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: fred-sdk>=2.0.7
+Requires-Dist: fred-runtime>=2.0.8
+Provides-Extra: dev
+Requires-Dist: bandit>=1.8.6; extra == "dev"
+Requires-Dist: basedpyright==1.31.0; extra == "dev"
+Requires-Dist: detect-secrets>=1.5.0; extra == "dev"
+Requires-Dist: pytest>=8.4.2; extra == "dev"
+Requires-Dist: pytest-cov>=6.2.1; extra == "dev"
+Requires-Dist: pytest-socket>=0.7.0; extra == "dev"
+Requires-Dist: ruff>=0.12.5; extra == "dev"
+Provides-Extra: eval
+Requires-Dist: deepeval; extra == "eval"
+Requires-Dist: litellm; extra == "eval"
+Requires-Dist: python-dotenv; extra == "eval"
+Requires-Dist: rich>=13.0; extra == "eval"
+Requires-Dist: temporalio; extra == "eval"
+
+# fred-deepeval-cli
+
+External CLI for evaluating one Fred agent turn through `POST /agents/evaluate`.
+
+## Purpose
+
+This project provides a small external CLI that:
+- calls a Fred pod `/agents/evaluate` endpoint
+- receives an `EvalTrace`
+- classifies the turn outcome
+- resolves an evaluation preset from `agent_tags`
+- computes structural checks
+- scores the trace with DeepEval
+
+## Commands
+
+```bash
+make dev
+make eval-dev
+make test
+make code-quality
+make cli
+make score BASE_URL=http://127.0.0.1:8000/fred/agents/v2 AGENT_ID=fred.test.assistant INPUT="echo bonjour" SESSION_ID=eval-001 USER_ID=alice
+make sql-scenarios BASE_URL=http://127.0.0.1:8000/fred/agents/v2
+
+## Documentation
+
+| Topic | File |
+| --- | --- |
+| Evaluate any Fred agent pod | `docs/evaluating-any-fred-agent.md` |
+| RAG evaluation — approach and metrics | `docs/rag-evaluation-rfc.md` |
+| RAG local setup guide | `docs/rag-local-setup.md` |
+| SQL evaluation | `docs/sql-evaluation.md` |
+| OTel export strategy | `fred/docs/swift/rfc/AGENT-EVALUATION-RFC.md §13` |
+
+## Architecture — EVAL-01 Phase 1
+
+This CLI is being restructured into a reusable library core so the Fred
+Control Plane evaluation worker can call it directly without spawning a subprocess.
+
+- `fred_deepeval_cli/core/` — callable library (models, evaluator, profiles, scorer, judge factory)
+- `fred_deepeval_cli/cli/` — thin CLI adapter over the core
+- `fred_deepeval_cli/worker_adapter.py` — public entrypoint for the Control Plane worker
+
+The CLI interface and JSON output remain unchanged.
+See EVAL-01 Phase 1 issue and `fred/docs/swift/rfc/AGENT-EVALUATION-RFC.md §7.3`.

fred_deepeval_cli-0.1.0/README.md

@@ -0,0 +1,46 @@
+# fred-deepeval-cli
+
+External CLI for evaluating one Fred agent turn through `POST /agents/evaluate`.
+
+## Purpose
+
+This project provides a small external CLI that:
+- calls a Fred pod `/agents/evaluate` endpoint
+- receives an `EvalTrace`
+- classifies the turn outcome
+- resolves an evaluation preset from `agent_tags`
+- computes structural checks
+- scores the trace with DeepEval
+
+## Commands
+
+```bash
+make dev
+make eval-dev
+make test
+make code-quality
+make cli
+make score BASE_URL=http://127.0.0.1:8000/fred/agents/v2 AGENT_ID=fred.test.assistant INPUT="echo bonjour" SESSION_ID=eval-001 USER_ID=alice
+make sql-scenarios BASE_URL=http://127.0.0.1:8000/fred/agents/v2
+
+## Documentation
+
+| Topic | File |
+| --- | --- |
+| Evaluate any Fred agent pod | `docs/evaluating-any-fred-agent.md` |
+| RAG evaluation — approach and metrics | `docs/rag-evaluation-rfc.md` |
+| RAG local setup guide | `docs/rag-local-setup.md` |
+| SQL evaluation | `docs/sql-evaluation.md` |
+| OTel export strategy | `fred/docs/swift/rfc/AGENT-EVALUATION-RFC.md §13` |
+
+## Architecture — EVAL-01 Phase 1
+
+This CLI is being restructured into a reusable library core so the Fred
+Control Plane evaluation worker can call it directly without spawning a subprocess.
+
+- `fred_deepeval_cli/core/` — callable library (models, evaluator, profiles, scorer, judge factory)
+- `fred_deepeval_cli/cli/` — thin CLI adapter over the core
+- `fred_deepeval_cli/worker_adapter.py` — public entrypoint for the Control Plane worker
+
+The CLI interface and JSON output remain unchanged.
+See EVAL-01 Phase 1 issue and `fred/docs/swift/rfc/AGENT-EVALUATION-RFC.md §7.3`.

fred_deepeval_cli-0.1.0/fred_deepeval_cli/cli/display.py

@@ -0,0 +1,208 @@
+from __future__ import annotations
+
+from rich.console import Console
+from rich.panel import Panel
+from rich.table import Table
+from rich import box
+from rich.text import Text
+
+from fred_deepeval_cli.core.models import EvaluationCaseRequest, EvaluationCaseResult
+
+console = Console(stderr=True)
+
+
+def _check_icon(value: object) -> str:
+    return "✅" if value is True else "❌"
+
+
+def _outcome_text(outcome: str) -> Text:
+    if outcome == "execution_error":
+        return Text(f"  {outcome}", style="bold red")
+    return Text(f"  {outcome}", style="bold green")
+
+
+def render_score(
+    result: EvaluationCaseResult,
+    request: EvaluationCaseRequest | None = None,
+) -> None:
+    # ── Header ──────────────────────────────────────────────────────────────
+    header = Table.grid(padding=(0, 2))
+    header.add_column(style="bold cyan")
+    header.add_column()
+    if request:
+        header.add_row("Agent", request.agent_id)
+        header.add_row("Session", request.session_id)
+        header.add_row("Input", request.input)
+    header.add_row("Profile", result.profile)
+
+    console.print()
+    console.print(Panel(header, title="[bold]fred-deepeval-cli[/bold]", border_style="cyan"))
+
+    # ── Output agent ────────────────────────────────────────────────────────
+    agent_output = result.actual_output or "—"
+    console.print(Panel(agent_output, title="Output", border_style="yellow"))
+
+    # ── Outcome ─────────────────────────────────────────────────────────────
+    console.print(Panel(
+        _outcome_text(result.outcome),
+        title="Outcome",
+        border_style="green" if result.outcome != "execution_error" else "red",
+    ))
+
+    # ── Structural Checks ───────────────────────────────────────────────────
+    if result.structural_checks:
+        table = Table(box=box.SIMPLE, show_header=True, header_style="bold magenta")
+        table.add_column("Check", style="cyan")
+        table.add_column("", justify="center")
+
+        for check in result.structural_checks:
+            table.add_row(check.name, _check_icon(check.passed))
+
+        console.print(Panel(table, title=f"Structural Checks [{result.profile}]", border_style="magenta"))
+
+    # ── DeepEval Metrics ────────────────────────────────────────────────────
+    if result.metrics:
+        table = Table(box=box.SIMPLE, show_header=True, header_style="bold blue")
+        table.add_column("Metric", style="cyan")
+        table.add_column("Score", justify="right")
+        table.add_column("", justify="center")
+        table.add_column("Reason", style="dim", no_wrap=False, max_width=60)
+
+        for m in result.metrics:
+            score_str = f"{m.score:.2f}" if isinstance(m.score, float) else "—"
+            icon = "✅" if m.verdict == "passed" else ("⏭" if m.verdict == "skipped" else "❌")
+            table.add_row(m.name, score_str, icon, m.explanation or m.error or "—")
+
+        console.print(Panel(table, title="DeepEval Metrics", border_style="blue"))
+
+    # ── Erreurs ─────────────────────────────────────────────────────────────
+    if result.scoring_errors:
+        console.print(Panel(
+            "\n".join(result.scoring_errors),
+            title="Scoring Errors",
+            border_style="red",
+        ))
+
+    console.print()
+
+
+# ── Campagne ────────────────────────────────────────────────────────────────
+
+_CAMPAIGN_METRICS = [
+    "AnswerRelevancyMetric",
+    "FaithfulnessMetric",
+    "ContextualRelevancyMetric",
+    "ContextualPrecisionMetric",
+    "ContextualRecallMetric",
+]
+
+
+def _fmt_score(metrics_by_name: dict, name: str, totals: dict) -> str:
+    m = metrics_by_name.get(name)
+    if m is None:
+        return "—"
+    score = m.get("score")
+    if score is None:
+        return "—"
+    totals[name].append(score)
+    icon = "✅" if m.get("verdict") == "passed" else "❌"
+    return f"{score:.2f}{icon}"
+
+
+def render_campaign(results: list[dict]) -> None:
+    """Affiche le tableau récapitulatif d'une campagne RAG."""
+    totals: dict[str, list[float]] = {m: [] for m in _CAMPAIGN_METRICS}
+
+    table = Table(box=box.SIMPLE, show_header=True, header_style="bold cyan")
+    table.add_column("ID", style="dim", width=22)
+    table.add_column("Outcome", width=10)
+    table.add_column("RAG", justify="center", width=5)
+    table.add_column("AnswerRel", justify="right", width=10)
+    table.add_column("Faithful", justify="right", width=10)
+    table.add_column("CtxRel", justify="right", width=8)
+    table.add_column("CtxPrec", justify="right", width=9)
+    table.add_column("CtxRecall", justify="right", width=10)
+
+    for r in results:
+        raw_metrics = r.get("metrics", {})
+        metrics_by_name = raw_metrics if isinstance(raw_metrics, dict) else {m["name"]: m for m in raw_metrics}
+        table.add_row(
+            r["id"],
+            r["outcome"],
+            "✅" if r.get("rag_ok") else "❌",
+            _fmt_score(metrics_by_name, "AnswerRelevancyMetric", totals),
+            _fmt_score(metrics_by_name, "FaithfulnessMetric", totals),
+            _fmt_score(metrics_by_name, "ContextualRelevancyMetric", totals),
+            _fmt_score(metrics_by_name, "ContextualPrecisionMetric", totals),
+            _fmt_score(metrics_by_name, "ContextualRecallMetric", totals),
+        )
+
+    console.print()
+    console.print(Panel(table, title="Résultats par scénario", border_style="cyan"))
+
+    # ── Moyennes ─────────────────────────────────────────────────────────────
+    avg_table = Table(box=box.SIMPLE, show_header=True, header_style="bold blue")
+    avg_table.add_column("Métrique", style="cyan")
+    avg_table.add_column("Moyenne", justify="right")
+    avg_table.add_column("N", justify="right", style="dim")
+
+    overall: list[float] = []
+    for name in _CAMPAIGN_METRICS:
+        scores = totals[name]
+        if scores:
+            avg = sum(scores) / len(scores)
+            overall.append(avg)
+            avg_table.add_row(name, f"{avg:.4f}  ({avg * 100:.1f}%)", str(len(scores)))
+        else:
+            avg_table.add_row(name, "—", "0")
+
+    if overall:
+        global_avg = sum(overall) / len(overall)
+        avg_table.add_row(
+            "OVERALL",
+            f"{global_avg:.4f}  ({global_avg * 100:.1f}%)",
+            "",
+            style="bold",
+        )
+
+    console.print(Panel(avg_table, title="Moyennes par métrique", border_style="blue"))
+    console.print()
+
+
+def render_sql_campaign(results: list[dict]) -> None:
+    """Affiche le tableau récapitulatif d'une campagne SQL."""
+    table = Table(box=box.SIMPLE, show_header=True, header_style="bold cyan")
+    table.add_column("ID", style="dim", width=22)
+    table.add_column("Outcome", width=12)
+    table.add_column("Query exec", justify="center", width=12)
+    table.add_column("No error", justify="center", width=10)
+    table.add_column("Pass", justify="center", width=6)
+    table.add_column("Failures", style="dim", no_wrap=False, max_width=50)
+
+    passed = 0
+    for r in results:
+        checks = r.get("observed_checks", {})
+        failures = r.get("failures", [])
+        is_pass = r.get("pass", False)
+        if is_pass:
+            passed += 1
+
+        table.add_row(
+            r["id"],
+            r["outcome"],
+            "✅" if checks.get("sql_query_executed") else "❌",
+            "✅" if checks.get("sql_no_execution_error") else "❌",
+            "✅" if is_pass else "❌",
+            " | ".join(failures) if failures else "—",
+        )
+
+    console.print()
+    console.print(Panel(table, title="Résultats SQL par scénario", border_style="cyan"))
+
+    total = len(results)
+    color = "green" if passed == total else "yellow" if passed > 0 else "red"
+    console.print(Panel(
+        f"[bold {color}]{passed}/{total} scénarios passés[/bold {color}]",
+        border_style=color,
+    ))
+    console.print()

fred_deepeval_cli-0.1.0/fred_deepeval_cli/cli/main.py

@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+import argparse
+import json
+import os
+
+from fred_deepeval_cli.core.models import EvaluationCaseRequest
+from fred_deepeval_cli.core.evaluator import evaluate_case_sync
+from fred_deepeval_cli.core.judge_factory import build_judge
+from fred_deepeval_cli.cli.display import render_score
+from dotenv import load_dotenv
+
+dotenv_path = os.getenv("ENV_FILE", "./config/.env")
+load_dotenv(dotenv_path)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="fred-deepeval-cli",
+        description="External CLI for evaluating Fred agent turns.",
+    )
+
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    score_parser = subparsers.add_parser(
+        "score",
+        help="Evaluate one Fred agent turn and score it with DeepEval.",
+    )
+    add_shared_eval_args(score_parser)
+
+    return parser
+
+
+def add_shared_eval_args(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("--base-url", required=True, help="Fred pod base URL.")
+    parser.add_argument("--agent-id", required=True, help="Agent identifier.")
+    parser.add_argument("--input", required=True, help="User input to evaluate.")
+    parser.add_argument("--session-id", required=True, help="Session identifier.")
+    parser.add_argument("--user-id", required=True, help="Runtime user identifier.")
+    parser.add_argument("--team-id", help="Optional runtime team identifier.")
+    parser.add_argument(
+        "--access-token",
+        default=os.environ.get("FRED_ACCESS_TOKEN"),
+        help="Optional bearer token for authenticated agent evaluation.",
+    )
+    parser.add_argument(
+        "--search-policy",
+        default=os.environ.get("FRED_SEARCH_POLICY"),
+        help="Optional runtime search policy override (for example: semantic).",
+    )
+    parser.add_argument(
+        "--profile",
+        default="auto",
+        choices=["auto", "rag", "sql", "workflow", "default"],
+        help="Evaluation profile. Defaults to auto-detection from agent_tags.",
+    )
+
+
+def run_score(args: argparse.Namespace) -> int:
+    runtime_context: dict = {"user_id": args.user_id}
+    if args.team_id:
+        runtime_context["team_id"] = args.team_id
+    if args.search_policy:
+        runtime_context["search_policy"] = args.search_policy
+
+    request = EvaluationCaseRequest(
+        agent_id=args.agent_id,
+        input=args.input,
+        session_id=args.session_id,
+        profile=args.profile,
+        runtime_context=runtime_context,
+    )
+
+    judge = build_judge()
+    result = evaluate_case_sync(
+        base_url=args.base_url,
+        request=request,
+        judge=judge,
+        access_token=args.access_token,
+    )
+
+    render_score(result, request=request)
+    print(json.dumps(result.model_dump(), indent=2, ensure_ascii=False))
+
+    return 1 if result.outcome == "execution_error" else 0
+
+
+def main() -> int:
+    parser = build_parser()
+    args = parser.parse_args()
+
+    if args.command == "score":
+        return run_score(args)
+
+    parser.error(f"Unknown command: {args.command}")
+    return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

fred_deepeval_cli-0.1.0/fred_deepeval_cli/core/config_loader.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import logging
+
+from fred_core.common import ConfigFiles, load_configuration_with_config_files
+from pydantic import BaseModel
+
+
+class JudgeProfileSettings(BaseModel):
+    api_base: str | None = None
+    api_key_env: str | None = None
+    request_timeout: int = 120
+
+
+class JudgeProfile(BaseModel):
+    profile_id: str
+    provider: str
+    model: str
+    settings: JudgeProfileSettings = JudgeProfileSettings()
+
+
+class JudgeConfig(BaseModel):
+    default: str
+    profiles: list[JudgeProfile]
+
+    def active(self) -> JudgeProfile:
+        for p in self.profiles:
+            if p.profile_id == self.default:
+                return p
+        raise ValueError(
+            f"Judge profile '{self.default}' not found. "
+            f"Available: {[p.profile_id for p in self.profiles]}"
+        )
+
+
+class Configuration(BaseModel):
+    version: str = "v1"
+    judge: JudgeConfig
+
+
+def parse_configuration(config_file: str) -> Configuration:
+    import yaml
+
+    with open(config_file, encoding="utf-8") as file:
+        payload = yaml.safe_load(file)
+
+    if payload is None:
+        raise ValueError(f"Configuration file is empty: {config_file}")
+    if not isinstance(payload, dict):
+        raise ValueError(f"Configuration file must be a mapping object: {config_file}")
+
+    return Configuration.model_validate(payload)
+
+
+_config_files = ConfigFiles(logger=logging.getLogger(__name__))
+
+
+def load_configuration() -> Configuration:
+    return load_configuration_with_config_files(
+        _config_files,
+        parse_configuration,
+    )

fred_deepeval_cli-0.1.0/fred_deepeval_cli/core/evaluator.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+import httpx
+
+from fred_deepeval_cli.core.models import (
+    EvaluationCaseRequest,
+    EvaluationCaseResult,
+)
+from fred_deepeval_cli.core.profiles import resolve_profile
+from fred_deepeval_cli.core.structural_checks import build_structural_checks
+from fred_deepeval_cli.core.scorer import score_trace
+
+
+def classify_outcome(trace: dict) -> str:
+    if trace.get("error"):
+        return "execution_error"
+    if any(step.get("kind") == "awaiting_human" for step in trace.get("steps", [])):
+        return "hitl_blocked"
+    if any(step.get("kind") == "node_error" for step in trace.get("steps", [])):
+        return "degraded"
+    if trace.get("output"):
+        return "success"
+    return "unknown"
+
+
+def fetch_trace(
+    base_url: str,
+    request: EvaluationCaseRequest,
+    access_token: str | None = None,
+) -> dict:
+    headers = {"Content-Type": "application/json"}
+    if access_token:
+        headers["Authorization"] = f"Bearer {access_token}"
+
+    payload = {
+        "agent_id": request.agent_id,
+        "input": request.input,
+        "session_id": request.session_id,
+        "runtime_context": request.runtime_context,
+    }
+
+    with httpx.Client(timeout=httpx.Timeout(30.0, connect=5.0, read=None)) as client:
+        response = client.post(
+            f"{base_url.rstrip('/')}/agents/evaluate",
+            json=payload,
+            headers=headers,
+        )
+        response.raise_for_status()
+        result = response.json()
+        if not isinstance(result, dict):
+            raise RuntimeError("Evaluate response must be a JSON object.")
+        return result
+
+
+def evaluate_case_sync(
+    base_url: str,
+    request: EvaluationCaseRequest,
+    judge=None,
+    access_token: str | None = None,
+) -> EvaluationCaseResult:
+    try:
+        trace = fetch_trace(base_url, request, access_token=access_token)
+    except Exception as e:
+        return EvaluationCaseResult(
+            outcome="execution_error",
+            profile=request.profile,
+            structural_checks=[],
+            metrics=[],
+            execution_error=str(e),
+        )
+
+    outcome = classify_outcome(trace)
+    profile = resolve_profile(trace, explicit_profile=request.profile)
+    structural_checks = build_structural_checks(trace, profile=profile)
+
+    metrics, scoring_errors = [], []
+    if judge is not None:
+        metrics, scoring_errors = score_trace(
+            trace,
+            profile=profile,
+            expected_output=request.expected_output,
+            judge=judge,
+        )
+
+    return EvaluationCaseResult(
+        outcome=outcome,
+        profile=profile,
+        structural_checks=structural_checks,
+        metrics=metrics,
+        actual_output=trace.get("output"),
+        latency_ms=trace.get("latency_ms"),
+        execution_error=trace.get("error"),
+        scoring_errors=scoring_errors,
+    )

fred_deepeval_cli-0.1.0/fred_deepeval_cli/core/judge_factory.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+import os
+
+from fred_deepeval_cli.core.config_loader import load_configuration
+
+
+def build_judge(config=None):
+    from deepeval.models.llms import GPTModel, LiteLLMModel
+
+    if config is None:
+        config = load_configuration()
+
+    profile = config.judge.active()
+    provider = profile.provider
+    model_name = profile.model
+    settings = profile.settings
+
+    if provider == "litellm":
+        api_key_env = settings.api_key_env or "LITELLM_API_KEY"
+        api_key = os.environ.get(api_key_env)
+        if not api_key:
+            raise RuntimeError(
+                f"Missing {api_key_env} in environment/.env for the litellm judge."
+            )
+        return LiteLLMModel(
+            model=model_name,
+            api_key=api_key,
+            base_url=settings.api_base,
+            request_timeout=settings.request_timeout,
+            num_retries=0,
+        )
+
+    if provider == "ollama":
+        return LiteLLMModel(
+            model=f"ollama/{model_name}",
+            api_key="ollama",
+            base_url=settings.api_base or "http://localhost:11434",
+            request_timeout=settings.request_timeout,
+            num_retries=0,
+        )
+
+    if provider == "openai":
+        api_key_env = settings.api_key_env or "OPENAI_API_KEY"
+        api_key = os.environ.get(api_key_env)
+        if not api_key:
+            raise RuntimeError(
+                f"Missing {api_key_env} in environment/.env for the openai judge."
+            )
+        return GPTModel(model=model_name)
+
+    raise ValueError(f"Unsupported judge provider: {provider}")

fred_deepeval_cli-0.1.0/fred_deepeval_cli/core/models.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from typing import Literal
+from pydantic import BaseModel
+
+
+class EvaluationMetricResult(BaseModel):
+    name: str
+    provider: str
+    score: float | None
+    threshold: float | None = None
+    verdict: Literal["passed", "failed", "skipped", "error"]
+    explanation: str | None = None
+    error: str | None = None
+
+
+class StructuralCheckResult(BaseModel):
+    name: str
+    passed: bool
+    detail: str | None = None
+
+
+class EvaluationCaseRequest(BaseModel):
+    agent_id: str
+    input: str
+    session_id: str
+    expected_output: str | None = None
+    profile: str = "auto"
+    runtime_context: dict = {}
+
+
+class EvaluationCaseResult(BaseModel):
+    schema_version: Literal["1"] = "1"
+    outcome: Literal["success", "execution_error", "degraded", "hitl_blocked", "unknown"]
+    profile: str
+    structural_checks: list[StructuralCheckResult]
+    metrics: list[EvaluationMetricResult]
+    latency_ms: int | None = None
+    actual_output: str | None = None
+    execution_error: str | None = None
+    scoring_errors: list[str] = []