fred-deepeval-cli 0.1.0__py3-none-any.whl

fred_deepeval_cli/cli/display.py

@@ -0,0 +1,208 @@
+from __future__ import annotations
+
+from rich.console import Console
+from rich.panel import Panel
+from rich.table import Table
+from rich import box
+from rich.text import Text
+
+from fred_deepeval_cli.core.models import EvaluationCaseRequest, EvaluationCaseResult
+
+console = Console(stderr=True)
+
+
+def _check_icon(value: object) -> str:
+    return "✅" if value is True else "❌"
+
+
+def _outcome_text(outcome: str) -> Text:
+    if outcome == "execution_error":
+        return Text(f"  {outcome}", style="bold red")
+    return Text(f"  {outcome}", style="bold green")
+
+
+def render_score(
+    result: EvaluationCaseResult,
+    request: EvaluationCaseRequest | None = None,
+) -> None:
+    # ── Header ──────────────────────────────────────────────────────────────
+    header = Table.grid(padding=(0, 2))
+    header.add_column(style="bold cyan")
+    header.add_column()
+    if request:
+        header.add_row("Agent", request.agent_id)
+        header.add_row("Session", request.session_id)
+        header.add_row("Input", request.input)
+    header.add_row("Profile", result.profile)
+
+    console.print()
+    console.print(Panel(header, title="[bold]fred-deepeval-cli[/bold]", border_style="cyan"))
+
+    # ── Output agent ────────────────────────────────────────────────────────
+    agent_output = result.actual_output or "—"
+    console.print(Panel(agent_output, title="Output", border_style="yellow"))
+
+    # ── Outcome ─────────────────────────────────────────────────────────────
+    console.print(Panel(
+        _outcome_text(result.outcome),
+        title="Outcome",
+        border_style="green" if result.outcome != "execution_error" else "red",
+    ))
+
+    # ── Structural Checks ───────────────────────────────────────────────────
+    if result.structural_checks:
+        table = Table(box=box.SIMPLE, show_header=True, header_style="bold magenta")
+        table.add_column("Check", style="cyan")
+        table.add_column("", justify="center")
+
+        for check in result.structural_checks:
+            table.add_row(check.name, _check_icon(check.passed))
+
+        console.print(Panel(table, title=f"Structural Checks [{result.profile}]", border_style="magenta"))
+
+    # ── DeepEval Metrics ────────────────────────────────────────────────────
+    if result.metrics:
+        table = Table(box=box.SIMPLE, show_header=True, header_style="bold blue")
+        table.add_column("Metric", style="cyan")
+        table.add_column("Score", justify="right")
+        table.add_column("", justify="center")
+        table.add_column("Reason", style="dim", no_wrap=False, max_width=60)
+
+        for m in result.metrics:
+            score_str = f"{m.score:.2f}" if isinstance(m.score, float) else "—"
+            icon = "✅" if m.verdict == "passed" else ("⏭" if m.verdict == "skipped" else "❌")
+            table.add_row(m.name, score_str, icon, m.explanation or m.error or "—")
+
+        console.print(Panel(table, title="DeepEval Metrics", border_style="blue"))
+
+    # ── Erreurs ─────────────────────────────────────────────────────────────
+    if result.scoring_errors:
+        console.print(Panel(
+            "\n".join(result.scoring_errors),
+            title="Scoring Errors",
+            border_style="red",
+        ))
+
+    console.print()
+
+
+# ── Campagne ────────────────────────────────────────────────────────────────
+
+_CAMPAIGN_METRICS = [
+    "AnswerRelevancyMetric",
+    "FaithfulnessMetric",
+    "ContextualRelevancyMetric",
+    "ContextualPrecisionMetric",
+    "ContextualRecallMetric",
+]
+
+
+def _fmt_score(metrics_by_name: dict, name: str, totals: dict) -> str:
+    m = metrics_by_name.get(name)
+    if m is None:
+        return "—"
+    score = m.get("score")
+    if score is None:
+        return "—"
+    totals[name].append(score)
+    icon = "✅" if m.get("verdict") == "passed" else "❌"
+    return f"{score:.2f}{icon}"
+
+
+def render_campaign(results: list[dict]) -> None:
+    """Affiche le tableau récapitulatif d'une campagne RAG."""
+    totals: dict[str, list[float]] = {m: [] for m in _CAMPAIGN_METRICS}
+
+    table = Table(box=box.SIMPLE, show_header=True, header_style="bold cyan")
+    table.add_column("ID", style="dim", width=22)
+    table.add_column("Outcome", width=10)
+    table.add_column("RAG", justify="center", width=5)
+    table.add_column("AnswerRel", justify="right", width=10)
+    table.add_column("Faithful", justify="right", width=10)
+    table.add_column("CtxRel", justify="right", width=8)
+    table.add_column("CtxPrec", justify="right", width=9)
+    table.add_column("CtxRecall", justify="right", width=10)
+
+    for r in results:
+        raw_metrics = r.get("metrics", {})
+        metrics_by_name = raw_metrics if isinstance(raw_metrics, dict) else {m["name"]: m for m in raw_metrics}
+        table.add_row(
+            r["id"],
+            r["outcome"],
+            "✅" if r.get("rag_ok") else "❌",
+            _fmt_score(metrics_by_name, "AnswerRelevancyMetric", totals),
+            _fmt_score(metrics_by_name, "FaithfulnessMetric", totals),
+            _fmt_score(metrics_by_name, "ContextualRelevancyMetric", totals),
+            _fmt_score(metrics_by_name, "ContextualPrecisionMetric", totals),
+            _fmt_score(metrics_by_name, "ContextualRecallMetric", totals),
+        )
+
+    console.print()
+    console.print(Panel(table, title="Résultats par scénario", border_style="cyan"))
+
+    # ── Moyennes ─────────────────────────────────────────────────────────────
+    avg_table = Table(box=box.SIMPLE, show_header=True, header_style="bold blue")
+    avg_table.add_column("Métrique", style="cyan")
+    avg_table.add_column("Moyenne", justify="right")
+    avg_table.add_column("N", justify="right", style="dim")
+
+    overall: list[float] = []
+    for name in _CAMPAIGN_METRICS:
+        scores = totals[name]
+        if scores:
+            avg = sum(scores) / len(scores)
+            overall.append(avg)
+            avg_table.add_row(name, f"{avg:.4f}  ({avg * 100:.1f}%)", str(len(scores)))
+        else:
+            avg_table.add_row(name, "—", "0")
+
+    if overall:
+        global_avg = sum(overall) / len(overall)
+        avg_table.add_row(
+            "OVERALL",
+            f"{global_avg:.4f}  ({global_avg * 100:.1f}%)",
+            "",
+            style="bold",
+        )
+
+    console.print(Panel(avg_table, title="Moyennes par métrique", border_style="blue"))
+    console.print()
+
+
+def render_sql_campaign(results: list[dict]) -> None:
+    """Affiche le tableau récapitulatif d'une campagne SQL."""
+    table = Table(box=box.SIMPLE, show_header=True, header_style="bold cyan")
+    table.add_column("ID", style="dim", width=22)
+    table.add_column("Outcome", width=12)
+    table.add_column("Query exec", justify="center", width=12)
+    table.add_column("No error", justify="center", width=10)
+    table.add_column("Pass", justify="center", width=6)
+    table.add_column("Failures", style="dim", no_wrap=False, max_width=50)
+
+    passed = 0
+    for r in results:
+        checks = r.get("observed_checks", {})
+        failures = r.get("failures", [])
+        is_pass = r.get("pass", False)
+        if is_pass:
+            passed += 1
+
+        table.add_row(
+            r["id"],
+            r["outcome"],
+            "✅" if checks.get("sql_query_executed") else "❌",
+            "✅" if checks.get("sql_no_execution_error") else "❌",
+            "✅" if is_pass else "❌",
+            " | ".join(failures) if failures else "—",
+        )
+
+    console.print()
+    console.print(Panel(table, title="Résultats SQL par scénario", border_style="cyan"))
+
+    total = len(results)
+    color = "green" if passed == total else "yellow" if passed > 0 else "red"
+    console.print(Panel(
+        f"[bold {color}]{passed}/{total} scénarios passés[/bold {color}]",
+        border_style=color,
+    ))
+    console.print()

fred_deepeval_cli/cli/main.py

@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+import argparse
+import json
+import os
+
+from fred_deepeval_cli.core.models import EvaluationCaseRequest
+from fred_deepeval_cli.core.evaluator import evaluate_case_sync
+from fred_deepeval_cli.core.judge_factory import build_judge
+from fred_deepeval_cli.cli.display import render_score
+from dotenv import load_dotenv
+
+dotenv_path = os.getenv("ENV_FILE", "./config/.env")
+load_dotenv(dotenv_path)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="fred-deepeval-cli",
+        description="External CLI for evaluating Fred agent turns.",
+    )
+
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    score_parser = subparsers.add_parser(
+        "score",
+        help="Evaluate one Fred agent turn and score it with DeepEval.",
+    )
+    add_shared_eval_args(score_parser)
+
+    return parser
+
+
+def add_shared_eval_args(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("--base-url", required=True, help="Fred pod base URL.")
+    parser.add_argument("--agent-id", required=True, help="Agent identifier.")
+    parser.add_argument("--input", required=True, help="User input to evaluate.")
+    parser.add_argument("--session-id", required=True, help="Session identifier.")
+    parser.add_argument("--user-id", required=True, help="Runtime user identifier.")
+    parser.add_argument("--team-id", help="Optional runtime team identifier.")
+    parser.add_argument(
+        "--access-token",
+        default=os.environ.get("FRED_ACCESS_TOKEN"),
+        help="Optional bearer token for authenticated agent evaluation.",
+    )
+    parser.add_argument(
+        "--search-policy",
+        default=os.environ.get("FRED_SEARCH_POLICY"),
+        help="Optional runtime search policy override (for example: semantic).",
+    )
+    parser.add_argument(
+        "--profile",
+        default="auto",
+        choices=["auto", "rag", "sql", "workflow", "default"],
+        help="Evaluation profile. Defaults to auto-detection from agent_tags.",
+    )
+
+
+def run_score(args: argparse.Namespace) -> int:
+    runtime_context: dict = {"user_id": args.user_id}
+    if args.team_id:
+        runtime_context["team_id"] = args.team_id
+    if args.search_policy:
+        runtime_context["search_policy"] = args.search_policy
+
+    request = EvaluationCaseRequest(
+        agent_id=args.agent_id,
+        input=args.input,
+        session_id=args.session_id,
+        profile=args.profile,
+        runtime_context=runtime_context,
+    )
+
+    judge = build_judge()
+    result = evaluate_case_sync(
+        base_url=args.base_url,
+        request=request,
+        judge=judge,
+        access_token=args.access_token,
+    )
+
+    render_score(result, request=request)
+    print(json.dumps(result.model_dump(), indent=2, ensure_ascii=False))
+
+    return 1 if result.outcome == "execution_error" else 0
+
+
+def main() -> int:
+    parser = build_parser()
+    args = parser.parse_args()
+
+    if args.command == "score":
+        return run_score(args)
+
+    parser.error(f"Unknown command: {args.command}")
+    return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

fred_deepeval_cli/core/config_loader.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import logging
+
+from fred_core.common import ConfigFiles, load_configuration_with_config_files
+from pydantic import BaseModel
+
+
+class JudgeProfileSettings(BaseModel):
+    api_base: str | None = None
+    api_key_env: str | None = None
+    request_timeout: int = 120
+
+
+class JudgeProfile(BaseModel):
+    profile_id: str
+    provider: str
+    model: str
+    settings: JudgeProfileSettings = JudgeProfileSettings()
+
+
+class JudgeConfig(BaseModel):
+    default: str
+    profiles: list[JudgeProfile]
+
+    def active(self) -> JudgeProfile:
+        for p in self.profiles:
+            if p.profile_id == self.default:
+                return p
+        raise ValueError(
+            f"Judge profile '{self.default}' not found. "
+            f"Available: {[p.profile_id for p in self.profiles]}"
+        )
+
+
+class Configuration(BaseModel):
+    version: str = "v1"
+    judge: JudgeConfig
+
+
+def parse_configuration(config_file: str) -> Configuration:
+    import yaml
+
+    with open(config_file, encoding="utf-8") as file:
+        payload = yaml.safe_load(file)
+
+    if payload is None:
+        raise ValueError(f"Configuration file is empty: {config_file}")
+    if not isinstance(payload, dict):
+        raise ValueError(f"Configuration file must be a mapping object: {config_file}")
+
+    return Configuration.model_validate(payload)
+
+
+_config_files = ConfigFiles(logger=logging.getLogger(__name__))
+
+
+def load_configuration() -> Configuration:
+    return load_configuration_with_config_files(
+        _config_files,
+        parse_configuration,
+    )

fred_deepeval_cli/core/evaluator.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+import httpx
+
+from fred_deepeval_cli.core.models import (
+    EvaluationCaseRequest,
+    EvaluationCaseResult,
+)
+from fred_deepeval_cli.core.profiles import resolve_profile
+from fred_deepeval_cli.core.structural_checks import build_structural_checks
+from fred_deepeval_cli.core.scorer import score_trace
+
+
+def classify_outcome(trace: dict) -> str:
+    if trace.get("error"):
+        return "execution_error"
+    if any(step.get("kind") == "awaiting_human" for step in trace.get("steps", [])):
+        return "hitl_blocked"
+    if any(step.get("kind") == "node_error" for step in trace.get("steps", [])):
+        return "degraded"
+    if trace.get("output"):
+        return "success"
+    return "unknown"
+
+
+def fetch_trace(
+    base_url: str,
+    request: EvaluationCaseRequest,
+    access_token: str | None = None,
+) -> dict:
+    headers = {"Content-Type": "application/json"}
+    if access_token:
+        headers["Authorization"] = f"Bearer {access_token}"
+
+    payload = {
+        "agent_id": request.agent_id,
+        "input": request.input,
+        "session_id": request.session_id,
+        "runtime_context": request.runtime_context,
+    }
+
+    with httpx.Client(timeout=httpx.Timeout(30.0, connect=5.0, read=None)) as client:
+        response = client.post(
+            f"{base_url.rstrip('/')}/agents/evaluate",
+            json=payload,
+            headers=headers,
+        )
+        response.raise_for_status()
+        result = response.json()
+        if not isinstance(result, dict):
+            raise RuntimeError("Evaluate response must be a JSON object.")
+        return result
+
+
+def evaluate_case_sync(
+    base_url: str,
+    request: EvaluationCaseRequest,
+    judge=None,
+    access_token: str | None = None,
+) -> EvaluationCaseResult:
+    try:
+        trace = fetch_trace(base_url, request, access_token=access_token)
+    except Exception as e:
+        return EvaluationCaseResult(
+            outcome="execution_error",
+            profile=request.profile,
+            structural_checks=[],
+            metrics=[],
+            execution_error=str(e),
+        )
+
+    outcome = classify_outcome(trace)
+    profile = resolve_profile(trace, explicit_profile=request.profile)
+    structural_checks = build_structural_checks(trace, profile=profile)
+
+    metrics, scoring_errors = [], []
+    if judge is not None:
+        metrics, scoring_errors = score_trace(
+            trace,
+            profile=profile,
+            expected_output=request.expected_output,
+            judge=judge,
+        )
+
+    return EvaluationCaseResult(
+        outcome=outcome,
+        profile=profile,
+        structural_checks=structural_checks,
+        metrics=metrics,
+        actual_output=trace.get("output"),
+        latency_ms=trace.get("latency_ms"),
+        execution_error=trace.get("error"),
+        scoring_errors=scoring_errors,
+    )

fred_deepeval_cli/core/judge_factory.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+import os
+
+from fred_deepeval_cli.core.config_loader import load_configuration
+
+
+def build_judge(config=None):
+    from deepeval.models.llms import GPTModel, LiteLLMModel
+
+    if config is None:
+        config = load_configuration()
+
+    profile = config.judge.active()
+    provider = profile.provider
+    model_name = profile.model
+    settings = profile.settings
+
+    if provider == "litellm":
+        api_key_env = settings.api_key_env or "LITELLM_API_KEY"
+        api_key = os.environ.get(api_key_env)
+        if not api_key:
+            raise RuntimeError(
+                f"Missing {api_key_env} in environment/.env for the litellm judge."
+            )
+        return LiteLLMModel(
+            model=model_name,
+            api_key=api_key,
+            base_url=settings.api_base,
+            request_timeout=settings.request_timeout,
+            num_retries=0,
+        )
+
+    if provider == "ollama":
+        return LiteLLMModel(
+            model=f"ollama/{model_name}",
+            api_key="ollama",
+            base_url=settings.api_base or "http://localhost:11434",
+            request_timeout=settings.request_timeout,
+            num_retries=0,
+        )
+
+    if provider == "openai":
+        api_key_env = settings.api_key_env or "OPENAI_API_KEY"
+        api_key = os.environ.get(api_key_env)
+        if not api_key:
+            raise RuntimeError(
+                f"Missing {api_key_env} in environment/.env for the openai judge."
+            )
+        return GPTModel(model=model_name)
+
+    raise ValueError(f"Unsupported judge provider: {provider}")

fred_deepeval_cli/core/models.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from typing import Literal
+from pydantic import BaseModel
+
+
+class EvaluationMetricResult(BaseModel):
+    name: str
+    provider: str
+    score: float | None
+    threshold: float | None = None
+    verdict: Literal["passed", "failed", "skipped", "error"]
+    explanation: str | None = None
+    error: str | None = None
+
+
+class StructuralCheckResult(BaseModel):
+    name: str
+    passed: bool
+    detail: str | None = None
+
+
+class EvaluationCaseRequest(BaseModel):
+    agent_id: str
+    input: str
+    session_id: str
+    expected_output: str | None = None
+    profile: str = "auto"
+    runtime_context: dict = {}
+
+
+class EvaluationCaseResult(BaseModel):
+    schema_version: Literal["1"] = "1"
+    outcome: Literal["success", "execution_error", "degraded", "hitl_blocked", "unknown"]
+    profile: str
+    structural_checks: list[StructuralCheckResult]
+    metrics: list[EvaluationMetricResult]
+    latency_ms: int | None = None
+    actual_output: str | None = None
+    execution_error: str | None = None
+    scoring_errors: list[str] = []

fred_deepeval_cli/core/profiles.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+SUPPORTED_PROFILES = {"rag", "sql", "workflow", "default"}
+
+
+def resolve_profile(trace: dict, explicit_profile: str = "auto") -> str:
+    if explicit_profile != "auto" and explicit_profile in SUPPORTED_PROFILES:
+        return explicit_profile
+
+    agent_tags = set(trace.get("agent_tags", []))
+
+    if "rag" in agent_tags:
+        return "rag"
+
+    if "sql" in agent_tags:
+        return "sql"
+
+    if "workflow" in agent_tags:
+        return "workflow"
+
+    return "default"

fred_deepeval_cli/core/scorer.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+import logging
+
+from deepeval.test_case import LLMTestCase
+
+from fred_deepeval_cli.core.models import EvaluationMetricResult
+
+logging.getLogger("LiteLLM").setLevel(logging.CRITICAL)
+logging.getLogger("root").setLevel(logging.CRITICAL)
+
+
+def _trace_to_test_case(trace: dict, expected_output: str | None = None) -> LLMTestCase:
+    return LLMTestCase(
+        input=trace.get("input", ""),
+        actual_output=trace.get("output") or "",
+        expected_output=expected_output,
+        retrieval_context=trace.get("retrieval_context", []) or [],
+    )
+
+
+def score_trace(
+    trace: dict,
+    profile: str = "default",
+    expected_output: str | None = None,
+    judge=None,
+) -> tuple[list[EvaluationMetricResult], list[str]]:
+    from deepeval.metrics import (
+        AnswerRelevancyMetric,
+        ContextualPrecisionMetric,
+        ContextualRecallMetric,
+        ContextualRelevancyMetric,
+        FaithfulnessMetric,
+    )
+
+    test_case = _trace_to_test_case(trace, expected_output=expected_output)
+    retrieval_context = trace.get("retrieval_context") or []
+
+    def _metric(cls, **kwargs):
+        return cls(model=judge, async_mode=False, **kwargs)
+
+    metrics = [_metric(AnswerRelevancyMetric)]
+
+    if profile == "rag" and retrieval_context:
+        metrics.append(_metric(FaithfulnessMetric))
+        metrics.append(_metric(ContextualRelevancyMetric))
+        if expected_output:
+            metrics.append(_metric(ContextualPrecisionMetric))
+            metrics.append(_metric(ContextualRecallMetric))
+
+    results: list[EvaluationMetricResult] = []
+    scoring_errors: list[str] = []
+
+    for metric in metrics:
+        try:
+            metric.measure(test_case)
+            results.append(EvaluationMetricResult(
+                name=metric.__class__.__name__,
+                provider="deepeval",
+                score=metric.score,
+                verdict="passed" if metric.success else "failed",
+                explanation=getattr(metric, "reason", None),
+            ))
+        except Exception as e:
+            scoring_errors.append(f"{metric.__class__.__name__}: {e}")
+            results.append(EvaluationMetricResult(
+                name=metric.__class__.__name__,
+                provider="deepeval",
+                score=None,
+                verdict="error",
+                error=str(e),
+            ))
+
+    return results, scoring_errors

fred_deepeval_cli/core/structural_checks.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+from fred_deepeval_cli.core.models import StructuralCheckResult
+
+
+def _tool_steps(trace: dict, kind: str, tool_name: str) -> list[dict]:
+    return [
+        step
+        for step in trace.get("steps", [])
+        if step.get("kind") == kind and step.get("tool_name") == tool_name
+    ]
+
+
+def _has_tool_call(trace: dict, tool_name: str) -> bool:
+    return bool(_tool_steps(trace, "tool_call", tool_name))
+
+
+def _has_successful_tool_result(trace: dict, tool_name: str) -> bool:
+    for step in _tool_steps(trace, "tool_result", tool_name):
+        if step.get("is_error"):
+            continue
+        content = step.get("content") or ""
+        if isinstance(content, str) and content.strip():
+            if not content.lstrip().startswith("Error:"):
+                return True
+    return False
+
+
+def build_structural_checks(trace: dict, profile: str = "default") -> list[StructuralCheckResult]:
+    checks = []
+
+    if profile == "rag":
+        checks.append(StructuralCheckResult(
+            name="rag_tool_used",
+            passed="knowledge_search" in trace.get("tools_called", []),
+        ))
+        checks.append(StructuralCheckResult(
+            name="rag_context_nonempty",
+            passed=bool(trace.get("retrieval_context")),
+        ))
+
+    elif profile == "sql":
+        checks.append(StructuralCheckResult(
+            name="sql_query_executed",
+            passed=_has_tool_call(trace, "read_query") and _has_successful_tool_result(trace, "read_query"),
+        ))
+        checks.append(StructuralCheckResult(
+            name="sql_no_execution_error",
+            passed=not trace.get("error") and not any(
+                s.get("kind") == "node_error" or s.get("is_error")
+                for s in trace.get("steps", [])
+            ),
+        ))
+
+    return checks

fred_deepeval_cli/dataset_workflow.py

@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+from datetime import timedelta
+
+from temporalio import activity, workflow
+from temporalio.client import Client
+from temporalio.worker import Worker, UnsandboxedWorkflowRunner
+from temporalio.testing import WorkflowEnvironment
+
+from fred_deepeval_cli.core.models import EvaluationCaseRequest
+from fred_deepeval_cli.core.evaluator import evaluate_case_sync
+from fred_deepeval_cli.core.judge_factory import build_judge
+
+
+@activity.defn
+async def evaluate_question_activity(params: dict) -> dict:
+    """Une question = une activity Temporal."""
+    request = EvaluationCaseRequest(
+        agent_id=params["agent_id"],
+        input=params["input"],
+        session_id=params["session_id"],
+        expected_output=params.get("expected_answer"),
+        profile=params.get("profile", "auto"),
+        runtime_context={
+            "user_id": params["user_id"],
+            **({"team_id": params["team_id"]} if params.get("team_id") else {}),
+            **({"search_policy": params["search_policy"]} if params.get("search_policy") else {}),
+        },
+    )
+
+    try:
+        judge = build_judge()
+        result = evaluate_case_sync(
+            base_url=params["base_url"],
+            request=request,
+            judge=judge,
+            access_token=params.get("access_token"),
+        )
+    except Exception as e:
+        return {
+            "id": params["id"],
+            "input": params["input"],
+            "outcome": "error",
+            "profile": "unknown",
+            "rag_ok": False,
+            "structural_checks": [],
+            "metrics": [],
+            "error": str(e),
+        }
+
+    rag_ok = all(c.passed for c in result.structural_checks)
+    metrics_by_name = {m.name: m.model_dump() for m in result.metrics}
+
+    return {
+        "id": params["id"],
+        "input": params["input"],
+        "outcome": result.outcome,
+        "profile": result.profile,
+        "rag_ok": rag_ok,
+        "structural_checks": [c.model_dump() for c in result.structural_checks],
+        "metrics": metrics_by_name,
+    }
+
+
+@workflow.defn
+class RagDatasetWorkflow:
+    @workflow.run
+    async def run(self, questions: list[dict]) -> list[dict]:
+        results = []
+        for q in questions:
+            result = await workflow.execute_activity(
+                evaluate_question_activity,
+                q,
+                start_to_close_timeout=timedelta(minutes=10),
+            )
+            results.append(result)
+        return results
+
+
+async def run_with_temporal(questions: list[dict]) -> list[dict]:
+    """Lance le workflow en mode in-memory (pas de serveur Temporal requis)."""
+    async with await WorkflowEnvironment.start_local() as env:
+        async with Worker(
+            env.client,
+            task_queue="rag-eval",
+            workflows=[RagDatasetWorkflow],
+            activities=[evaluate_question_activity],
+            workflow_runner=UnsandboxedWorkflowRunner(),
+        ):
+            results: list[dict] = await env.client.execute_workflow(
+                RagDatasetWorkflow.run,
+                questions,
+                id="rag-dataset-eval",
+                task_queue="rag-eval",
+            )
+    return results
+
+
+async def run_with_temporal_server(questions: list[dict], server_url: str) -> list[dict]:
+    """Lance le workflow sur un serveur Temporal réel (production)."""
+    client = await Client.connect(server_url)
+    async with Worker(
+        client,
+        task_queue="rag-eval",
+        workflows=[RagDatasetWorkflow],
+        activities=[evaluate_question_activity],
+        workflow_runner=UnsandboxedWorkflowRunner(),
+    ):
+        results: list[dict] = await client.execute_workflow(
+            RagDatasetWorkflow.run,
+            questions,
+            id="rag-dataset-eval",
+            task_queue="rag-eval",
+        )
+    return results

fred_deepeval_cli/test_helpers.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+import httpx
+
+
+def make_response(payload: dict) -> httpx.Response:
+    request = httpx.Request(
+        "POST",
+        "http://127.0.0.1:8000/fred/agents/v2/agents/evaluate",
+    )
+    return httpx.Response(200, json=payload, request=request)
+
+
+def make_trace(
+    *,
+    session_id: str = "eval-001",
+    agent_id: str = "fred.test.assistant",
+    agent_tags: list[str] | None = None,
+    input: str = "echo bonjour",
+    output: str | None = "Echo: echo bonjour",
+    error: str | None = None,
+    latency_ms: int = 123,
+    model_name: str | None = None,
+    token_usage: dict | None = None,
+    finish_reason: str | None = None,
+    steps: list[dict] | None = None,
+    retrieval_context: list[str] | None = None,
+    tools_called: list[str] | None = None,
+) -> dict:
+    return {
+        "session_id": session_id,
+        "agent_id": agent_id,
+        "agent_tags": agent_tags or [],
+        "input": input,
+        "output": output,
+        "error": error,
+        "latency_ms": latency_ms,
+        "model_name": model_name,
+        "token_usage": token_usage,
+        "finish_reason": finish_reason,
+        "steps": steps or [],
+        "retrieval_context": retrieval_context or [],
+        "tools_called": tools_called or [],
+    }

fred_deepeval_cli/worker_adapter.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+import asyncio
+
+from fred_deepeval_cli.core.evaluator import evaluate_case_sync
+from fred_deepeval_cli.core.models import EvaluationCaseRequest, EvaluationCaseResult
+
+
+async def evaluate_case(
+    request: EvaluationCaseRequest,
+    *,
+    base_url: str,
+    judge=None,
+    access_token: str | None = None,
+) -> EvaluationCaseResult:
+    return await asyncio.to_thread(
+        evaluate_case_sync,
+        base_url,
+        request,
+        judge=judge,
+        access_token=access_token,
+    )

fred_deepeval_cli-0.1.0.dist-info/METADATA

@@ -0,0 +1,70 @@
+Metadata-Version: 2.4
+Name: fred-deepeval-cli
+Version: 0.1.0
+Summary: External CLI for evaluating Fred agent turns via /agents/evaluate
+License: Apache-2.0
+Requires-Python: <3.13,>=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: fred-sdk>=2.0.7
+Requires-Dist: fred-runtime>=2.0.8
+Provides-Extra: dev
+Requires-Dist: bandit>=1.8.6; extra == "dev"
+Requires-Dist: basedpyright==1.31.0; extra == "dev"
+Requires-Dist: detect-secrets>=1.5.0; extra == "dev"
+Requires-Dist: pytest>=8.4.2; extra == "dev"
+Requires-Dist: pytest-cov>=6.2.1; extra == "dev"
+Requires-Dist: pytest-socket>=0.7.0; extra == "dev"
+Requires-Dist: ruff>=0.12.5; extra == "dev"
+Provides-Extra: eval
+Requires-Dist: deepeval; extra == "eval"
+Requires-Dist: litellm; extra == "eval"
+Requires-Dist: python-dotenv; extra == "eval"
+Requires-Dist: rich>=13.0; extra == "eval"
+Requires-Dist: temporalio; extra == "eval"
+
+# fred-deepeval-cli
+
+External CLI for evaluating one Fred agent turn through `POST /agents/evaluate`.
+
+## Purpose
+
+This project provides a small external CLI that:
+- calls a Fred pod `/agents/evaluate` endpoint
+- receives an `EvalTrace`
+- classifies the turn outcome
+- resolves an evaluation preset from `agent_tags`
+- computes structural checks
+- scores the trace with DeepEval
+
+## Commands
+
+```bash
+make dev
+make eval-dev
+make test
+make code-quality
+make cli
+make score BASE_URL=http://127.0.0.1:8000/fred/agents/v2 AGENT_ID=fred.test.assistant INPUT="echo bonjour" SESSION_ID=eval-001 USER_ID=alice
+make sql-scenarios BASE_URL=http://127.0.0.1:8000/fred/agents/v2
+
+## Documentation
+
+| Topic | File |
+| --- | --- |
+| Evaluate any Fred agent pod | `docs/evaluating-any-fred-agent.md` |
+| RAG evaluation — approach and metrics | `docs/rag-evaluation-rfc.md` |
+| RAG local setup guide | `docs/rag-local-setup.md` |
+| SQL evaluation | `docs/sql-evaluation.md` |
+| OTel export strategy | `fred/docs/swift/rfc/AGENT-EVALUATION-RFC.md §13` |
+
+## Architecture — EVAL-01 Phase 1
+
+This CLI is being restructured into a reusable library core so the Fred
+Control Plane evaluation worker can call it directly without spawning a subprocess.
+
+- `fred_deepeval_cli/core/` — callable library (models, evaluator, profiles, scorer, judge factory)
+- `fred_deepeval_cli/cli/` — thin CLI adapter over the core
+- `fred_deepeval_cli/worker_adapter.py` — public entrypoint for the Control Plane worker
+
+The CLI interface and JSON output remain unchanged.
+See EVAL-01 Phase 1 issue and `fred/docs/swift/rfc/AGENT-EVALUATION-RFC.md §7.3`.

fred_deepeval_cli-0.1.0.dist-info/RECORD

@@ -0,0 +1,20 @@
+fred_deepeval_cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fred_deepeval_cli/dataset_workflow.py,sha256=6sbuVdqQd5cAHWDC2_7RVLp3eDMUL5GgerH5BwXnGLs,3845
+fred_deepeval_cli/test_helpers.py,sha256=hNXs76H61svc-c05_Cgv1w1Sa0_owZRX-pve0WG8QQk,1278
+fred_deepeval_cli/worker_adapter.py,sha256=vkbT1BYUVlI-Pe6P7y-dKCZLWPKUysGGSdXeS-CDCwY,531
+fred_deepeval_cli/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fred_deepeval_cli/cli/display.py,sha256=Z4ZWImcdwmGM0s1E0UCmohqZhK_Nt6FzEeeRLaNfeY4,8764
+fred_deepeval_cli/cli/main.py,sha256=BpW0XYnRyTukxOUfj0fS6wHZSSSUgoQfwPKRPjuNS5M,3148
+fred_deepeval_cli/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fred_deepeval_cli/core/config_loader.py,sha256=JzeYdjrcCY7WaoqxLesIMorzdE70zcjMtmzhqrGrldA,1574
+fred_deepeval_cli/core/evaluator.py,sha256=TYMW_nlXQWl8d2hCJ8QhCWY0T_PwjYvp-rr385RFMSs,2873
+fred_deepeval_cli/core/judge_factory.py,sha256=23Crpu1-KLVjh27_DP9asmfOQit9mDqKDy-ja06Q8ik,1603
+fred_deepeval_cli/core/models.py,sha256=Q1LAxPyYJTyiwSsgJEGfsjVxJBWSWtY0fJ2OuVljY9g,1071
+fred_deepeval_cli/core/profiles.py,sha256=6LmKMt6SjQFBKfz0CNiYkmVvlTf_vJl8jid1Lq73UJE,512
+fred_deepeval_cli/core/scorer.py,sha256=dmdLNKWg4ujR2Pn9qkYjUCYanFM8dP6-uz679GjxPGE,2409
+fred_deepeval_cli/core/structural_checks.py,sha256=-NNCZyo4rNeJ0BIKcOh-XdYNrCwOqPZBjQ1MzS35lwA,1827
+fred_deepeval_cli-0.1.0.dist-info/METADATA,sha256=FUbvNEH_yQ8fqPRz3B85QlftG7K9xlC5erZ9toLbSm4,2513
+fred_deepeval_cli-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+fred_deepeval_cli-0.1.0.dist-info/entry_points.txt,sha256=Eq5rEKpSr9gTzWhuU4NOJrOE389NzfnMk8kPxAvmVXw,70
+fred_deepeval_cli-0.1.0.dist-info/top_level.txt,sha256=wUGpDJqehShF9gIzSY1xf07RuzSHi6oq5oJ1SUCBLkk,18
+fred_deepeval_cli-0.1.0.dist-info/RECORD,,

fred_deepeval_cli-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

fred_deepeval_cli-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+fred-deepeval-cli = fred_deepeval_cli.cli.main:main

fred_deepeval_cli-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+fred_deepeval_cli