fluxloop-cli 0.2.18__tar.gz → 0.2.20__tar.gz

{fluxloop_cli-0.2.18 → fluxloop_cli-0.2.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fluxloop-cli
-Version: 0.2.18
+Version: 0.2.20
 Summary: FluxLoop CLI for running agent simulations
 Author-email: FluxLoop Team <team@fluxloop.dev>
 License: Apache-2.0
@@ -66,7 +66,7 @@ The legacy `setting.yaml` is still supported, but new projects created with
 - `fluxloop generate inputs` – produce input variations for the active project
 - `fluxloop run experiment` – execute an experiment using `configs/simulation.yaml`
 - `fluxloop parse experiment` – convert experiment outputs into readable artifacts
-- `fluxloop evaluate experiment` – score experiment outputs and generate evaluation reports
+- `fluxloop evaluate experiment` – score experiment outputs using rule-based and LLM evaluators, generate reports with success criteria, analysis, and customizable templates
 - `fluxloop config set-llm` – update LLM provider/model in `configs/input.yaml`
 - `fluxloop record enable|disable|status` – toggle recording mode across `.env` and simulation config
 - `fluxloop doctor` – summarize Python, FluxLoop CLI/MCP, and MCP index state for the active environment

{fluxloop_cli-0.2.18 → fluxloop_cli-0.2.20}/README.md

@@ -26,7 +26,7 @@ The legacy `setting.yaml` is still supported, but new projects created with
 - `fluxloop generate inputs` – produce input variations for the active project
 - `fluxloop run experiment` – execute an experiment using `configs/simulation.yaml`
 - `fluxloop parse experiment` – convert experiment outputs into readable artifacts
-- `fluxloop evaluate experiment` – score experiment outputs and generate evaluation reports
+- `fluxloop evaluate experiment` – score experiment outputs using rule-based and LLM evaluators, generate reports with success criteria, analysis, and customizable templates
 - `fluxloop config set-llm` – update LLM provider/model in `configs/input.yaml`
 - `fluxloop record enable|disable|status` – toggle recording mode across `.env` and simulation config
 - `fluxloop doctor` – summarize Python, FluxLoop CLI/MCP, and MCP index state for the active environment

{fluxloop_cli-0.2.18 → fluxloop_cli-0.2.20}/fluxloop_cli/__init__.py

@@ -2,7 +2,7 @@
 FluxLoop CLI - Command-line interface for running agent simulations.
 """
 
-__version__ = "0.2.18"
+__version__ = "0.2.20"
 
 from .main import app
 

{fluxloop_cli-0.2.18 → fluxloop_cli-0.2.20}/fluxloop_cli/commands/init.py

@@ -137,27 +137,11 @@ def project(
         section_path = section_paths[key]
         section_path.write_text(content)
     
-    # Create project .env (unified)
-    root_env_file = root_dir / ".env"
+    # Create project .env (single source of truth)
     console.print("🔐 Creating project .env...")
     recordings_dir = project_path / "recordings"
     recordings_dir.mkdir(exist_ok=True)
-
-    # If a root .env exists, seed project .env with its contents so project can override locally
-    if root_env_file.exists():
-        try:
-            root_contents = root_env_file.read_text()
-        except Exception:
-            root_contents = ""
-        merged = (
-            "# Project .env (seeded from FluxLoop root .env)\n"
-            + (root_contents if root_contents.endswith("\n") else root_contents + "\n")
-            + "\n# Project-specific overrides (take precedence)\n"
-        )
-        env_file.write_text(merged)
-        console.print("[dim]Seeded from root .env (project overrides take precedence).[/dim]")
-    else:
-        env_file.write_text(create_env_file())
+    env_file.write_text(create_env_file())
     
     # Update .gitignore
     if not gitignore_file.exists():

{fluxloop_cli-0.2.18 → fluxloop_cli-0.2.20}/fluxloop_cli/evaluation/config.py

@@ -32,6 +32,7 @@ class EvaluatorConfig:
     prompt_template: Optional[str] = None
     max_score: Optional[float] = None
     parser: Optional[str] = None
+    model_parameters: Dict[str, Any] = field(default_factory=dict)
     metadata: Dict[str, Any] = field(default_factory=dict)
 
 
@@ -281,6 +282,14 @@ def _parse_evaluators(raw_evaluators: Any) -> List[EvaluatorConfig]:
         if evaluator_type not in {"rule_based", "llm_judge"}:
             raise ValueError(f"Unsupported evaluator type: {evaluator_type}")
 
+        model_parameters_raw = entry.get("model_parameters")
+        if model_parameters_raw is None:
+            model_parameters: Dict[str, Any] = {}
+        elif isinstance(model_parameters_raw, dict):
+            model_parameters = dict(model_parameters_raw)
+        else:
+            raise ValueError("model_parameters must be a mapping when provided")
+
         config = EvaluatorConfig(
             name=str(entry["name"]),
             type=evaluator_type,  # type: ignore[arg-type]
@@ -291,6 +300,7 @@ def _parse_evaluators(raw_evaluators: Any) -> List[EvaluatorConfig]:
             prompt_template=entry.get("prompt_template"),
             max_score=entry.get("max_score"),
             parser=entry.get("parser"),
+            model_parameters=model_parameters,
             metadata={
                 k: v
                 for k, v in entry.items()
@@ -305,6 +315,7 @@ def _parse_evaluators(raw_evaluators: Any) -> List[EvaluatorConfig]:
                     "prompt_template",
                     "max_score",
                     "parser",
+                    "model_parameters",
                 }
             },
         )

{fluxloop_cli-0.2.18 → fluxloop_cli-0.2.20}/fluxloop_cli/evaluation/engine/analysis.py

@@ -188,6 +188,141 @@ def _compare_to_baseline(current: Dict[str, Any], baseline: Dict[str, Any]) -> D
     return comparisons
 
 
+def _format_percent(value: Optional[float]) -> str:
+    if value is None:
+        return "—"
+    return f"{value * 100:.1f}%"
+
+
+def _generate_recommendations(
+    summary: Dict[str, Any],
+    analysis: Dict[str, Any],
+) -> List[Dict[str, Any]]:
+    items: List[Dict[str, Any]] = []
+
+    pass_rate = summary.get("pass_rate")
+    threshold = summary.get("threshold")
+    if isinstance(pass_rate, (int, float)) and isinstance(threshold, (int, float)):
+        if pass_rate < threshold:
+            top_reason = None
+            top_reasons = summary.get("top_reasons") or []
+            if top_reasons:
+                top_reason = top_reasons[0][0]
+            summary_text = (
+                f"Pass rate is {_format_percent(pass_rate)} (goal {_format_percent(threshold)})."
+            )
+            if top_reason:
+                summary_text += f" Most common failure reason: {top_reason}."
+            items.append(
+                {
+                    "title": "Boost overall pass rate",
+                    "priority": "high",
+                    "summary": summary_text,
+                    "metrics": {
+                        "pass_rate": pass_rate,
+                        "threshold": threshold,
+                    },
+                }
+            )
+
+    persona_breakdown = summary.get("persona_breakdown") or {}
+    if isinstance(persona_breakdown, dict) and isinstance(threshold, (int, float)):
+        for persona, stats in persona_breakdown.items():
+            persona_pass_rate = stats.get("pass_rate")
+            if isinstance(persona_pass_rate, (int, float)) and persona_pass_rate < threshold:
+                items.append(
+                    {
+                        "title": f"Target persona '{persona}'",
+                        "priority": "medium",
+                        "summary": (
+                            f"{persona} pass rate is {_format_percent(persona_pass_rate)} "
+                            f"(goal {_format_percent(threshold)}). Prioritize playbooks or prompts for this persona."
+                        ),
+                        "metrics": {
+                            "persona": persona,
+                            "pass_rate": persona_pass_rate,
+                            "threshold": threshold,
+                        },
+                    }
+                )
+
+    comparison = analysis.get("comparison")
+    if isinstance(comparison, dict):
+        evaluator_deltas = comparison.get("evaluator_averages") or {}
+        if isinstance(evaluator_deltas, dict):
+            for evaluator_name, payload in evaluator_deltas.items():
+                delta = payload.get("delta")
+                if isinstance(delta, (int, float)) and delta < -0.05:
+                    items.append(
+                        {
+                            "title": f"Recover regression in {evaluator_name}",
+                            "priority": "high",
+                            "summary": (
+                                f"{evaluator_name} average score dropped by {delta:.3f} "
+                                "vs. baseline. Investigate prompt or flow changes."
+                            ),
+                            "metrics": {
+                                "evaluator": evaluator_name,
+                                "delta": delta,
+                                "baseline": payload.get("baseline"),
+                                "current": payload.get("current"),
+                            },
+                        }
+                    )
+
+    failures = analysis.get("failures")
+    if isinstance(failures, dict):
+        categorized = failures.get("categorized") or {}
+        if isinstance(categorized, dict) and categorized:
+            top_failure = max(categorized.items(), key=lambda item: item[1])
+            category, count = top_failure
+            if count:
+                items.append(
+                    {
+                        "title": f"Address {category.replace('_', ' ')} failures",
+                        "priority": "medium",
+                        "summary": (
+                            f"{count} traces failed due to {category.replace('_', ' ')} issues. "
+                            "Review tooling and guardrails for this category."
+                        ),
+                        "metrics": {
+                            "category": category,
+                            "count": count,
+                        },
+                    }
+                )
+
+    performance = analysis.get("performance")
+    if isinstance(performance, dict):
+        trends = performance.get("trends") or {}
+        if isinstance(trends, dict):
+            final_score_trend = trends.get("final_score")
+            if isinstance(final_score_trend, dict) and final_score_trend.get("trend") == "decreasing":
+                slope = final_score_trend.get("slope")
+                items.append(
+                    {
+                        "title": "Investigate declining trend",
+                        "priority": "medium",
+                        "summary": (
+                            "Average final score shows a decreasing trend. "
+                            "Review recent dataset or agent updates."
+                        )
+                        + (f" Slope: {slope:.3f}." if isinstance(slope, (int, float)) else ""),
+                        "metrics": {
+                            "trend": final_score_trend.get("trend"),
+                            "slope": slope,
+                        },
+                    }
+                )
+
+    # Deduplicate by title to avoid noisy recommendations.
+    unique: Dict[str, Dict[str, Any]] = {}
+    for item in items:
+        unique.setdefault(item["title"], item)
+
+    return list(unique.values())
+
+
 def compute_additional_analysis(
     results: List["TraceOutcome"],
     summary: Dict[str, Any],
@@ -255,6 +390,10 @@ def compute_additional_analysis(
         analysis.setdefault("comparison", {})
         analysis["comparison"]["message"] = "Comparison enabled but no baseline path provided."
 
+    recommendations = _generate_recommendations(summary, analysis)
+    if recommendations:
+        analysis["recommendations"] = recommendations
+
     return analysis