fluxloop-cli 0.2.31__tar.gz → 0.2.32__tar.gz

{fluxloop_cli-0.2.31 → fluxloop_cli-0.2.32}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fluxloop-cli
-Version: 0.2.31
+Version: 0.2.32
 Summary: FluxLoop CLI for running agent simulations
 Author-email: FluxLoop Team <team@fluxloop.dev>
 License: Apache-2.0
@@ -28,6 +28,7 @@ Requires-Dist: python-dotenv>=1.0.0
 Requires-Dist: fluxloop>=0.1.0
 Requires-Dist: ruamel.yaml>=0.17.0
 Requires-Dist: Jinja2>=3.0
+Requires-Dist: weasyprint>=61.0
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
 Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"

{fluxloop_cli-0.2.31 → fluxloop_cli-0.2.32}/fluxloop_cli/__init__.py

@@ -2,7 +2,7 @@
 FluxLoop CLI - Command-line interface for running agent simulations.
 """
 
-__version__ = "0.2.31"
+__version__ = "0.2.32"
 
 from .main import app
 

{fluxloop_cli-0.2.31 → fluxloop_cli-0.2.32}/fluxloop_cli/commands/evaluate.py

@@ -181,8 +181,14 @@ def experiment(
     ]
     console.print("\n".join(message_lines))
 
-    report_path = asyncio.run(pipeline.run(trace_summaries))
-    console.print(f"\n✅ Report ready: [bold cyan]{report_path}[/bold cyan]")
+    artifacts = asyncio.run(pipeline.run(trace_summaries))
+    console.print(f"\n✅ Report ready: [bold cyan]{artifacts.html_path}[/bold cyan]")
+    if artifacts.pdf_path:
+        console.print(f"🖨️ PDF ready: [bold green]{artifacts.pdf_path}[/bold green]")
+    else:
+        console.print(
+            "⚠️ PDF export skipped (WeasyPrint is unavailable or PDF rendering failed)."
+        )
 
 
 

{fluxloop_cli-0.2.31 → fluxloop_cli-0.2.32}/fluxloop_cli/evaluation/config.py

@@ -229,6 +229,8 @@ class EvaluationConfig:
     additional_analysis: AdditionalAnalysisConfig = field(default_factory=AdditionalAnalysisConfig)
     report: ReportConfig = field(default_factory=ReportConfig)
     advanced: AdvancedConfig = field(default_factory=AdvancedConfig)
+    metrics: Dict[str, Any] = field(default_factory=dict)
+    efficiency: Dict[str, Any] = field(default_factory=dict)
 
     def set_source_dir(self, source_dir: Path) -> None:
         """Remember the directory where this config file was loaded from."""
@@ -383,12 +385,45 @@ def _parse_limits(raw: Any) -> LimitsConfig:
     )
 
 
+def _parse_metrics(raw: Any) -> Dict[str, Any]:
+    if raw is None:
+        return {}
+    if not isinstance(raw, dict):
+        raise ValueError("metrics must be a mapping")
+    normalized: Dict[str, Any] = {}
+    for key, value in raw.items():
+        if not isinstance(value, dict):
+            raise ValueError(f"metrics.{key} must be a mapping")
+        normalized[key] = dict(value)
+    return normalized
+
+
+def _parse_efficiency(raw: Any) -> Dict[str, Any]:
+    if raw is None:
+        return {}
+    if not isinstance(raw, dict):
+        raise ValueError("efficiency must be a mapping")
+    normalized: Dict[str, Any] = {}
+    for key, value in raw.items():
+        if not isinstance(value, dict):
+            raise ValueError(f"efficiency.{key} must be a mapping")
+        normalized[key] = dict(value)
+    return normalized
+
+
 def _parse_evaluation_goal(raw: Any) -> EvaluationGoalConfig:
     if raw is None:
         return EvaluationGoalConfig()
+
+    if isinstance(raw, str):
+        # Backwards compatibility for older templates that used a bare string.
+        return EvaluationGoalConfig(text=raw.strip())
+
     if not isinstance(raw, dict):
-        raise ValueError("evaluation_goal must be a mapping")
-    text = str(raw.get("text", "")) if raw.get("text") is not None else ""
+        raise ValueError("evaluation_goal must be a mapping or string")
+
+    text_value = raw.get("text")
+    text = str(text_value) if text_value is not None else ""
     return EvaluationGoalConfig(text=text)
 
 
@@ -711,6 +746,8 @@ def load_evaluation_config(path: Path) -> EvaluationConfig:
     additional_analysis = _parse_additional_analysis(data.get("additional_analysis"))
     report = _parse_report(data.get("report"))
     advanced = _parse_advanced(data.get("advanced"))
+    metrics = _parse_metrics(data.get("metrics"))
+    efficiency = _parse_efficiency(data.get("efficiency"))
 
     config_dir = path.parent.resolve()
 
@@ -723,6 +760,8 @@ def load_evaluation_config(path: Path) -> EvaluationConfig:
         additional_analysis=additional_analysis,
         report=report,
         advanced=advanced,
+        metrics=metrics,
+        efficiency=efficiency,
     )
 
     config.set_source_dir(config_dir)

{fluxloop_cli-0.2.31 → fluxloop_cli-0.2.32}/fluxloop_cli/evaluation/report/aggregator.py

@@ -2,6 +2,7 @@
 Rule-based aggregation and statistics for evaluation reports.
 """
 
+import json
 import logging
 import statistics
 from datetime import datetime
@@ -115,6 +116,7 @@ class StatsAggregator:
         if not conversation and "conversation_state" in trace_summary:
             conversation = trace_summary.get("conversation_state", {}).get("turns", [])
             
+        raw_output = trace_summary.get("output") or trace_summary.get("final_output")
         return {
             "trace_id": pt_result.get("trace_id"),
             "persona": trace_summary.get("persona", "unknown"),
@@ -124,7 +126,7 @@ class StatsAggregator:
             "overall_eval": self._classify_overall_success(metrics),
             "primary_issue": self._find_primary_issue(metrics),
             "duration_ms": trace_summary.get("duration_ms", 0),
-            "output_tokens": self._count_output_tokens(conversation),
+            "output_tokens": self._estimate_output_tokens(conversation, raw_output),
             "conversation_turns": self._count_conversation_turns(conversation),
         }
 
@@ -174,14 +176,34 @@ class StatsAggregator:
                     return metric
         return None
 
-    def _count_output_tokens(self, conversation: List[Dict[str, Any]]) -> int:
+    def _estimate_output_tokens(
+        self,
+        conversation: List[Dict[str, Any]],
+        raw_output: Optional[Any],
+    ) -> int:
+        """Approximate output tokens from assistant turns, with fallbacks."""
+
         total_chars = 0
         for turn in conversation:
-            if turn.get("role") == "assistant":
-                content = turn.get("content", "")
-                if isinstance(content, str):
-                    total_chars += len(content)
-        return total_chars // 4
+            if turn.get("role") != "assistant":
+                continue
+
+            content = turn.get("content", "")
+            if isinstance(content, dict):
+                content = json.dumps(content, ensure_ascii=False)
+            elif not isinstance(content, str):
+                content = str(content)
+
+            total_chars += len(content)
+
+        if total_chars == 0 and raw_output:
+            fallback = raw_output
+            if isinstance(fallback, dict):
+                fallback = fallback.get("output") or fallback.get("final_output")
+            if isinstance(fallback, str):
+                total_chars = len(fallback)
+
+        return total_chars // 4 if total_chars else 0
 
     def _count_conversation_turns(self, conversation: List[Dict[str, Any]]) -> int:
         return sum(1 for t in conversation if t.get("role") == "user")
@@ -475,23 +497,59 @@ class StatsAggregator:
                 "avg_turns": statistics.mean(d["turns"]) if d["turns"] else 0
             }
             
+        def _slug(text: str) -> str:
+            return (
+                text.lower()
+                .replace(" ", "_")
+                .replace("-", "_")
+                .replace(".", "")
+            )
+
+        def _label(text: str) -> str:
+            return text.replace("_", " ").title()
+
         # Comparison (first 2 personas)
         p1, p2 = persona_names[0], persona_names[1]
         comparisons = []
         
-        for metric, label in [("avg_tokens", "Output Tokens"), ("avg_latency", "Latency"), ("avg_turns", "Conversation Depth")]:
+        class_cycle = ["persona-a", "persona-b", "persona-c"]
+
+        for metric, label in [
+            ("avg_tokens", "Output Tokens"),
+            ("avg_latency", "Latency"),
+            ("avg_turns", "Conversation Depth"),
+        ]:
             v1 = avgs[p1][metric]
             v2 = avgs[p2][metric]
             
             gap = ((v1 - v2) / v2 * 100) if v2 > 0 else 0
             max_val = max(v1, v2) if max(v1, v2) > 0 else 1
             
-            comparisons.append({
-                "label": label,
-                p1: {"value": str(round(v1, 1)), "width": int(v1/max_val*100)},
-                p2: {"value": str(round(v2, 1)), "width": int(v2/max_val*100)},
-                "gap": f"{gap:+.1f}%"
-            })
+            bars = []
+            for idx, (persona_name, value) in enumerate(
+                [
+                    (p1, v1),
+                    (p2, v2),
+                ]
+            ):
+                css_class = class_cycle[idx] if idx < len(class_cycle) else f"persona-{idx}"
+                bars.append(
+                    {
+                        "persona": persona_name,
+                        "label": _label(persona_name),
+                        "class": css_class,
+                        "value": str(round(value, 1)),
+                        "width": int(value / max_val * 100),
+                    }
+                )
+
+            comparisons.append(
+                {
+                    "label": label,
+                    "bars": bars,
+                    "gap": f"{gap:+.1f}%",
+                }
+            )
             
         return {"comparisons": comparisons, "persona_averages": avgs}
 

{fluxloop_cli-0.2.31 → fluxloop_cli-0.2.32}/fluxloop_cli/evaluation/report/generator.py

@@ -620,9 +620,7 @@ class TraceEvaluator:
         if isinstance(goal, dict):
             goal = goal.get("text", str(goal))
 
-        output_lang = self.eval_config.get("report", {}).get("output", {}).get("language", "en")
-        if not isinstance(output_lang, str): # Handle if output is complex dict
-             output_lang = "en"
+        output_lang = self._resolve_output_language(persona.get("language"))
 
         return PT_SYSTEM_PROMPT.format(
             evaluation_goal=goal,
@@ -632,6 +630,33 @@ class TraceEvaluator:
             output_language=output_lang
         )
 
+    def _resolve_output_language(self, persona_language: Optional[str]) -> str:
+        """
+        Determine the best output language with backwards compatibility.
+
+        Preference order:
+        1. Explicit language inside report.output.language
+        2. Legacy report.language (if added later)
+        3. Persona language metadata
+        4. Default to English
+        """
+        report_cfg = self.eval_config.get("report") or {}
+        output_cfg = report_cfg.get("output")
+
+        if isinstance(output_cfg, dict):
+            lang = output_cfg.get("language")
+            if isinstance(lang, str) and lang.strip():
+                return lang.strip()
+
+        legacy_lang = report_cfg.get("language")
+        if isinstance(legacy_lang, str) and legacy_lang.strip():
+            return legacy_lang.strip()
+
+        if isinstance(persona_language, str) and persona_language.strip():
+            return persona_language.strip()
+
+        return "en"
+
 
 class OverallEvaluator:
     """Analyzes aggregated results (LLM-OV)."""
@@ -700,9 +725,7 @@ class OverallEvaluator:
         if isinstance(goal, dict):
             goal = goal.get("text", str(goal))
             
-        output_lang = self.eval_config.get("report", {}).get("output", {}).get("language", "en")
-        if not isinstance(output_lang, str):
-             output_lang = "en"
+        output_lang = self._resolve_output_language()
 
         return OV_PROMPT_TEMPLATE.format(
             evaluation_goal=goal,
@@ -714,3 +737,23 @@ class OverallEvaluator:
             output_language=output_lang
         )
 
+    def _resolve_output_language(self) -> str:
+        """
+        Determine the preferred language for overall evaluation prompts.
+
+        Mirrors TraceEvaluator logic but without persona metadata fallback.
+        """
+        report_cfg = self.eval_config.get("report") or {}
+        output_cfg = report_cfg.get("output")
+
+        if isinstance(output_cfg, dict):
+            lang = output_cfg.get("language")
+            if isinstance(lang, str) and lang.strip():
+                return lang.strip()
+
+        legacy_lang = report_cfg.get("language")
+        if isinstance(legacy_lang, str) and legacy_lang.strip():
+            return legacy_lang.strip()
+
+        return "en"
+

fluxloop_cli-0.2.32/fluxloop_cli/evaluation/report/pdf_exporter.py

@@ -0,0 +1,71 @@
+"""Utilities for exporting the rendered HTML report as a PDF."""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Optional
+
+try:  # pragma: no cover - import guard exercised via tests/log statements
+    from weasyprint import CSS, HTML  # type: ignore
+except ModuleNotFoundError:  # pragma: no cover - handled gracefully at runtime
+    HTML = None  # type: ignore
+    CSS = None  # type: ignore
+
+logger = logging.getLogger(__name__)
+
+
+class PdfExportError(RuntimeError):
+    """Raised when PDF generation fails."""
+
+
+class ReportPdfExporter:
+    """Converts the generated HTML report into a PDF file."""
+
+    def __init__(self, output_dir: Path) -> None:
+        self.output_dir = output_dir
+
+    def export(self, html_report: Path) -> Optional[Path]:
+        """Render ``report.html`` to ``report.pdf`` if WeasyPrint is available."""
+
+        if HTML is None:
+            logger.info(
+                "WeasyPrint is not installed; skipping PDF export for %s", html_report
+            )
+            return None
+
+        if not html_report.exists():
+            raise PdfExportError(f"HTML report not found at {html_report}")
+
+        pdf_path = self.output_dir / "report.pdf"
+        try:
+            stylesheets = [CSS(string=self._default_portrait_css())] if CSS else None
+            HTML(filename=str(html_report), base_url=str(html_report.parent)).write_pdf(
+                target=str(pdf_path),
+                stylesheets=stylesheets,
+            )
+            logger.info("📄 PDF report generated at %s", pdf_path)
+        except Exception as exc:  # pragma: no cover - depends on external renderer
+            raise PdfExportError(f"Failed to generate PDF: {exc}") from exc
+
+        return pdf_path
+
+    def _default_portrait_css(self) -> str:
+        """Ensure the PDF renders in portrait mode for better readability."""
+
+        return """
+        @page {
+            size: A4 portrait;
+            margin: 15mm;
+        }
+
+        html, body {
+            width: 100%;
+            height: auto;
+        }
+
+        .page-break {
+            page-break-after: always;
+        }
+        """
+

{fluxloop_cli-0.2.31 → fluxloop_cli-0.2.32}/fluxloop_cli/evaluation/report/pipeline.py

@@ -4,16 +4,26 @@ Orchestration pipeline for generating evaluation reports.
 
 import asyncio
 import logging
+from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
 
 from .aggregator import StatsAggregator
 from .generator import OverallEvaluator, ReportLLMClient, TraceEvaluator
+from .pdf_exporter import PdfExportError, ReportPdfExporter
 from .renderer import ReportRenderer
 
 logger = logging.getLogger(__name__)
 
 
+@dataclass
+class ReportArtifacts:
+    """Artifacts produced by the evaluation report pipeline."""
+
+    html_path: Path
+    pdf_path: Optional[Path] = None
+
+
 class ReportPipeline:
     """
     Orchestrates the 5-stage evaluation pipeline:
@@ -35,8 +45,9 @@ class ReportPipeline:
         self.aggregator = StatsAggregator(config)
         self.overall_evaluator = OverallEvaluator(self.client, config)
         self.renderer = ReportRenderer(output_dir)
+        self.pdf_exporter = ReportPdfExporter(output_dir)
 
-    async def run(self, trace_summaries: List[Dict[str, Any]]) -> Path:
+    async def run(self, trace_summaries: List[Dict[str, Any]]) -> ReportArtifacts:
         """
         Run the full pipeline.
         
@@ -65,10 +76,21 @@ class ReportPipeline:
         
         # Stage 4 & 5: Render
         logger.info("Stage 4 & 5: Rendering HTML report...")
-        report_path = self.renderer.render(rule_based_data, llm_ov_data, trace_summaries, self.config)
+        report_path = self.renderer.render(
+            rule_based_data, llm_ov_data, trace_summaries, self.config
+        )
         
-        logger.info(f"✅ Pipeline Complete! Report: {report_path}")
-        return report_path
+        pdf_path: Optional[Path] = None
+        try:
+            pdf_path = self.pdf_exporter.export(report_path)
+        except PdfExportError as exc:
+            logger.warning("PDF export failed: %s", exc)
+
+        logger.info("✅ Pipeline Complete! Report: %s", report_path)
+        if pdf_path:
+            logger.info("✅ PDF ready: %s", pdf_path)
+
+        return ReportArtifacts(html_path=report_path, pdf_path=pdf_path)
 
     async def _run_pt_evaluations(self, traces: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
         """Run TraceEvaluator concurrently."""

{fluxloop_cli-0.2.31 → fluxloop_cli-0.2.32}/fluxloop_cli/evaluation/report/renderer.py

@@ -110,9 +110,17 @@ class ReportRenderer:
         # Extract variations from traces if possible, otherwise placeholder
         # In fluxloop execution, we might not have 'generated.yaml' equivalent handy here unless passed.
         # We'll approximate from input config.
+        goal_value = eval_cfg.get("evaluation_goal", "Evaluate AI agent performance")
+        if isinstance(goal_value, dict):
+            goal_value = goal_value.get("text") or ""
+        if goal_value is None:
+            goal_value = ""
+        goal_value = str(goal_value).strip()
+        if not goal_value:
+            goal_value = "Evaluate AI agent performance"
         
         return {
-            "goal": eval_cfg.get("evaluation_goal", "Evaluate AI agent performance"),
+            "goal": goal_value,
             "test_design": {
                 "total_traces": rule_based.get("meta", {}).get("total_traces", 0),
                 "personas_count": len(personas),
@@ -360,12 +368,27 @@ class ReportRenderer:
                     turns.append({"role": "user", "content": t["input"], "turn_index": 0})
                 if t.get("output"):
                     turns.append({"role": "assistant", "content": str(t["output"]), "turn_index": 1})
-            
+
+            summary_section = t.get("summary") or {}
+            success_flag = t.get("success")
+            if success_flag is None:
+                success_flag = summary_section.get("success")
+            if isinstance(success_flag, str):
+                success_flag = success_flag.lower() == "true"
+            duration_ms = t.get("duration_ms")
+            if duration_ms is None:
+                duration_ms = summary_section.get("duration_ms")
+                if isinstance(duration_ms, str) and duration_ms.isdigit():
+                    duration_ms = float(duration_ms)
+
             convs[tid] = {
                 "trace_id": tid,
                 "persona": t.get("persona", ""),
                 "turn_count": len([x for x in turns if x["role"] == "user"]),
-                "turns": turns
+                "turns": turns,
+                "duration_ms": duration_ms,
+                "success": success_flag,
+                "overall_eval": t.get("overall_eval") or summary_section.get("overall_eval"),
             }
         return convs