fluxloop-cli 0.2.31__tar.gz → 0.2.33__tar.gz

{fluxloop_cli-0.2.31 → fluxloop_cli-0.2.33}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fluxloop-cli
-Version: 0.2.31
+Version: 0.2.33
 Summary: FluxLoop CLI for running agent simulations
 Author-email: FluxLoop Team <team@fluxloop.dev>
 License: Apache-2.0
@@ -28,6 +28,7 @@ Requires-Dist: python-dotenv>=1.0.0
 Requires-Dist: fluxloop>=0.1.0
 Requires-Dist: ruamel.yaml>=0.17.0
 Requires-Dist: Jinja2>=3.0
+Requires-Dist: weasyprint>=61.0
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
 Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"

{fluxloop_cli-0.2.31 → fluxloop_cli-0.2.33}/fluxloop_cli/__init__.py

@@ -2,7 +2,7 @@
 FluxLoop CLI - Command-line interface for running agent simulations.
 """
 
-__version__ = "0.2.31"
+__version__ = "0.2.33"
 
 from .main import app
 

{fluxloop_cli-0.2.31 → fluxloop_cli-0.2.33}/fluxloop_cli/commands/evaluate.py

@@ -8,7 +8,7 @@ import os
 import shutil
 from dataclasses import asdict
 from pathlib import Path
-from typing import Optional
+from typing import Dict, List, Optional
 
 import typer
 import yaml
@@ -16,11 +16,12 @@ from rich.console import Console
 
 from ..environment import load_env_chain
 from ..evaluation import load_evaluation_config
-from ..evaluation.artifacts import load_per_trace_records
+from ..evaluation.artifacts import load_per_trace_records, load_trace_summary_records
 from ..evaluation.report.pipeline import ReportPipeline
 
 console = Console()
 app = typer.Typer(help="Evaluate experiment outputs and generate interactive reports.")
+logger = logging.getLogger(__name__)
 
 
 def _load_yaml_file(path: Optional[Path]) -> dict:
@@ -62,6 +63,72 @@ def _prepare_output_directory(path: Path, overwrite: bool) -> None:
     path.mkdir(parents=True, exist_ok=True)
 
 
+def _load_generated_inputs_data(input_config: Dict[str, any], project_root: Path) -> Dict[str, any]:
+    """
+    Load generated inputs (variations) from the configured inputs file, if available.
+    """
+
+    inputs_file_value = input_config.get("inputs_file") or "inputs/generated.yaml"
+    inputs_path = Path(inputs_file_value)
+    if not inputs_path.is_absolute():
+        inputs_path = (project_root / inputs_path).resolve()
+
+    if not inputs_path.exists():
+        logger.debug("Generated inputs file not found at %s", inputs_path)
+        return {}
+
+    try:
+        with inputs_path.open("r", encoding="utf-8") as handle:
+            payload = yaml.safe_load(handle) or {}
+    except Exception as exc:  # noqa: BLE001
+        logger.warning("Failed to load generated inputs file %s: %s", inputs_path, exc)
+        return {}
+
+    inputs_list: List[Dict[str, any]] = []
+    generation_cfg: Dict[str, any] = {}
+    if isinstance(payload, dict):
+        inputs_list = payload.get("inputs") or payload.get("variations") or []
+        generation_cfg = payload.get("generation_config") or {}
+    elif isinstance(payload, list):
+        inputs_list = payload
+    else:
+        logger.debug("Generated inputs file %s did not contain a supported structure", inputs_path)
+        return {}
+
+    variations: List[Dict[str, str]] = []
+    for entry in inputs_list:
+        if not isinstance(entry, dict):
+            continue
+        text = entry.get("input")
+        if not text:
+            continue
+        metadata = entry.get("metadata") or {}
+        persona = entry.get("persona") or metadata.get("persona")
+        strategy = entry.get("strategy") or metadata.get("variation_strategy") or metadata.get("strategy")
+
+        variations.append(
+            {
+                "persona": (persona or "unknown").strip(),
+                "strategy": (strategy or "base").strip(),
+                "input": text.strip(),
+            }
+        )
+
+    generator_model = generation_cfg.get("model") or generation_cfg.get("generator_model")
+    provider = generation_cfg.get("provider")
+    if generator_model and provider and "/" not in str(generator_model):
+        generator_model = f"{provider}/{generator_model}"
+
+    return {
+        "path": str(inputs_path),
+        "variations": variations,
+        "generator_model": generator_model or input_config.get("input_generation", {})
+        .get("llm", {})
+        .get("model"),
+        "strategies": generation_cfg.get("strategies") or input_config.get("variation_strategies", []),
+    }
+
+
 @app.command()
 def experiment(
     experiment_dir: Path = typer.Argument(
@@ -120,6 +187,7 @@ def experiment(
         raise typer.BadParameter(f"Experiment directory not found: {resolved_experiment_dir}")
 
     config_path = config.resolve() if config.is_absolute() else (Path.cwd() / config).resolve()
+    project_root = _resolve_project_root(config_path)
 
     if per_trace is not None:
         per_trace_path = per_trace.resolve() if per_trace.is_absolute() else (Path.cwd() / per_trace).resolve()
@@ -127,10 +195,15 @@ def experiment(
         per_trace_path = resolved_experiment_dir / "per_trace_analysis" / "per_trace.jsonl"
 
     per_trace_records = load_per_trace_records(resolved_experiment_dir, per_trace_path)
-    trace_summaries = [record.trace for record in per_trace_records]
-    if not trace_summaries:
+    trace_records = [record.trace for record in per_trace_records]
+    if not trace_records:
         raise typer.BadParameter("No traces found in per-trace artifacts.")
 
+    trace_summary_path = resolved_experiment_dir / "trace_summary.jsonl"
+    trace_summaries = load_trace_summary_records(resolved_experiment_dir, trace_summary_path)
+    if not trace_summaries:
+        raise typer.BadParameter("No traces found in trace summary artifacts.")
+
     try:
         evaluation_config = load_evaluation_config(config_path)
     except FileNotFoundError as exc:
@@ -160,11 +233,13 @@ def experiment(
 
     input_config = _load_yaml_file(input_config_path)
     project_config = _load_yaml_file(project_config_path)
+    generated_inputs = _load_generated_inputs_data(input_config, project_root)
 
     config_bundle = {
         "name": project_config.get("name") or resolved_experiment_dir.name,
         "evaluation": asdict(evaluation_config),
         "input": input_config,
+        "generated_inputs": generated_inputs,
     }
 
     pipeline = ReportPipeline(
@@ -177,12 +252,19 @@ def experiment(
         f"📊 Evaluating experiment at [cyan]{resolved_experiment_dir}[/cyan]",
         f"⚙️  Config: [magenta]{config_path}[/magenta]",
         f"🧵 Per-trace data: [blue]{per_trace_path}[/blue]",
+        f"📄 Trace summary: [blue]{trace_summary_path}[/blue]",
         f"📁 Output: [green]{output_dir}[/green]",
     ]
     console.print("\n".join(message_lines))
 
-    report_path = asyncio.run(pipeline.run(trace_summaries))
-    console.print(f"\n✅ Report ready: [bold cyan]{report_path}[/bold cyan]")
+    artifacts = asyncio.run(pipeline.run(trace_records, trace_summaries))
+    console.print(f"\n✅ Report ready: [bold cyan]{artifacts.html_path}[/bold cyan]")
+    if artifacts.pdf_path:
+        console.print(f"🖨️ PDF ready: [bold green]{artifacts.pdf_path}[/bold green]")
+    else:
+        console.print(
+            "⚠️ PDF export skipped (WeasyPrint is unavailable or PDF rendering failed)."
+        )
 
 
 

{fluxloop_cli-0.2.31 → fluxloop_cli-0.2.33}/fluxloop_cli/commands/parse.py

@@ -78,6 +78,7 @@ class TraceSummary:
     conversation: Optional[List[Dict[str, Any]]] = None
     conversation_state: Optional[Dict[str, Any]] = None
     termination_reason: Optional[str] = None
+    token_usage: Optional[Dict[str, Any]] = None
 
     def to_payload(self) -> dict:
         """Return a JSON-serialisable representation of the summary entry."""
@@ -94,6 +95,7 @@ class TraceSummary:
             "conversation": self.conversation,
             "conversation_state": self.conversation_state,
             "termination_reason": self.termination_reason,
+            "token_usage": self.token_usage,
         }
 
 
@@ -173,6 +175,7 @@ def _load_trace_summaries(path: Path) -> Iterable[TraceSummary]:
                 conversation=payload.get("conversation"),
                 conversation_state=payload.get("conversation_state"),
                 termination_reason=payload.get("termination_reason"),
+                token_usage=payload.get("token_usage"),
                 raw=payload,
             )
 
@@ -313,6 +316,8 @@ def _build_structured_record(
         "timeline": timeline,
         "conversation": trace.conversation or summary_payload.get("conversation") or [],
     }
+    if trace.token_usage:
+        record["token_usage"] = trace.token_usage
 
     def _maybe_decode_content(value: Any) -> Any:
         if isinstance(value, str):

{fluxloop_cli-0.2.31 → fluxloop_cli-0.2.33}/fluxloop_cli/evaluation/artifacts.py

@@ -1,5 +1,5 @@
 """
-Utilities for working with structured per-trace artifacts produced by `fluxloop parse`.
+Utilities for working with structured experiment artifacts (per-trace + summaries).
 """
 
 from __future__ import annotations
@@ -31,6 +31,10 @@ def _default_per_trace_path(experiment_dir: Path) -> Path:
     return experiment_dir / "per_trace_analysis" / "per_trace.jsonl"
 
 
+def _default_trace_summary_path(experiment_dir: Path) -> Path:
+    return experiment_dir / "trace_summary.jsonl"
+
+
 def load_per_trace_records(
     experiment_dir: Path,
     source_path: Optional[Path] = None,
@@ -70,5 +74,42 @@ def load_per_trace_records(
     return records
 
 
-__all__ = ["PerTraceRecord", "load_per_trace_records"]
+def load_trace_summary_records(
+    experiment_dir: Path,
+    source_path: Optional[Path] = None,
+) -> List[Dict[str, Any]]:
+    """
+    Load trace summary entries produced by `fluxloop run`.
+
+    Returns:
+        List of dictionaries mirroring each line inside ``trace_summary.jsonl``.
+    """
+
+    path = source_path or _default_trace_summary_path(experiment_dir)
+    if not path.exists():
+        raise FileNotFoundError(
+            f"Trace summary artifacts not found at {path}. "
+            "Ensure `fluxloop run` completed successfully."
+        )
+
+    records: List[Dict[str, Any]] = []
+    with path.open("r", encoding="utf-8") as handle:
+        for line_no, line in enumerate(handle, start=1):
+            stripped = line.strip()
+            if not stripped:
+                continue
+            try:
+                payload = json.loads(stripped)
+            except json.JSONDecodeError as exc:
+                raise ValueError(f"Invalid JSON in {path} at line {line_no}: {exc}") from exc
+            if not isinstance(payload, dict):
+                raise ValueError(
+                    f"Trace summary entry at line {line_no} must be a JSON object, "
+                    f"but got {type(payload).__name__}"
+                )
+            records.append(payload)
+
+    return records
+
 
+__all__ = ["PerTraceRecord", "load_per_trace_records", "load_trace_summary_records"]

{fluxloop_cli-0.2.31 → fluxloop_cli-0.2.33}/fluxloop_cli/evaluation/config.py

@@ -229,6 +229,8 @@ class EvaluationConfig:
     additional_analysis: AdditionalAnalysisConfig = field(default_factory=AdditionalAnalysisConfig)
     report: ReportConfig = field(default_factory=ReportConfig)
     advanced: AdvancedConfig = field(default_factory=AdvancedConfig)
+    metrics: Dict[str, Any] = field(default_factory=dict)
+    efficiency: Dict[str, Any] = field(default_factory=dict)
 
     def set_source_dir(self, source_dir: Path) -> None:
         """Remember the directory where this config file was loaded from."""
@@ -383,12 +385,45 @@ def _parse_limits(raw: Any) -> LimitsConfig:
     )
 
 
+def _parse_metrics(raw: Any) -> Dict[str, Any]:
+    if raw is None:
+        return {}
+    if not isinstance(raw, dict):
+        raise ValueError("metrics must be a mapping")
+    normalized: Dict[str, Any] = {}
+    for key, value in raw.items():
+        if not isinstance(value, dict):
+            raise ValueError(f"metrics.{key} must be a mapping")
+        normalized[key] = dict(value)
+    return normalized
+
+
+def _parse_efficiency(raw: Any) -> Dict[str, Any]:
+    if raw is None:
+        return {}
+    if not isinstance(raw, dict):
+        raise ValueError("efficiency must be a mapping")
+    normalized: Dict[str, Any] = {}
+    for key, value in raw.items():
+        if not isinstance(value, dict):
+            raise ValueError(f"efficiency.{key} must be a mapping")
+        normalized[key] = dict(value)
+    return normalized
+
+
 def _parse_evaluation_goal(raw: Any) -> EvaluationGoalConfig:
     if raw is None:
         return EvaluationGoalConfig()
+
+    if isinstance(raw, str):
+        # Backwards compatibility for older templates that used a bare string.
+        return EvaluationGoalConfig(text=raw.strip())
+
     if not isinstance(raw, dict):
-        raise ValueError("evaluation_goal must be a mapping")
-    text = str(raw.get("text", "")) if raw.get("text") is not None else ""
+        raise ValueError("evaluation_goal must be a mapping or string")
+
+    text_value = raw.get("text")
+    text = str(text_value) if text_value is not None else ""
     return EvaluationGoalConfig(text=text)
 
 
@@ -711,6 +746,8 @@ def load_evaluation_config(path: Path) -> EvaluationConfig:
     additional_analysis = _parse_additional_analysis(data.get("additional_analysis"))
     report = _parse_report(data.get("report"))
     advanced = _parse_advanced(data.get("advanced"))
+    metrics = _parse_metrics(data.get("metrics"))
+    efficiency = _parse_efficiency(data.get("efficiency"))
 
     config_dir = path.parent.resolve()
 
@@ -723,6 +760,8 @@ def load_evaluation_config(path: Path) -> EvaluationConfig:
         additional_analysis=additional_analysis,
         report=report,
         advanced=advanced,
+        metrics=metrics,
+        efficiency=efficiency,
     )
 
     config.set_source_dir(config_dir)

{fluxloop_cli-0.2.31 → fluxloop_cli-0.2.33}/fluxloop_cli/evaluation/report/aggregator.py

@@ -2,6 +2,7 @@
 Rule-based aggregation and statistics for evaluation reports.
 """
 
+import json
 import logging
 import statistics
 from datetime import datetime
@@ -115,6 +116,7 @@ class StatsAggregator:
         if not conversation and "conversation_state" in trace_summary:
             conversation = trace_summary.get("conversation_state", {}).get("turns", [])
             
+        raw_output = trace_summary.get("output") or trace_summary.get("final_output")
         return {
             "trace_id": pt_result.get("trace_id"),
             "persona": trace_summary.get("persona", "unknown"),
@@ -124,7 +126,7 @@ class StatsAggregator:
             "overall_eval": self._classify_overall_success(metrics),
             "primary_issue": self._find_primary_issue(metrics),
             "duration_ms": trace_summary.get("duration_ms", 0),
-            "output_tokens": self._count_output_tokens(conversation),
+            "output_tokens": self._estimate_output_tokens(conversation, raw_output),
             "conversation_turns": self._count_conversation_turns(conversation),
         }
 
@@ -174,14 +176,34 @@ class StatsAggregator:
                     return metric
         return None
 
-    def _count_output_tokens(self, conversation: List[Dict[str, Any]]) -> int:
+    def _estimate_output_tokens(
+        self,
+        conversation: List[Dict[str, Any]],
+        raw_output: Optional[Any],
+    ) -> int:
+        """Approximate output tokens from assistant turns, with fallbacks."""
+
         total_chars = 0
         for turn in conversation:
-            if turn.get("role") == "assistant":
-                content = turn.get("content", "")
-                if isinstance(content, str):
-                    total_chars += len(content)
-        return total_chars // 4
+            if turn.get("role") != "assistant":
+                continue
+
+            content = turn.get("content", "")
+            if isinstance(content, dict):
+                content = json.dumps(content, ensure_ascii=False)
+            elif not isinstance(content, str):
+                content = str(content)
+
+            total_chars += len(content)
+
+        if total_chars == 0 and raw_output:
+            fallback = raw_output
+            if isinstance(fallback, dict):
+                fallback = fallback.get("output") or fallback.get("final_output")
+            if isinstance(fallback, str):
+                total_chars = len(fallback)
+
+        return total_chars // 4 if total_chars else 0
 
     def _count_conversation_turns(self, conversation: List[Dict[str, Any]]) -> int:
         return sum(1 for t in conversation if t.get("role") == "user")
@@ -475,23 +497,59 @@ class StatsAggregator:
                 "avg_turns": statistics.mean(d["turns"]) if d["turns"] else 0
             }
             
+        def _slug(text: str) -> str:
+            return (
+                text.lower()
+                .replace(" ", "_")
+                .replace("-", "_")
+                .replace(".", "")
+            )
+
+        def _label(text: str) -> str:
+            return text.replace("_", " ").title()
+
         # Comparison (first 2 personas)
         p1, p2 = persona_names[0], persona_names[1]
         comparisons = []
         
-        for metric, label in [("avg_tokens", "Output Tokens"), ("avg_latency", "Latency"), ("avg_turns", "Conversation Depth")]:
+        class_cycle = ["persona-a", "persona-b", "persona-c"]
+
+        for metric, label in [
+            ("avg_tokens", "Output Tokens"),
+            ("avg_latency", "Latency"),
+            ("avg_turns", "Conversation Depth"),
+        ]:
             v1 = avgs[p1][metric]
             v2 = avgs[p2][metric]
             
             gap = ((v1 - v2) / v2 * 100) if v2 > 0 else 0
             max_val = max(v1, v2) if max(v1, v2) > 0 else 1
             
-            comparisons.append({
-                "label": label,
-                p1: {"value": str(round(v1, 1)), "width": int(v1/max_val*100)},
-                p2: {"value": str(round(v2, 1)), "width": int(v2/max_val*100)},
-                "gap": f"{gap:+.1f}%"
-            })
+            bars = []
+            for idx, (persona_name, value) in enumerate(
+                [
+                    (p1, v1),
+                    (p2, v2),
+                ]
+            ):
+                css_class = class_cycle[idx] if idx < len(class_cycle) else f"persona-{idx}"
+                bars.append(
+                    {
+                        "persona": persona_name,
+                        "label": _label(persona_name),
+                        "class": css_class,
+                        "value": str(round(value, 1)),
+                        "width": int(value / max_val * 100),
+                    }
+                )
+
+            comparisons.append(
+                {
+                    "label": label,
+                    "bars": bars,
+                    "gap": f"{gap:+.1f}%",
+                }
+            )
             
         return {"comparisons": comparisons, "persona_averages": avgs}
 
@@ -520,7 +578,7 @@ class StatsAggregator:
                     "tag": analysis.get("tag", "Unknown"),
                     "failed_badges": [{"icon": "✗", "label": m} for m in base["failed_metrics"]],
                     "issue_summary": analysis.get("issue_summary", ""),
-                    "conversation_timeline": analysis.get("conversation_timeline", []),
+                    "conversation_timeline": self._format_timeline(analysis.get("conversation_timeline", [])),
                     "root_cause": analysis.get("root_cause", ""),
                     "quick_fixes": analysis.get("quick_fixes", [])
                 })
@@ -566,3 +624,19 @@ class StatsAggregator:
                     badges.append({"status": "fail", "icon": "✗", "label": label})
         return badges
 
+    def _format_timeline(self, timeline: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        formatted: List[Dict[str, Any]] = []
+        for entry in timeline:
+            if not isinstance(entry, dict):
+                continue
+            text = entry.get("text") or entry.get("summary") or ""
+            formatted.append(
+                {
+                    "turn": entry.get("turn"),
+                    "role": entry.get("role", "assistant"),
+                    "text": text,
+                    "is_highlight": entry.get("is_highlight", False),
+                }
+            )
+        return formatted
+