fluxloop-cli 0.2.32__tar.gz → 0.2.34__tar.gz

{fluxloop_cli-0.2.32 → fluxloop_cli-0.2.34}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fluxloop-cli
-Version: 0.2.32
+Version: 0.2.34
 Summary: FluxLoop CLI for running agent simulations
 Author-email: FluxLoop Team <team@fluxloop.dev>
 License: Apache-2.0
@@ -28,7 +28,6 @@ Requires-Dist: python-dotenv>=1.0.0
 Requires-Dist: fluxloop>=0.1.0
 Requires-Dist: ruamel.yaml>=0.17.0
 Requires-Dist: Jinja2>=3.0
-Requires-Dist: weasyprint>=61.0
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
 Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"

{fluxloop_cli-0.2.32 → fluxloop_cli-0.2.34}/fluxloop_cli/__init__.py

@@ -2,7 +2,7 @@
 FluxLoop CLI - Command-line interface for running agent simulations.
 """
 
-__version__ = "0.2.32"
+__version__ = "0.2.34"
 
 from .main import app
 

{fluxloop_cli-0.2.32 → fluxloop_cli-0.2.34}/fluxloop_cli/commands/evaluate.py

@@ -8,7 +8,7 @@ import os
 import shutil
 from dataclasses import asdict
 from pathlib import Path
-from typing import Optional
+from typing import Dict, List, Optional
 
 import typer
 import yaml
@@ -16,11 +16,12 @@ from rich.console import Console
 
 from ..environment import load_env_chain
 from ..evaluation import load_evaluation_config
-from ..evaluation.artifacts import load_per_trace_records
+from ..evaluation.artifacts import load_per_trace_records, load_trace_summary_records
 from ..evaluation.report.pipeline import ReportPipeline
 
 console = Console()
 app = typer.Typer(help="Evaluate experiment outputs and generate interactive reports.")
+logger = logging.getLogger(__name__)
 
 
 def _load_yaml_file(path: Optional[Path]) -> dict:
@@ -62,6 +63,72 @@ def _prepare_output_directory(path: Path, overwrite: bool) -> None:
     path.mkdir(parents=True, exist_ok=True)
 
 
+def _load_generated_inputs_data(input_config: Dict[str, any], project_root: Path) -> Dict[str, any]:
+    """
+    Load generated inputs (variations) from the configured inputs file, if available.
+    """
+
+    inputs_file_value = input_config.get("inputs_file") or "inputs/generated.yaml"
+    inputs_path = Path(inputs_file_value)
+    if not inputs_path.is_absolute():
+        inputs_path = (project_root / inputs_path).resolve()
+
+    if not inputs_path.exists():
+        logger.debug("Generated inputs file not found at %s", inputs_path)
+        return {}
+
+    try:
+        with inputs_path.open("r", encoding="utf-8") as handle:
+            payload = yaml.safe_load(handle) or {}
+    except Exception as exc:  # noqa: BLE001
+        logger.warning("Failed to load generated inputs file %s: %s", inputs_path, exc)
+        return {}
+
+    inputs_list: List[Dict[str, any]] = []
+    generation_cfg: Dict[str, any] = {}
+    if isinstance(payload, dict):
+        inputs_list = payload.get("inputs") or payload.get("variations") or []
+        generation_cfg = payload.get("generation_config") or {}
+    elif isinstance(payload, list):
+        inputs_list = payload
+    else:
+        logger.debug("Generated inputs file %s did not contain a supported structure", inputs_path)
+        return {}
+
+    variations: List[Dict[str, str]] = []
+    for entry in inputs_list:
+        if not isinstance(entry, dict):
+            continue
+        text = entry.get("input")
+        if not text:
+            continue
+        metadata = entry.get("metadata") or {}
+        persona = entry.get("persona") or metadata.get("persona")
+        strategy = entry.get("strategy") or metadata.get("variation_strategy") or metadata.get("strategy")
+
+        variations.append(
+            {
+                "persona": (persona or "unknown").strip(),
+                "strategy": (strategy or "base").strip(),
+                "input": text.strip(),
+            }
+        )
+
+    generator_model = generation_cfg.get("model") or generation_cfg.get("generator_model")
+    provider = generation_cfg.get("provider")
+    if generator_model and provider and "/" not in str(generator_model):
+        generator_model = f"{provider}/{generator_model}"
+
+    return {
+        "path": str(inputs_path),
+        "variations": variations,
+        "generator_model": generator_model or input_config.get("input_generation", {})
+        .get("llm", {})
+        .get("model"),
+        "strategies": generation_cfg.get("strategies") or input_config.get("variation_strategies", []),
+    }
+
+
 @app.command()
 def experiment(
     experiment_dir: Path = typer.Argument(
@@ -120,6 +187,7 @@ def experiment(
         raise typer.BadParameter(f"Experiment directory not found: {resolved_experiment_dir}")
 
     config_path = config.resolve() if config.is_absolute() else (Path.cwd() / config).resolve()
+    project_root = _resolve_project_root(config_path)
 
     if per_trace is not None:
         per_trace_path = per_trace.resolve() if per_trace.is_absolute() else (Path.cwd() / per_trace).resolve()
@@ -127,10 +195,15 @@ def experiment(
         per_trace_path = resolved_experiment_dir / "per_trace_analysis" / "per_trace.jsonl"
 
     per_trace_records = load_per_trace_records(resolved_experiment_dir, per_trace_path)
-    trace_summaries = [record.trace for record in per_trace_records]
-    if not trace_summaries:
+    trace_records = [record.trace for record in per_trace_records]
+    if not trace_records:
         raise typer.BadParameter("No traces found in per-trace artifacts.")
 
+    trace_summary_path = resolved_experiment_dir / "trace_summary.jsonl"
+    trace_summaries = load_trace_summary_records(resolved_experiment_dir, trace_summary_path)
+    if not trace_summaries:
+        raise typer.BadParameter("No traces found in trace summary artifacts.")
+
     try:
         evaluation_config = load_evaluation_config(config_path)
     except FileNotFoundError as exc:
@@ -160,11 +233,13 @@ def experiment(
 
     input_config = _load_yaml_file(input_config_path)
     project_config = _load_yaml_file(project_config_path)
+    generated_inputs = _load_generated_inputs_data(input_config, project_root)
 
     config_bundle = {
         "name": project_config.get("name") or resolved_experiment_dir.name,
         "evaluation": asdict(evaluation_config),
         "input": input_config,
+        "generated_inputs": generated_inputs,
     }
 
     pipeline = ReportPipeline(
@@ -177,18 +252,13 @@ def experiment(
         f"📊 Evaluating experiment at [cyan]{resolved_experiment_dir}[/cyan]",
         f"⚙️  Config: [magenta]{config_path}[/magenta]",
         f"🧵 Per-trace data: [blue]{per_trace_path}[/blue]",
+        f"📄 Trace summary: [blue]{trace_summary_path}[/blue]",
         f"📁 Output: [green]{output_dir}[/green]",
     ]
     console.print("\n".join(message_lines))
 
-    artifacts = asyncio.run(pipeline.run(trace_summaries))
+    artifacts = asyncio.run(pipeline.run(trace_records, trace_summaries))
     console.print(f"\n✅ Report ready: [bold cyan]{artifacts.html_path}[/bold cyan]")
-    if artifacts.pdf_path:
-        console.print(f"🖨️ PDF ready: [bold green]{artifacts.pdf_path}[/bold green]")
-    else:
-        console.print(
-            "⚠️ PDF export skipped (WeasyPrint is unavailable or PDF rendering failed)."
-        )
 
 
 

{fluxloop_cli-0.2.32 → fluxloop_cli-0.2.34}/fluxloop_cli/commands/parse.py

@@ -78,6 +78,7 @@ class TraceSummary:
     conversation: Optional[List[Dict[str, Any]]] = None
     conversation_state: Optional[Dict[str, Any]] = None
     termination_reason: Optional[str] = None
+    token_usage: Optional[Dict[str, Any]] = None
 
     def to_payload(self) -> dict:
         """Return a JSON-serialisable representation of the summary entry."""
@@ -94,6 +95,7 @@ class TraceSummary:
             "conversation": self.conversation,
             "conversation_state": self.conversation_state,
             "termination_reason": self.termination_reason,
+            "token_usage": self.token_usage,
         }
 
 
@@ -173,6 +175,7 @@ def _load_trace_summaries(path: Path) -> Iterable[TraceSummary]:
                 conversation=payload.get("conversation"),
                 conversation_state=payload.get("conversation_state"),
                 termination_reason=payload.get("termination_reason"),
+                token_usage=payload.get("token_usage"),
                 raw=payload,
             )
 
@@ -313,6 +316,8 @@ def _build_structured_record(
         "timeline": timeline,
         "conversation": trace.conversation or summary_payload.get("conversation") or [],
     }
+    if trace.token_usage:
+        record["token_usage"] = trace.token_usage
 
     def _maybe_decode_content(value: Any) -> Any:
         if isinstance(value, str):

{fluxloop_cli-0.2.32 → fluxloop_cli-0.2.34}/fluxloop_cli/evaluation/artifacts.py

@@ -1,5 +1,5 @@
 """
-Utilities for working with structured per-trace artifacts produced by `fluxloop parse`.
+Utilities for working with structured experiment artifacts (per-trace + summaries).
 """
 
 from __future__ import annotations
@@ -31,6 +31,10 @@ def _default_per_trace_path(experiment_dir: Path) -> Path:
     return experiment_dir / "per_trace_analysis" / "per_trace.jsonl"
 
 
+def _default_trace_summary_path(experiment_dir: Path) -> Path:
+    return experiment_dir / "trace_summary.jsonl"
+
+
 def load_per_trace_records(
     experiment_dir: Path,
     source_path: Optional[Path] = None,
@@ -70,5 +74,42 @@ def load_per_trace_records(
     return records
 
 
-__all__ = ["PerTraceRecord", "load_per_trace_records"]
+def load_trace_summary_records(
+    experiment_dir: Path,
+    source_path: Optional[Path] = None,
+) -> List[Dict[str, Any]]:
+    """
+    Load trace summary entries produced by `fluxloop run`.
+
+    Returns:
+        List of dictionaries mirroring each line inside ``trace_summary.jsonl``.
+    """
+
+    path = source_path or _default_trace_summary_path(experiment_dir)
+    if not path.exists():
+        raise FileNotFoundError(
+            f"Trace summary artifacts not found at {path}. "
+            "Ensure `fluxloop run` completed successfully."
+        )
+
+    records: List[Dict[str, Any]] = []
+    with path.open("r", encoding="utf-8") as handle:
+        for line_no, line in enumerate(handle, start=1):
+            stripped = line.strip()
+            if not stripped:
+                continue
+            try:
+                payload = json.loads(stripped)
+            except json.JSONDecodeError as exc:
+                raise ValueError(f"Invalid JSON in {path} at line {line_no}: {exc}") from exc
+            if not isinstance(payload, dict):
+                raise ValueError(
+                    f"Trace summary entry at line {line_no} must be a JSON object, "
+                    f"but got {type(payload).__name__}"
+                )
+            records.append(payload)
+
+    return records
+
 
+__all__ = ["PerTraceRecord", "load_per_trace_records", "load_trace_summary_records"]

{fluxloop_cli-0.2.32 → fluxloop_cli-0.2.34}/fluxloop_cli/evaluation/report/aggregator.py

@@ -578,7 +578,7 @@ class StatsAggregator:
                     "tag": analysis.get("tag", "Unknown"),
                     "failed_badges": [{"icon": "✗", "label": m} for m in base["failed_metrics"]],
                     "issue_summary": analysis.get("issue_summary", ""),
-                    "conversation_timeline": analysis.get("conversation_timeline", []),
+                    "conversation_timeline": self._format_timeline(analysis.get("conversation_timeline", [])),
                     "root_cause": analysis.get("root_cause", ""),
                     "quick_fixes": analysis.get("quick_fixes", [])
                 })
@@ -624,3 +624,19 @@ class StatsAggregator:
                     badges.append({"status": "fail", "icon": "✗", "label": label})
         return badges
 
+    def _format_timeline(self, timeline: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        formatted: List[Dict[str, Any]] = []
+        for entry in timeline:
+            if not isinstance(entry, dict):
+                continue
+            text = entry.get("text") or entry.get("summary") or ""
+            formatted.append(
+                {
+                    "turn": entry.get("turn"),
+                    "role": entry.get("role", "assistant"),
+                    "text": text,
+                    "is_highlight": entry.get("is_highlight", False),
+                }
+            )
+        return formatted
+

{fluxloop_cli-0.2.32 → fluxloop_cli-0.2.34}/fluxloop_cli/evaluation/report/generator.py

@@ -4,7 +4,7 @@ LLM-based evaluators for report generation (Per-Trace and Overall).
 
 import json
 import logging
-from typing import Any, Dict, Optional, Tuple
+from typing import Any, Dict, List, Optional, Tuple
 
 logger = logging.getLogger(__name__)
 
@@ -592,8 +592,9 @@ class TraceEvaluator:
         persona = self.personas.get(persona_name, {"name": persona_name, "description": "", "characteristics": []})
 
         system_prompt = self._build_system_prompt(persona)
+        prompt_payload = self._build_prompt_payload(trace)
         user_message = PT_USER_MESSAGE_TEMPLATE.format(
-            trace_json=json.dumps(trace, ensure_ascii=False, indent=2)
+            trace_json=json.dumps(prompt_payload, ensure_ascii=False, indent=2)
         )
 
         model = self.eval_config.get("advanced", {}).get("llm_judge", {}).get("model", "gpt-4o")
@@ -657,6 +658,142 @@ class TraceEvaluator:
 
         return "en"
 
+    def _build_prompt_payload(self, trace: Dict[str, Any]) -> Dict[str, Any]:
+        """Create a compact payload for prompt construction."""
+
+        allowed_fields = [
+            "trace_id",
+            "iteration",
+            "persona",
+            "input",
+            "output",
+            "final_output",
+            "duration_ms",
+            "success",
+            "token_usage",
+            "observation_count",
+        ]
+
+        payload = {field: trace.get(field) for field in allowed_fields if field in trace}
+
+        conversation = trace.get("conversation")
+        if not conversation and isinstance(trace.get("conversation_state"), dict):
+            conversation = trace["conversation_state"].get("turns")
+        simplified_conversation = self._simplify_conversation(conversation)
+        if simplified_conversation:
+            payload["conversation"] = simplified_conversation
+
+        timeline = trace.get("timeline")
+        if isinstance(timeline, list):
+            simplified_timeline = [
+                self._simplify_timeline_entry(entry)
+                for entry in timeline
+                if isinstance(entry, dict)
+            ]
+            simplified_timeline = [entry for entry in simplified_timeline if entry]
+            if simplified_timeline:
+                payload["timeline"] = simplified_timeline
+
+        metadata = trace.get("metadata")
+        if isinstance(metadata, dict):
+            payload["metadata"] = self._sanitize_mapping(metadata)
+
+        return payload
+
+    def _simplify_conversation(self, conversation: Optional[List[Any]]) -> Optional[List[Dict[str, str]]]:
+        if not conversation or not isinstance(conversation, list):
+            return None
+
+        simplified: List[Dict[str, str]] = []
+        for entry in conversation:
+            if not isinstance(entry, dict):
+                continue
+            role = entry.get("role") or "assistant"
+            content = self._normalize_content(entry.get("content"))
+            if content:
+                simplified.append({"role": role, "content": content})
+        return simplified or None
+
+    def _normalize_content(self, content: Any) -> Optional[str]:
+        if content is None:
+            return None
+        if isinstance(content, str):
+            text = content.strip()
+            if text.startswith("{") and text.endswith("}"):
+                try:
+                    parsed = json.loads(text)
+                    return self._normalize_content(parsed)
+                except json.JSONDecodeError:
+                    return text
+            return text
+        if isinstance(content, dict):
+            transcript = content.get("transcript")
+            if isinstance(transcript, list):
+                lines: List[str] = []
+                for item in transcript:
+                    if not isinstance(item, dict):
+                        continue
+                    user_text = item.get("user")
+                    assistant_text = item.get("assistant")
+                    if user_text:
+                        lines.append(f"User: {user_text}")
+                    if assistant_text:
+                        lines.append(f"Assistant: {assistant_text}")
+                if lines:
+                    return "\n".join(lines)
+            text_value = content.get("text")
+            if isinstance(text_value, str):
+                return text_value
+            if "content" in content and isinstance(content["content"], str):
+                return content["content"]
+            return json.dumps(content, ensure_ascii=False)
+        if isinstance(content, list):
+            joined = "\n".join(
+                part for part in (self._normalize_content(item) for item in content) if part
+            )
+            return joined or None
+        return str(content)
+
+    def _simplify_timeline_entry(self, entry: Dict[str, Any]) -> Dict[str, Any]:
+        simplified = {
+            "type": entry.get("type"),
+            "name": entry.get("name"),
+            "start_time": entry.get("start_time"),
+            "end_time": entry.get("end_time"),
+            "duration_ms": entry.get("duration_ms"),
+            "level": entry.get("level"),
+        }
+        actions = entry.get("actions")
+        if isinstance(actions, (list, str)) and actions:
+            simplified["actions"] = actions
+        status = entry.get("status")
+        if status:
+            simplified["status"] = status
+        return {k: v for k, v in simplified.items() if v is not None}
+
+    def _sanitize_mapping(self, data: Dict[str, Any]) -> Dict[str, Any]:
+        """Remove heavy/raw fields recursively."""
+
+        cleaned: Dict[str, Any] = {}
+        for key, value in data.items():
+            if key == "raw" or value is None:
+                continue
+            cleaned[key] = self._strip_raw_fields(value)
+        return cleaned
+
+    def _strip_raw_fields(self, value: Any, depth: int = 0) -> Any:
+        if depth > 3:
+            return value
+        if isinstance(value, dict):
+            return {
+                k: self._strip_raw_fields(v, depth + 1)
+                for k, v in value.items()
+                if k != "raw" and v is not None
+            }
+        if isinstance(value, list):
+            return [self._strip_raw_fields(item, depth + 1) for item in value]
+        return value
+
 
 class OverallEvaluator:
     """Analyzes aggregated results (LLM-OV)."""

fluxloop_cli-0.2.34/fluxloop_cli/evaluation/report/pipeline.py

@@ -0,0 +1,156 @@
+"""
+Orchestration pipeline for generating evaluation reports.
+"""
+
+import asyncio
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from .aggregator import StatsAggregator
+from .generator import OverallEvaluator, ReportLLMClient, TraceEvaluator
+from .renderer import ReportRenderer
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ReportArtifacts:
+    """Artifacts produced by the evaluation report pipeline."""
+
+    html_path: Path
+
+
+class ReportPipeline:
+    """
+    Orchestrates the 5-stage evaluation pipeline:
+    1. Per-Trace Analysis (LLM-PT)
+    2. Rule-Based Aggregation
+    3. Overall Analysis (LLM-OV)
+    4. Data Preparation
+    5. Report Rendering
+    """
+
+    def __init__(self, config: Dict[str, Any], output_dir: Path, api_key: str = None):
+        self.config = config
+        self.output_dir = output_dir
+        self.client = ReportLLMClient(api_key=api_key)
+
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+
+        self.trace_evaluator = TraceEvaluator(self.client, config)
+        self.aggregator = StatsAggregator(config)
+        self.overall_evaluator = OverallEvaluator(self.client, config)
+        self.renderer = ReportRenderer(output_dir)
+
+    async def run(
+        self,
+        trace_records: List[Dict[str, Any]],
+        summary_records: Optional[List[Dict[str, Any]]] = None,
+    ) -> ReportArtifacts:
+        """
+        Run the full pipeline.
+        
+        Args:
+            trace_records: Detailed trace payloads (per-trace artifacts) used for LLM-PT
+            summary_records: Lightweight trace summaries (from `trace_summary.jsonl`). When omitted,
+                ``trace_records`` are re-used for aggregation.
+            
+        Returns:
+            Path to the generated HTML report
+        """
+        logger.info("🚀 Starting Evaluation Report Pipeline")
+        traces_for_rules = summary_records or trace_records
+        
+        # Stage 1: Per-Trace Analysis (Parallel)
+        logger.info(f"Stage 1: Running LLM-PT on {len(trace_records)} traces...")
+        pt_results = await self._run_pt_evaluations(trace_records)
+        
+        # Stage 2: Aggregation
+        logger.info("Stage 2: Aggregating statistics...")
+        rule_based_data = self.aggregator.aggregate(pt_results, traces_for_rules)
+        
+        # Stage 3: Overall Analysis
+        logger.info("Stage 3: Running LLM-OV analysis...")
+        llm_ov_data = await self.overall_evaluator.analyze(rule_based_data)
+        if "error" in llm_ov_data:
+            logger.warning(f"LLM-OV finished with error: {llm_ov_data['error']}")
+            # Continue rendering even if OV fails (partial report is better than none)
+        
+        # Stage 4 & 5: Render
+        logger.info("Stage 4 & 5: Rendering HTML report...")
+        report_path = self.renderer.render(
+            rule_based_data, llm_ov_data, traces_for_rules, self.config
+        )
+
+        logger.info("✅ Pipeline Complete! Report: %s", report_path)
+        return ReportArtifacts(html_path=report_path)
+
+    async def _run_pt_evaluations(self, traces: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """Run TraceEvaluator concurrently."""
+        limit = self._resolve_concurrency_limit()
+        semaphore = asyncio.Semaphore(limit)
+
+        async def _evaluate_with_retry(trace: Dict[str, Any], index: int, total: int) -> Dict[str, Any]:
+            attempt = 0
+            max_attempts = self._resolve_retry_limit()
+            while True:
+                attempt += 1
+                async with semaphore:
+                    logger.debug(
+                        "PT[%s/%s] Evaluating trace %s (attempt %s)",
+                        index,
+                        total,
+                        trace.get("trace_id", "unknown")[:8],
+                        attempt,
+                    )
+                    result = await self.trace_evaluator.evaluate_trace(trace)
+                error = result.get("error")
+                if not error:
+                    return result
+                if attempt >= max_attempts:
+                    logger.error(
+                        "PT[%s/%s] Trace %s failed after %s attempts: %s",
+                        index,
+                        total,
+                        trace.get("trace_id", "unknown"),
+                        attempt,
+                        error,
+                    )
+                    return result
+                sleep_seconds = min(5 * attempt, 30)
+                logger.warning(
+                    "PT[%s/%s] Trace %s attempt %s failed: %s — retrying in %.1fs",
+                    index,
+                    total,
+                    trace.get("trace_id", "unknown")[:8],
+                    attempt,
+                    error,
+                    sleep_seconds,
+                )
+                await asyncio.sleep(sleep_seconds)
+
+        tasks = [
+            _evaluate_with_retry(trace, idx + 1, len(traces))
+            for idx, trace in enumerate(traces)
+        ]
+        return list(await asyncio.gather(*tasks))
+
+    def _resolve_concurrency_limit(self) -> int:
+        """Determine max concurrent LLM calls."""
+        eval_cfg = self.config.get("evaluation", {})
+        advanced = eval_cfg.get("advanced", {})
+        limit = advanced.get("llm_concurrency")
+        if isinstance(limit, int) and limit > 0:
+            return max(1, min(limit, 16))
+        return 4
+
+    def _resolve_retry_limit(self) -> int:
+        eval_cfg = self.config.get("evaluation", {})
+        advanced = eval_cfg.get("advanced", {})
+        retries = advanced.get("llm_retries")
+        if isinstance(retries, int) and retries > 0:
+            return min(max(1, retries), 5)
+        return 3
+