PyPI - fluxloop-cli - Versions diffs - 0.2.26__tar.gz → 0.2.27__tar.gz - Mend

fluxloop-cli 0.2.26tar.gz → 0.2.27tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (65) hide show

{fluxloop_cli-0.2.26 → fluxloop_cli-0.2.27}/PKG-INFO RENAMED Viewed

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fluxloop-cli
-Version: 0.2.26
+Version: 0.2.27
 Summary: FluxLoop CLI for running agent simulations
 Author-email: FluxLoop Team <team@fluxloop.dev>
 License: Apache-2.0

{fluxloop_cli-0.2.26 → fluxloop_cli-0.2.27}/fluxloop_cli/__init__.py RENAMED Viewed

@@ -2,7 +2,7 @@
 FluxLoop CLI - Command-line interface for running agent simulations.
 """
-__version__ = "0.2.26"
+__version__ = "0.2.27"
 from .main import app

{fluxloop_cli-0.2.26 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/engine/reporting/html.py RENAMED Viewed

@@ -40,6 +40,7 @@ DEFAULT_TEMPLATE = r"""<!DOCTYPE html>
 <head>
   <meta charset="utf-8" />
   <meta name="viewport" content="width=device-width, initial-scale=1" />
+  <meta name="report:generated_at" content="[[DATE]]" />
   <title>[[TITLE]]</title>
   <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/tailwindcss@2.2.19/dist/tailwind.min.css">
   <script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.1/dist/chart.umd.min.js"></script>
@@ -47,9 +48,19 @@ DEFAULT_TEMPLATE = r"""<!DOCTYPE html>
 <body class="bg-slate-950 text-slate-100">
   <main class="max-w-6xl mx-auto p-8 space-y-6">
     <header class="space-y-6">
-      <div>
-      <h1 class="text-3xl font-bold">[[TITLE]]</h1>
-      <p class="text-sm text-slate-300">Generated at [[DATE]]</p>
+      <div class="flex flex-col gap-4 md:flex-row md:items-center md:justify-between">
+        <div>
+          <h1 class="text-3xl font-bold">[[TITLE]]</h1>
+          <p class="text-sm text-slate-300">[[SUBTITLE]]</p>
+        </div>
+        <a
+          href="https://fluxloop.io/"
+          class="inline-flex items-center gap-2 self-start rounded-full border border-sky-400 px-4 py-2 text-sm font-semibold text-sky-200 transition hover:bg-sky-500/10 focus:outline-none focus:ring-2 focus:ring-sky-400 focus:ring-offset-2 focus:ring-offset-slate-950"
+          target="_blank"
+          rel="noopener noreferrer"
+        >
+          <span>Built with FluxLoop</span>
+        </a>
       </div>
       <nav class="flex flex-wrap gap-2" role="tablist" aria-label="Report sections">
         <button class="tab-button px-4 py-2 rounded-full bg-sky-500/20 border border-sky-400 text-sky-200 font-semibold" data-tab-button="summary">Executive Summary</button>
@@ -62,6 +73,11 @@ DEFAULT_TEMPLATE = r"""<!DOCTYPE html>
     <section data-tab-panel="summary" class="tab-panel space-y-6">
       <div id="summaryCards" class="grid gap-4 md:grid-cols-2 xl:grid-cols-4"></div>
+      <section class="space-y-4">
+        <h2 class="text-xl font-semibold">Evaluation Goal</h2>
+        <div id="evaluationGoal" class="whitespace-pre-wrap rounded-xl border border-slate-800 bg-slate-900 p-4 text-sm text-slate-300"></div>
+      </section>
       <section class="space-y-4">
         <h2 class="text-xl font-semibold">Success Criteria</h2>
         <div id="criteriaList" class="space-y-3"></div>
@@ -214,6 +230,21 @@ DEFAULT_TEMPLATE = r"""<!DOCTYPE html>
         .join("");
     }
+    function renderEvaluationGoal() {
+      const container = document.getElementById("evaluationGoal");
+      if (!container) return;
+      const goal = summary?.evaluation_goal;
+      if (!goal) {
+        container.textContent = "No evaluation goal provided.";
+        container.classList.remove("text-slate-300");
+        container.classList.add("text-slate-400");
+        return;
+      }
+      container.classList.remove("text-slate-400");
+      container.classList.add("text-slate-300");
+      container.textContent = goal;
+    }
     function renderCriteria() {
       const container = document.getElementById("criteriaList");
       if (!container || !criteria || !Object.keys(criteria).length) {
@@ -458,6 +489,7 @@ DEFAULT_TEMPLATE = r"""<!DOCTYPE html>
     initTabs();
     renderSummaryCards();
+    renderEvaluationGoal();
     renderCriteria();
     renderRecommendations();
     renderScoreChart();
@@ -471,6 +503,45 @@ DEFAULT_TEMPLATE = r"""<!DOCTYPE html>
 """
+def _resolve_experiment_label(summary: Dict[str, Any], output_path: Path) -> Optional[str]:
+    def _stringify(value: Any) -> Optional[str]:
+        if value is None:
+            return None
+        if isinstance(value, (list, tuple, set)):
+            parts = [str(item) for item in value if item not in (None, "")]
+            if not parts:
+                return None
+            return ", ".join(parts)
+        if isinstance(value, str):
+            return value.strip() or None
+        return str(value)
+    for key in ("experiment_id", "experiment_name"):
+        candidate = _stringify(summary.get(key))
+        if candidate:
+            return candidate
+    experiment_meta = summary.get("experiment")
+    if isinstance(experiment_meta, dict):
+        for key in ("name", "id", "label"):
+            candidate = _stringify(experiment_meta.get(key))
+            if candidate:
+                return candidate
+    output_dir = output_path.parent
+    if output_dir != output_path:
+        experiment_dir = output_dir.parent
+        if experiment_dir and experiment_dir != output_dir:
+            candidate = _stringify(experiment_dir.name)
+            if candidate:
+                return candidate
+        candidate = _stringify(output_dir.name)
+        if candidate:
+            return candidate
+    return None
 def select_html_template(options: "EvaluationOptions", config: "EvaluationConfig") -> Tuple[str, Optional[str]]:
     if options.report_template:
         template_text = load_template_from_path(options.report_template)
@@ -497,9 +568,19 @@ def write_html_report(
     success_criteria = summary.get("success_criteria_results") or {}
     analysis = summary.get("analysis") or {}
+    generated_at = datetime.now(UTC)
+    generated_iso = generated_at.isoformat(timespec="seconds").replace("+00:00", "Z")
+    human_readable = generated_at.strftime("%Y-%m-%d %H:%M:%SZ")
+    experiment_label = _resolve_experiment_label(summary, output_path)
+    subtitle_parts = [f"Generated on {human_readable}"]
+    if experiment_label:
+        subtitle_parts.append(f"Experiment: {experiment_label}")
+    subtitle = " · ".join(subtitle_parts)
     replacements = {
-        "[[TITLE]]": summary.get("evaluation_goal") or "FluxLoop Evaluation Report",
-        "[[DATE]]": datetime.now(UTC).isoformat(timespec="seconds").replace("+00:00", "Z"),
+        "[[TITLE]]": "FluxLoop Evaluation Report",
+        "[[DATE]]": generated_iso,
+        "[[SUBTITLE]]": subtitle,
         "[[SUMMARY_JSON]]": json.dumps(summary, ensure_ascii=False),
         "[[PER_TRACE_JSON]]": json.dumps(per_trace_payload, ensure_ascii=False),
         "[[CRITERIA_JSON]]": json.dumps(success_criteria, ensure_ascii=False),

{fluxloop_cli-0.2.26 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/rules.py RENAMED Viewed

@@ -189,6 +189,172 @@ def _evaluate_latency_under(context: RuleContext, definition: RuleDefinition) ->
     return RuleResult(rule=definition, score=score, reason=reason)
+def _coerce_usage_values(data: Dict[str, Any]) -> Dict[str, float]:
+    usage: Dict[str, float] = {}
+    for key in ("prompt_tokens", "completion_tokens", "total_tokens"):
+        value = data.get(key)
+        if isinstance(value, (int, float)):
+            usage[key] = float(value)
+    return usage
+def _usage_from_candidate(candidate: Any) -> Dict[str, float]:
+    if not isinstance(candidate, dict):
+        return {}
+    direct = _coerce_usage_values(candidate)
+    if direct:
+        return direct
+    token_usage = candidate.get("token_usage")
+    if isinstance(token_usage, dict):
+        values = _coerce_usage_values(token_usage)
+        if values:
+            return values
+    nested = candidate.get("usage")
+    if isinstance(nested, dict):
+        return _coerce_usage_values(nested)
+    return {}
+def _entry_token_usage(entry: Dict[str, Any]) -> Dict[str, float]:
+    candidates = [entry]
+    output = entry.get("output")
+    if isinstance(output, dict):
+        candidates.append(output)
+        messages = output.get("messages")
+        if isinstance(messages, dict):
+            candidates.append(messages)
+            response_metadata = messages.get("response_metadata")
+            if isinstance(response_metadata, dict):
+                candidates.append(response_metadata)
+    raw = entry.get("raw")
+    if isinstance(raw, dict):
+        candidates.append(raw)
+        raw_output = raw.get("output")
+        if isinstance(raw_output, dict):
+            candidates.append(raw_output)
+            response_metadata = raw_output.get("response_metadata")
+            if isinstance(response_metadata, dict):
+                candidates.append(response_metadata)
+            messages = raw_output.get("messages")
+            if isinstance(messages, dict):
+                candidates.append(messages)
+                response_metadata = messages.get("response_metadata")
+                if isinstance(response_metadata, dict):
+                    candidates.append(response_metadata)
+        metadata = raw.get("metadata")
+        if isinstance(metadata, dict):
+            candidates.append(metadata)
+        response_metadata = raw.get("response_metadata")
+        if isinstance(response_metadata, dict):
+            candidates.append(response_metadata)
+    for candidate in candidates:
+        usage = _usage_from_candidate(candidate)
+        if usage:
+            return usage
+    return {}
+def _collect_token_usage(trace: Dict[str, Any]) -> Optional[Dict[str, float]]:
+    totals = {"prompt_tokens": 0.0, "completion_tokens": 0.0, "total_tokens": 0.0}
+    found = False
+    summary = trace.get("summary")
+    if isinstance(summary, dict):
+        summary_usage = {}
+        if isinstance(summary.get("token_usage"), dict):
+            summary_usage = _usage_from_candidate(summary["token_usage"])
+        if not summary_usage:
+            raw = summary.get("raw")
+            if isinstance(raw, dict) and isinstance(raw.get("token_usage"), dict):
+                summary_usage = _usage_from_candidate(raw["token_usage"])
+        if summary_usage:
+            found = True
+            for key, value in summary_usage.items():
+                totals[key] += value
+    timeline = trace.get("timeline")
+    if isinstance(timeline, list):
+        for entry in timeline:
+            if not isinstance(entry, dict):
+                continue
+            usage = _entry_token_usage(entry)
+            if not usage:
+                continue
+            found = True
+            for key, value in usage.items():
+                totals[key] += value
+    if not found:
+        return None
+    prompt = totals["prompt_tokens"]
+    completion = totals["completion_tokens"]
+    total = totals["total_tokens"]
+    if total <= 0:
+        total = prompt + completion
+    return {
+        "prompt": prompt,
+        "completion": completion,
+        "total": total,
+    }
+def _coerce_budget_value(value: Any, label: str) -> Optional[float]:
+    if value is None:
+        return None
+    try:
+        budget = float(value)
+    except (TypeError, ValueError) as exc:  # noqa: BLE001
+        raise ValueError(f"token_usage_under {label} must be numeric") from exc
+    if budget <= 0:
+        raise ValueError(f"token_usage_under {label} must be greater than 0")
+    return budget
+def _evaluate_token_usage_under(context: RuleContext, definition: RuleDefinition) -> RuleResult:
+    usage = _collect_token_usage(context.trace)
+    if usage is None:
+        return RuleResult(
+            rule=definition,
+            score=0.0,
+            reason="token usage data not available",
+        )
+    max_total = _coerce_budget_value(definition.params.get("max_total_tokens"), "max_total_tokens")
+    max_prompt = _coerce_budget_value(
+        definition.params.get("max_prompt_tokens"),
+        "max_prompt_tokens",
+    )
+    max_completion = _coerce_budget_value(
+        definition.params.get("max_completion_tokens"),
+        "max_completion_tokens",
+    )
+    if max_total is None and max_prompt is None and max_completion is None:
+        raise ValueError("token_usage_under requires at least one max_*_tokens parameter")
+    score = 1.0
+    reasons: List[str] = []
+    def apply_budget(actual: float, budget: Optional[float], label: str) -> None:
+        nonlocal score
+        if budget is None:
+            return
+        if actual <= budget:
+            return
+        ratio = budget / actual if actual > 0 else 0.0
+        ratio = max(0.0, min(1.0, ratio))
+        score = min(score, ratio)
+        reasons.append(f"{label} {actual:.0f} exceeds budget {budget:.0f}")
+    apply_budget(usage["total"], max_total, "total tokens")
+    apply_budget(usage["prompt"], max_prompt, "prompt tokens")
+    apply_budget(usage["completion"], max_completion, "completion tokens")
+    reason = "; ".join(reasons) if reasons else None
+    return RuleResult(rule=definition, score=score, reason=reason)
 def _evaluate_similarity(context: RuleContext, definition: RuleDefinition) -> RuleResult:
     target = definition.params.get("target", "output")
     expected_path = definition.params.get("expected_path")
@@ -233,6 +399,7 @@ RULE_DISPATCH = {
     "not_contains": _evaluate_not_contains,
     "matches_regex": _evaluate_matches_regex,
     "latency_under": _evaluate_latency_under,
+    "token_usage_under": _evaluate_token_usage_under,
     "similarity": _evaluate_similarity,
     "success": _evaluate_success_flag,
 }

{fluxloop_cli-0.2.26 → fluxloop_cli-0.2.27}/fluxloop_cli/templates.py RENAMED Viewed

@@ -216,13 +216,13 @@ def create_evaluation_config() -> str:
             rules:
               - check: output_not_empty
-          - name: latency_budget
+          - name: token_budget
             type: rule_based
             enabled: true
             weight: 0.2
             rules:
-              - check: latency_under
-                budget_ms: 1500
+              - check: token_usage_under
+                max_total_tokens: 4000
           - name: keyword_quality
             type: rule_based
@@ -340,7 +340,7 @@ def create_evaluation_config() -> str:
           performance:
             all_traces_successful: true
             avg_response_time:
-              enabled: true
+              enabled: false
               threshold_ms: 2000
             max_response_time:
               enabled: false

{fluxloop_cli-0.2.26 → fluxloop_cli-0.2.27}/fluxloop_cli.egg-info/PKG-INFO RENAMED Viewed

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fluxloop-cli
-Version: 0.2.26
+Version: 0.2.27
 Summary: FluxLoop CLI for running agent simulations
 Author-email: FluxLoop Team <team@fluxloop.dev>
 License: Apache-2.0

{fluxloop_cli-0.2.26 → fluxloop_cli-0.2.27}/pyproject.toml RENAMED Viewed

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "fluxloop-cli"
-version = "0.2.26"
+version = "0.2.27"
 description = "FluxLoop CLI for running agent simulations"
 authors = [
     {name = "FluxLoop Team", email = "team@fluxloop.dev"}

{fluxloop_cli-0.2.26 → fluxloop_cli-0.2.27}/tests/test_evaluate_command.py RENAMED Viewed

@@ -19,6 +19,11 @@ def _write_trace_summary(path: Path) -> None:
             "output": "Sure, I can help you.",
             "duration_ms": 500,
             "success": True,
+            "token_usage": {
+                "prompt_tokens": 600,
+                "completion_tokens": 200,
+                "total_tokens": 800,
+            },
         },
         {
             "trace_id": "trace-2",
@@ -28,6 +33,11 @@ def _write_trace_summary(path: Path) -> None:
             "output": "I cannot assist right now.",
             "duration_ms": 1500,
             "success": False,
+            "token_usage": {
+                "prompt_tokens": 900,
+                "completion_tokens": 500,
+                "total_tokens": 1400,
+            },
         },
     ]
     lines = [json.dumps(item) for item in traces]
@@ -43,8 +53,8 @@ def _write_eval_config(path: Path, include_llm: bool = False) -> None:
         weight: 1.0
         rules:
           - check: output_not_empty
-          - check: latency_under
-            budget_ms: 1200
+          - check: token_usage_under
+            max_total_tokens: 1200
           - check: success
     aggregate:
@@ -61,6 +71,8 @@ def _write_eval_config(path: Path, include_llm: bool = False) -> None:
             weight: 1.0
             rules:
               - check: output_not_empty
+              - check: token_usage_under
+                max_total_tokens: 1200
           - name: llm_quality
             type: llm_judge
             enabled: true
@@ -91,6 +103,11 @@ def _write_phase2_trace_summary(path: Path) -> None:
             "output": "Alerts configured successfully.",
             "duration_ms": 900,
             "success": True,
+            "token_usage": {
+                "prompt_tokens": 500,
+                "completion_tokens": 200,
+                "total_tokens": 700,
+            },
         },
         {
             "trace_id": "p2-2",
@@ -100,6 +117,11 @@ def _write_phase2_trace_summary(path: Path) -> None:
             "output": "Retrying the tool call now.",
             "duration_ms": 1800,
             "success": True,
+            "token_usage": {
+                "prompt_tokens": 650,
+                "completion_tokens": 250,
+                "total_tokens": 900,
+            },
         },
         {
             "trace_id": "p2-3",
@@ -109,6 +131,11 @@ def _write_phase2_trace_summary(path: Path) -> None:
             "output": "",
             "duration_ms": 650,
             "success": False,
+            "token_usage": {
+                "prompt_tokens": 450,
+                "completion_tokens": 300,
+                "total_tokens": 750,
+            },
         },
         {
             "trace_id": "p2-4",
@@ -118,6 +145,11 @@ def _write_phase2_trace_summary(path: Path) -> None:
             "output": "Incident report created.",
             "duration_ms": 2100,
             "success": True,
+            "token_usage": {
+                "prompt_tokens": 800,
+                "completion_tokens": 350,
+                "total_tokens": 1150,
+            },
         },
     ]
     lines = [json.dumps(item) for item in traces]
@@ -131,13 +163,13 @@ def _write_phase2_eval_config(path: Path) -> None:
         Validate extended Phase 2 evaluation outputs for persona-aware runs.
     evaluators:
-      - name: latency_checker
+      - name: token_checker
         type: rule_based
         enabled: true
         weight: 1.0
         rules:
-          - check: latency_under
-            budget_ms: 1500
+          - check: token_usage_under
+            max_total_tokens: 1500
       - name: intent_recognition
         type: rule_based
         enabled: true
@@ -201,7 +233,7 @@ def _write_baseline_summary(path: Path) -> None:
         "total_traces": 4,
         "passed_traces": 3,
         "evaluator_stats": {
-            "latency_checker": {"average": 0.75, "min": 0.0, "max": 1.0, "count": 4},
+            "token_checker": {"average": 0.75, "min": 0.0, "max": 1.0, "count": 4},
             "intent_recognition": {"average": 1.0, "min": 1.0, "max": 1.0, "count": 4},
         },
     }