fluxloop-cli 0.2.20__tar.gz → 0.2.21__tar.gz

{fluxloop_cli-0.2.20 → fluxloop_cli-0.2.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fluxloop-cli
-Version: 0.2.20
+Version: 0.2.21
 Summary: FluxLoop CLI for running agent simulations
 Author-email: FluxLoop Team <team@fluxloop.dev>
 License: Apache-2.0

{fluxloop_cli-0.2.20 → fluxloop_cli-0.2.21}/fluxloop_cli/__init__.py

@@ -2,7 +2,7 @@
 FluxLoop CLI - Command-line interface for running agent simulations.
 """
 
-__version__ = "0.2.20"
+__version__ = "0.2.21"
 
 from .main import app
 

{fluxloop_cli-0.2.20 → fluxloop_cli-0.2.21}/fluxloop_cli/commands/evaluate.py

@@ -2,12 +2,14 @@
 
 from __future__ import annotations
 
+import os
 from pathlib import Path
 from typing import Optional
 
 import typer
 from rich.console import Console
 
+from ..environment import load_env_chain
 from ..evaluation import EvaluationOptions, load_evaluation_config, run_evaluation
 
 console = Console()
@@ -134,6 +136,20 @@ def experiment(
     except Exception as exc:  # noqa: BLE001
         raise typer.BadParameter(f"Failed to load evaluation config: {exc}") from exc
 
+    def _log_env_error(path: Path, exc: Exception) -> None:
+        console.log(
+            f"[yellow]Warning:[/yellow] Failed to load environment from {path}: {exc}"
+        )
+
+    load_env_chain(
+        evaluation_config.get_source_dir(),
+        refresh_config=True,
+        on_error=_log_env_error,
+    )
+
+    if llm_api_key is None:
+        llm_api_key = os.getenv("FLUXLOOP_LLM_API_KEY") or os.getenv("OPENAI_API_KEY")
+
     output_dir = output
     if not output_dir.is_absolute():
         output_dir = resolved_experiment_dir / output_dir

{fluxloop_cli-0.2.20 → fluxloop_cli-0.2.21}/fluxloop_cli/commands/init.py

@@ -179,12 +179,16 @@ def project(
     # Show next steps
     console.print("\n[bold]Next steps:[/bold]")
     console.print("1. Review configs in [cyan]configs/[/cyan] (project/input/simulation/evaluation)")
-    console.print("2. Set up environment variables in [cyan].env[/cyan]")
+    console.print("2. Configure secrets via [cyan].env[/cyan] or [green]fluxloop config set-llm[/green]")
     if with_example:
-        console.print("3. Try running: [green]fluxloop run experiment[/green]")
+        console.print("3. Customize the sample agent in [cyan]examples/simple_agent.py[/cyan]")
     else:
-        console.print("3. Add your agent code")
-        console.print("4. Run: [green]fluxloop run experiment[/green]")
+        console.print("3. Create your agent: [green]fluxloop init agent <name>[/green]")
+    console.print("4. Generate inputs: [green]fluxloop generate inputs[/green]")
+    console.print("5. Run the experiment: [green]fluxloop run experiment[/green]")
+    console.print("6. Parse outputs: [green]fluxloop parse experiment[/green]")
+    console.print("7. Evaluate results (optional): [green]fluxloop evaluate experiment[/green]")
+    console.print("8. Diagnose environment anytime: [green]fluxloop doctor[/green]")
 
 
 @app.command()

fluxloop_cli-0.2.21/fluxloop_cli/environment.py

@@ -0,0 +1,71 @@
+"""
+Utilities for applying project environment settings across CLI commands.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Callable, Iterable, Optional, Set
+
+import fluxloop
+
+EnvErrorHandler = Callable[[Path, Exception], None]
+
+
+def load_env_chain(
+    source_dir: Optional[Path],
+    *,
+    additional_paths: Optional[Iterable[Path]] = None,
+    refresh_config: bool = True,
+    on_error: Optional[EnvErrorHandler] = None,
+) -> None:
+    """
+    Load environment variables from `.env` files related to a project.
+
+    Parameters
+    ----------
+    source_dir:
+        Directory associated with the current config (usually the config file's parent).
+    additional_paths:
+        Extra `.env` files to consider (applied after the standard chain).
+    refresh_config:
+        Whether to refresh the fluxloop config after loading.
+    on_error:
+        Optional callback invoked when loading a path fails.
+    """
+
+    if source_dir is None:
+        candidates: Iterable[Path] = additional_paths or []
+    else:
+        resolved_source = source_dir.resolve()
+        parents: list[Path] = []
+        parent = resolved_source.parent
+        if parent != resolved_source:
+            parents.append(parent / ".env")
+        parents.append(resolved_source / ".env")
+
+        extra = list(additional_paths or [])
+        candidates = [*parents, *extra]
+
+    seen: Set[Path] = set()
+    for path in candidates:
+        # Avoid duplicate loads and normalize the path
+        resolved = path.resolve()
+        if resolved in seen:
+            continue
+        seen.add(resolved)
+
+        if not resolved.exists():
+            continue
+
+        try:
+            fluxloop.load_env(resolved, override=True, refresh_config=refresh_config)
+        except Exception as exc:  # noqa: BLE001
+            if on_error is None:
+                raise
+            on_error(resolved, exc)
+
+
+__all__ = ["load_env_chain", "EnvErrorHandler"]
+
+

{fluxloop_cli-0.2.20 → fluxloop_cli-0.2.21}/fluxloop_cli/evaluation/config.py

@@ -230,6 +230,14 @@ class EvaluationConfig:
     report: ReportConfig = field(default_factory=ReportConfig)
     advanced: AdvancedConfig = field(default_factory=AdvancedConfig)
 
+    def set_source_dir(self, source_dir: Path) -> None:
+        """Remember the directory where this config file was loaded from."""
+        self._source_dir = source_dir
+
+    def get_source_dir(self) -> Optional[Path]:
+        """Return the directory where the config file was loaded from."""
+        return getattr(self, "_source_dir", None)
+
 
 def _load_yaml(path: Path) -> Dict[str, Any]:
     if not path.exists():
@@ -704,7 +712,9 @@ def load_evaluation_config(path: Path) -> EvaluationConfig:
     report = _parse_report(data.get("report"))
     advanced = _parse_advanced(data.get("advanced"))
 
-    return EvaluationConfig(
+    config_dir = path.parent.resolve()
+
+    config = EvaluationConfig(
         evaluators=evaluators,
         aggregate=aggregate,
         limits=limits,
@@ -715,4 +725,8 @@ def load_evaluation_config(path: Path) -> EvaluationConfig:
         advanced=advanced,
     )
 
+    config.set_source_dir(config_dir)
+
+    return config
+
 

{fluxloop_cli-0.2.20 → fluxloop_cli-0.2.21}/fluxloop_cli/evaluation/engine/reporting/html.py

@@ -48,8 +48,8 @@ DEFAULT_TEMPLATE = r"""<!DOCTYPE html>
   <main class="max-w-6xl mx-auto p-8 space-y-6">
     <header class="space-y-6">
       <div>
-        <h1 class="text-3xl font-bold">[[TITLE]]</h1>
-        <p class="text-sm text-slate-300">Generated at [[DATE]]</p>
+      <h1 class="text-3xl font-bold">[[TITLE]]</h1>
+      <p class="text-sm text-slate-300">Generated at [[DATE]]</p>
       </div>
       <nav class="flex flex-wrap gap-2" role="tablist" aria-label="Report sections">
         <button class="tab-button px-4 py-2 rounded-full bg-sky-500/20 border border-sky-400 text-sky-200 font-semibold" data-tab-button="summary">Executive Summary</button>
@@ -203,14 +203,14 @@ DEFAULT_TEMPLATE = r"""<!DOCTYPE html>
         });
       }
       container.innerHTML = cards
-        .map(
+            .map(
           (card) => `
             <div class="bg-slate-900 rounded-xl p-4 border border-slate-800 shadow-inner">
               <p class="text-slate-400 text-xs uppercase tracking-wide">${card.label}</p>
               <p class="text-2xl font-semibold mt-2">${card.value}</p>
-            </div>
-          `
-        )
+                </div>
+              `
+            )
         .join("");
     }
 
@@ -236,21 +236,21 @@ DEFAULT_TEMPLATE = r"""<!DOCTYPE html>
           if (!payload) return;
           const checks = Object.entries(payload);
           if (!checks.length) return;
-          const sectionTitle = section.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase());
+        const sectionTitle = section.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase());
           const wrapper = document.createElement("div");
           wrapper.className = "bg-slate-900 border border-slate-800 rounded-xl p-4 space-y-2";
           wrapper.innerHTML = `<h3 class="text-lg font-semibold">${sectionTitle}</h3>`;
-          const list = document.createElement("ul");
-          list.className = "space-y-1 text-sm text-slate-300";
-          for (const [name, details] of checks) {
+        const list = document.createElement("ul");
+        list.className = "space-y-1 text-sm text-slate-300";
+        for (const [name, details] of checks) {
             const prettyName = toTitleCase(name);
-            const status = details.met === true ? "✅ Met" : details.met === false ? "❌ Not met" : "⚪️ Not evaluated";
+          const status = details.met === true ? "✅ Met" : details.met === false ? "❌ Not met" : "⚪️ Not evaluated";
             const meta = { ...details };
             delete meta.met;
             const extra =
               Object.keys(meta).length > 0 ? `<span class="text-slate-400"> ${JSON.stringify(meta)}</span>` : "";
             list.innerHTML += `<li>${status} · ${prettyName}${extra}</li>`;
-          }
+        }
           wrapper.appendChild(list);
           container.appendChild(wrapper);
         });
@@ -453,7 +453,7 @@ DEFAULT_TEMPLATE = r"""<!DOCTYPE html>
         const shown = filtered.length;
         const suffix = total === 1 ? "" : "s";
         hint.textContent = `${shown} of ${total} trace${suffix} shown`;
-      }
+    }
     }
 
     initTabs();

{fluxloop_cli-0.2.20 → fluxloop_cli-0.2.21}/fluxloop_cli/runner.py

@@ -21,6 +21,7 @@ from fluxloop.buffer import EventBuffer
 from fluxloop.schemas import ExperimentConfig, PersonaConfig
 from rich.console import Console
 
+from .environment import load_env_chain
 from .target_loader import TargetLoader
 from .arg_binder import ArgBinder
 
@@ -97,21 +98,17 @@ class ExperimentRunner:
         """Load environment variables from .env and runner settings."""
 
         source_dir = self.config.get_source_dir()
-        env_candidates: List[Path] = []
 
-        # Load parent (root) .env first, then project .env so project overrides take precedence
-        if source_dir:
-            parent = source_dir.parent
-            if parent != source_dir:
-                env_candidates.append(parent / ".env")
-            env_candidates.append(source_dir / ".env")
+        def _log_env_error(path: Path, exc: Exception) -> None:
+            console.log(
+                f"[yellow]Warning:[/yellow] Failed to load environment from {path}: {exc}"
+            )
 
-        for candidate in env_candidates:
-            if candidate.exists():
-                try:
-                    fluxloop.load_env(candidate, override=True, refresh_config=True)
-                except Exception:
-                    console.log(f"[yellow]Warning:[/yellow] Failed to load environment from {candidate}")
+        load_env_chain(
+            source_dir,
+            refresh_config=True,
+            on_error=_log_env_error,
+        )
 
         env_vars = getattr(self.config.runner, "environment_vars", {}) or {}
         for key, value in env_vars.items():

{fluxloop_cli-0.2.20 → fluxloop_cli-0.2.21}/fluxloop_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fluxloop-cli
-Version: 0.2.20
+Version: 0.2.21
 Summary: FluxLoop CLI for running agent simulations
 Author-email: FluxLoop Team <team@fluxloop.dev>
 License: Apache-2.0

{fluxloop_cli-0.2.20 → fluxloop_cli-0.2.21}/fluxloop_cli.egg-info/SOURCES.txt

@@ -5,6 +5,7 @@ fluxloop_cli/arg_binder.py
 fluxloop_cli/config_loader.py
 fluxloop_cli/config_schema.py
 fluxloop_cli/constants.py
+fluxloop_cli/environment.py
 fluxloop_cli/input_generator.py
 fluxloop_cli/llm_generator.py
 fluxloop_cli/main.py

{fluxloop_cli-0.2.20 → fluxloop_cli-0.2.21}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "fluxloop-cli"
-version = "0.2.20"
+version = "0.2.21"
 description = "FluxLoop CLI for running agent simulations"
 authors = [
     {name = "FluxLoop Team", email = "team@fluxloop.dev"}

{fluxloop_cli-0.2.20 → fluxloop_cli-0.2.21}/tests/test_evaluate_command.py

@@ -333,3 +333,51 @@ def test_evaluate_phase2_extended_outputs(tmp_path: Path) -> None:
     assert "outliers" in performance_analysis and "trends" in performance_analysis
     comparison = analysis["comparison"]
     assert "baseline_path" in comparison and comparison["baseline_path"].endswith("baseline_summary.json")
+
+
+def test_evaluate_loads_env_for_llm(tmp_path: Path, monkeypatch) -> None:
+    project_dir = tmp_path
+    configs_dir = project_dir / "configs"
+    configs_dir.mkdir()
+
+    env_path = project_dir / ".env"
+    env_path.write_text("FLUXLOOP_LLM_API_KEY=sk-test-key\n", encoding="utf-8")
+
+    eval_config = configs_dir / "evaluation.yaml"
+    _write_eval_config(eval_config, include_llm=True)
+
+    experiment_dir = project_dir / "experiments" / "run_1"
+    experiment_dir.mkdir(parents=True, exist_ok=True)
+    _write_trace_summary(experiment_dir / "trace_summary.jsonl")
+
+    captured: dict[str, object] = {}
+
+    def fake_run_evaluation(exp_dir, config, options):
+        captured["llm_api_key"] = options.llm_api_key
+        captured["sample_rate"] = options.sample_rate
+        return {}
+
+    monkeypatch.delenv("FLUXLOOP_LLM_API_KEY", raising=False)
+    monkeypatch.delenv("OPENAI_API_KEY", raising=False)
+    monkeypatch.setattr(
+        "fluxloop_cli.commands.evaluate.run_evaluation", fake_run_evaluation
+    )
+
+    result = runner.invoke(
+        cli_main.app,
+        [
+            "evaluate",
+            "experiment",
+            str(experiment_dir),
+            "--config",
+            str(eval_config),
+            "--output",
+            "evaluation",
+        ],
+    )
+
+    monkeypatch.delenv("FLUXLOOP_LLM_API_KEY", raising=False)
+    monkeypatch.delenv("OPENAI_API_KEY", raising=False)
+
+    assert result.exit_code == 0, result.output
+    assert captured["llm_api_key"] == "sk-test-key"