fluxloop-cli 0.2.20__tar.gz → 0.2.22__tar.gz

{fluxloop_cli-0.2.20 → fluxloop_cli-0.2.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fluxloop-cli
-Version: 0.2.20
+Version: 0.2.22
 Summary: FluxLoop CLI for running agent simulations
 Author-email: FluxLoop Team <team@fluxloop.dev>
 License: Apache-2.0
@@ -71,6 +71,18 @@ The legacy `setting.yaml` is still supported, but new projects created with
 - `fluxloop record enable|disable|status` – toggle recording mode across `.env` and simulation config
 - `fluxloop doctor` – summarize Python, FluxLoop CLI/MCP, and MCP index state for the active environment
 
+### Multi-turn supervisor options
+
+`fluxloop run experiment` supports multi-turn orchestration out of the box:
+
+- Toggle with `--multi-turn/--no-multi-turn`
+- Limit depth via `--max-turns`
+- Control tool approvals with `--auto-approve-tools/--manual-approve-tools`
+- Override the supervisor persona target: `--persona-override`
+- Point at a specific LLM: `--supervisor-provider`, `--supervisor-model`, `--supervisor-temperature`, `--supervisor-api-key`
+
+These flags override the values in `configs/simulation.yaml` (`multi_turn` block). When enabled, the runner consults the supervisor after every turn to decide whether to continue and to synthesize the next realistic user message.
+
 Run `fluxloop --help` or `fluxloop <command> --help` for more detail.
 
 ## Quick Setup Script

{fluxloop_cli-0.2.20 → fluxloop_cli-0.2.22}/README.md

@@ -31,6 +31,18 @@ The legacy `setting.yaml` is still supported, but new projects created with
 - `fluxloop record enable|disable|status` – toggle recording mode across `.env` and simulation config
 - `fluxloop doctor` – summarize Python, FluxLoop CLI/MCP, and MCP index state for the active environment
 
+### Multi-turn supervisor options
+
+`fluxloop run experiment` supports multi-turn orchestration out of the box:
+
+- Toggle with `--multi-turn/--no-multi-turn`
+- Limit depth via `--max-turns`
+- Control tool approvals with `--auto-approve-tools/--manual-approve-tools`
+- Override the supervisor persona target: `--persona-override`
+- Point at a specific LLM: `--supervisor-provider`, `--supervisor-model`, `--supervisor-temperature`, `--supervisor-api-key`
+
+These flags override the values in `configs/simulation.yaml` (`multi_turn` block). When enabled, the runner consults the supervisor after every turn to decide whether to continue and to synthesize the next realistic user message.
+
 Run `fluxloop --help` or `fluxloop <command> --help` for more detail.
 
 ## Quick Setup Script

{fluxloop_cli-0.2.20 → fluxloop_cli-0.2.22}/fluxloop_cli/__init__.py

@@ -2,7 +2,7 @@
 FluxLoop CLI - Command-line interface for running agent simulations.
 """
 
-__version__ = "0.2.20"
+__version__ = "0.2.22"
 
 from .main import app
 

{fluxloop_cli-0.2.20 → fluxloop_cli-0.2.22}/fluxloop_cli/arg_binder.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 import inspect
 import json
 from pathlib import Path
-from typing import Any, Callable, Dict, Optional
+from typing import Any, Callable, Dict, Optional, Sequence
 
 
 class _AttrDict(dict):
@@ -26,7 +26,7 @@ class _AttrDict(dict):
         except KeyError as exc:  # pragma: no cover
             raise AttributeError(item) from exc
 
-from fluxloop.schemas import ExperimentConfig, ReplayArgsConfig
+from fluxloop.schemas import ExperimentConfig, ReplayArgsConfig, PersonaConfig
 
 
 class _AwaitableNone:
@@ -98,9 +98,17 @@ class ArgBinder:
         *,
         runtime_input: str,
         iteration: int = 0,
+        conversation_state: Optional[Dict[str, Any]] = None,
+        persona: Optional[PersonaConfig] = None,
+        auto_approve: Optional[bool] = None,
     ) -> Dict[str, Any]:
         """Construct kwargs for calling *func* based on replay or inspection."""
 
+        signature = inspect.signature(func)
+        parameters = list(signature.parameters.values())
+        if parameters and parameters[0].name == "self":
+            parameters = parameters[1:]
+
         if self._recording:
             kwargs = self._recording.get("kwargs", {}).copy()
 
@@ -111,22 +119,42 @@ class ArgBinder:
                 try:
                     self._set_by_path(kwargs, replay.override_param_path, runtime_input)
                 except (KeyError, TypeError):
-                    # If path missing, fall back to plain binding
-                    return self._bind_by_signature(func, runtime_input)
-
+                    kwargs = self._bind_runtime_input(parameters, runtime_input)
+            else:
+                fallback = self._bind_runtime_input(parameters, runtime_input)
+                for key, value in fallback.items():
+                    kwargs.setdefault(key, value)
             self._restore_callables(kwargs, replay)
             self._ensure_no_unmapped_callables(kwargs, replay)
-            return self._hydrate_structures(kwargs)
-
-        return self._bind_by_signature(func, runtime_input)
-
-    def _bind_by_signature(self, func: Callable, runtime_input: str) -> Dict[str, Any]:
-        signature = inspect.signature(func)
-        parameters = list(signature.parameters.values())
+            kwargs = self._hydrate_structures(kwargs)
+        else:
+            kwargs = self._bind_runtime_input(parameters, runtime_input)
+
+        return self._inject_optional_kwargs(
+            parameters=parameters,
+            kwargs=kwargs,
+            conversation_state=conversation_state,
+            persona=persona,
+            auto_approve=auto_approve,
+            iteration=iteration,
+        )
 
-        if parameters and parameters[0].name == "self":
-            parameters = parameters[1:]
+    def _bind_runtime_input(
+        self, parameters: Sequence[inspect.Parameter], runtime_input: str
+    ) -> Dict[str, Any]:
+        candidate = self._find_runtime_parameter(parameters)
+        if candidate:
+            return {candidate: runtime_input}
+        if parameters:
+            return {parameters[0].name: runtime_input}
+        raise ValueError(
+            "Cannot determine where to bind runtime input for the provided function."
+        )
 
+    @staticmethod
+    def _find_runtime_parameter(
+        parameters: Sequence[inspect.Parameter],
+    ) -> Optional[str]:
         candidate_names = [
             "input",
             "input_text",
@@ -134,18 +162,70 @@ class ArgBinder:
             "query",
             "text",
             "content",
+            "user_message",
         ]
+        for name in candidate_names:
+            for param in parameters:
+                if param.name == name:
+                    return name
+        return None
 
-        for param in parameters:
-            if param.name in candidate_names:
-                return {param.name: runtime_input}
+    def _inject_optional_kwargs(
+        self,
+        *,
+        parameters: Sequence[inspect.Parameter],
+        kwargs: Dict[str, Any],
+        conversation_state: Optional[Dict[str, Any]],
+        persona: Optional[PersonaConfig],
+        auto_approve: Optional[bool],
+        iteration: Optional[int],
+    ) -> Dict[str, Any]:
+        param_names = {param.name for param in parameters}
+
+        def assign(value: Any, candidates: Sequence[str]) -> bool:
+            if value is None:
+                return False
+            for name in candidates:
+                if name in param_names and name not in kwargs:
+                    kwargs[name] = value
+                    return True
+            return False
+
+        if conversation_state is not None:
+            assign(conversation_state, ["conversation_state", "state", "dialog_state"])
+            if isinstance(conversation_state, dict):
+                metadata = conversation_state.get("metadata")
+                if metadata:
+                    assign(
+                        metadata,
+                        ["conversation_metadata", "state_metadata", "conversation_meta"],
+                    )
+                turns = conversation_state.get("turns")
+                if turns:
+                    assign(turns, ["messages", "history", "turns"])
+
+        if persona is not None:
+            assign(persona, ["persona", "user_persona", "persona_config"])
+            try:
+                persona_prompt = persona.to_prompt()
+            except Exception:
+                persona_prompt = None
+            if persona_prompt:
+                assign(
+                    persona_prompt,
+                    ["persona_prompt", "persona_description", "persona_text"],
+                )
+
+        if auto_approve is not None:
+            assign(
+                auto_approve,
+                ["auto_approve", "auto_approve_tools", "approve_tools", "autoapprove"],
+            )
 
-        if parameters:
-            return {parameters[0].name: runtime_input}
+        if iteration is not None:
+            assign(iteration, ["iteration", "run_iteration", "loop_index"])
 
-        raise ValueError(
-            f"Cannot determine where to bind runtime input for function '{func.__name__}'."
-        )
+        return kwargs
 
     def _restore_callables(self, kwargs: Dict[str, Any], replay: ReplayArgsConfig) -> None:
         for param_name, provider in replay.callable_providers.items():

{fluxloop_cli-0.2.20 → fluxloop_cli-0.2.22}/fluxloop_cli/commands/evaluate.py

@@ -2,12 +2,14 @@
 
 from __future__ import annotations
 
+import os
 from pathlib import Path
 from typing import Optional
 
 import typer
 from rich.console import Console
 
+from ..environment import load_env_chain
 from ..evaluation import EvaluationOptions, load_evaluation_config, run_evaluation
 
 console = Console()
@@ -134,6 +136,20 @@ def experiment(
     except Exception as exc:  # noqa: BLE001
         raise typer.BadParameter(f"Failed to load evaluation config: {exc}") from exc
 
+    def _log_env_error(path: Path, exc: Exception) -> None:
+        console.log(
+            f"[yellow]Warning:[/yellow] Failed to load environment from {path}: {exc}"
+        )
+
+    load_env_chain(
+        evaluation_config.get_source_dir(),
+        refresh_config=True,
+        on_error=_log_env_error,
+    )
+
+    if llm_api_key is None:
+        llm_api_key = os.getenv("FLUXLOOP_LLM_API_KEY") or os.getenv("OPENAI_API_KEY")
+
     output_dir = output
     if not output_dir.is_absolute():
         output_dir = resolved_experiment_dir / output_dir

{fluxloop_cli-0.2.20 → fluxloop_cli-0.2.22}/fluxloop_cli/commands/init.py

@@ -179,12 +179,16 @@ def project(
     # Show next steps
     console.print("\n[bold]Next steps:[/bold]")
     console.print("1. Review configs in [cyan]configs/[/cyan] (project/input/simulation/evaluation)")
-    console.print("2. Set up environment variables in [cyan].env[/cyan]")
+    console.print("2. Configure secrets via [cyan].env[/cyan] or [green]fluxloop config set-llm[/green]")
     if with_example:
-        console.print("3. Try running: [green]fluxloop run experiment[/green]")
+        console.print("3. Customize the sample agent in [cyan]examples/simple_agent.py[/cyan]")
     else:
-        console.print("3. Add your agent code")
-        console.print("4. Run: [green]fluxloop run experiment[/green]")
+        console.print("3. Create your agent: [green]fluxloop init agent <name>[/green]")
+    console.print("4. Generate inputs: [green]fluxloop generate inputs[/green]")
+    console.print("5. Run the experiment: [green]fluxloop run experiment[/green]")
+    console.print("6. Parse outputs: [green]fluxloop parse experiment[/green]")
+    console.print("7. Evaluate results (optional): [green]fluxloop evaluate experiment[/green]")
+    console.print("8. Diagnose environment anytime: [green]fluxloop doctor[/green]")
 
 
 @app.command()

{fluxloop_cli-0.2.20 → fluxloop_cli-0.2.22}/fluxloop_cli/commands/run.py

@@ -21,6 +21,7 @@ from ..project_paths import (
     resolve_config_path,
     resolve_project_relative,
 )
+from fluxloop.schemas import MultiTurnConfig
 
 app = typer.Typer()
 console = Console()
@@ -56,6 +57,46 @@ def experiment(
         "-p",
         help="Comma-separated list of personas to use",
     ),
+    multi_turn: Optional[bool] = typer.Option(
+        None,
+        "--multi-turn/--no-multi-turn",
+        help="Enable or disable multi-turn supervisor loop",
+    ),
+    max_turns: Optional[int] = typer.Option(
+        None,
+        "--max-turns",
+        help="Override maximum number of turns per conversation",
+    ),
+    auto_approve_tools: Optional[bool] = typer.Option(
+        None,
+        "--auto-approve-tools/--manual-approve-tools",
+        help="Override automatic tool approval behaviour",
+    ),
+    persona_override: Optional[str] = typer.Option(
+        None,
+        "--persona-override",
+        help="Force a specific persona id during multi-turn execution",
+    ),
+    supervisor_provider: Optional[str] = typer.Option(
+        None,
+        "--supervisor-provider",
+        help="Override conversation supervisor provider (e.g. openai, mock)",
+    ),
+    supervisor_model: Optional[str] = typer.Option(
+        None,
+        "--supervisor-model",
+        help="Override conversation supervisor model identifier",
+    ),
+    supervisor_temperature: Optional[float] = typer.Option(
+        None,
+        "--supervisor-temperature",
+        help="Override conversation supervisor sampling temperature",
+    ),
+    supervisor_api_key: Optional[str] = typer.Option(
+        None,
+        "--supervisor-api-key",
+        help="API key to use for supervisor calls (overrides environment)",
+    ),
     output_dir: Optional[Path] = typer.Option(
         None,
         "--output",
@@ -114,6 +155,39 @@ def experiment(
     if output_dir:
         resolved_output = resolve_project_relative(output_dir, project, root)
         config.output_directory = str(resolved_output)
+
+    if any(
+        value is not None
+        for value in [
+            multi_turn,
+            max_turns,
+            auto_approve_tools,
+            persona_override,
+            supervisor_provider,
+            supervisor_model,
+            supervisor_temperature,
+            supervisor_api_key,
+        ]
+    ):
+        if config.multi_turn is None:
+            config.multi_turn = MultiTurnConfig()
+        mt = config.multi_turn
+        if multi_turn is not None:
+            mt.enabled = multi_turn
+        if max_turns is not None:
+            mt.max_turns = max_turns
+        if auto_approve_tools is not None:
+            mt.auto_approve_tools = auto_approve_tools
+        if persona_override:
+            mt.persona_override = persona_override
+        if supervisor_provider:
+            mt.supervisor.provider = supervisor_provider
+        if supervisor_model:
+            mt.supervisor.model = supervisor_model
+        if supervisor_temperature is not None:
+            mt.supervisor.temperature = supervisor_temperature
+        if supervisor_api_key:
+            mt.supervisor.api_key = supervisor_api_key
     
     # Load inputs to ensure accurate counts before showing the summary
     try:
@@ -137,6 +211,18 @@ def experiment(
         "Input Source",
         "external file" if config.has_external_inputs() else "base_inputs",
     )
+    if config.multi_turn:
+        summary.add_row(
+            "Multi-turn",
+            "enabled" if config.multi_turn.enabled else "disabled",
+        )
+        summary.add_row("Max Turns", str(config.multi_turn.max_turns))
+        if config.multi_turn.persona_override:
+            summary.add_row("Persona Override", config.multi_turn.persona_override)
+        summary.add_row(
+            "Supervisor Model",
+            config.multi_turn.supervisor.model,
+        )
     summary.add_row("Total Runs", str(total_runs))
     summary.add_row("Output", config.output_directory)
     

fluxloop_cli-0.2.22/fluxloop_cli/conversation_supervisor.py

@@ -0,0 +1,258 @@
+"""
+Conversation supervisor for multi-turn simulations.
+
+This module encapsulates the logic required to consult an LLM (or mock strategy)
+to decide whether a conversation should continue and, if so, to generate the
+next user utterance consistent with the experiment persona and service context.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from dataclasses import dataclass
+from typing import Any, Dict, List, Literal, Optional, Sequence
+
+import httpx
+
+from fluxloop.schemas import MultiTurnSupervisorConfig
+
+SupervisorDecisionType = Literal["continue", "terminate"]
+
+
+@dataclass
+class SupervisorDecision:
+    """Structured decision returned by the conversation supervisor."""
+
+    decision: SupervisorDecisionType
+    next_user_message: Optional[str] = None
+    termination_reason: Optional[str] = None
+    closing_user_message: Optional[str] = None
+    raw_response: Optional[Dict[str, Any]] = None
+
+    def requires_follow_up(self) -> bool:
+        return self.decision == "continue"
+
+
+def format_transcript(turns: Sequence[Dict[str, Any]]) -> str:
+    """Render a conversation transcript into a bullet list for supervisor prompts."""
+
+    lines: List[str] = []
+    for idx, turn in enumerate(turns, start=1):
+        role = turn.get("role", "unknown").capitalize()
+        content = turn.get("content") or ""
+        content = content.strip()
+
+        if turn.get("tool_calls"):
+            tool_calls = turn["tool_calls"]
+            if isinstance(tool_calls, list):
+                tool_summaries = []
+                for call in tool_calls:
+                    name = call.get("name") or call.get("tool") or "tool"
+                    args = call.get("arguments") or call.get("args") or {}
+                    tool_summaries.append(f"{name}({json.dumps(args, ensure_ascii=False)})")
+                if tool_summaries:
+                    content = f"{content}\n    [Tool Calls] " + "; ".join(tool_summaries)
+        lines.append(f"{idx}. {role}: {content}")
+
+    return "\n".join(lines)
+
+
+def build_supervisor_prompt(
+    turns: Sequence[Dict[str, Any]],
+    persona_description: Optional[str],
+    service_context: Optional[str],
+    instructions: Optional[str],
+) -> str:
+    """Create the textual prompt sent to the supervisor LLM."""
+
+    persona_text = persona_description or "Generic customer"
+    service_text = service_context or "Customer support scenario"
+    transcript_text = format_transcript(turns)
+
+    guidance = instructions or (
+        "You supervise an AI assistant. Review the transcript and decide whether the "
+        "conversation should continue. When continuing, craft the next user message "
+        "consistent with the persona. When terminating, explain why and provide any "
+        "closing notes."
+    )
+
+    prompt = (
+        f"{guidance}\n\n"
+        f"Service Context: {service_text}\n"
+        f"Persona: {persona_text}\n\n"
+        "Transcript so far:\n"
+        f"{transcript_text}\n\n"
+        "Respond ONLY in JSON with the following schema:\n"
+        "{\n"
+        '  "decision": "continue" | "terminate",\n'
+        '  "next_user_message": string | null,\n'
+        '  "termination_reason": string | null,\n'
+        '  "closing_user_message": string | null\n'
+        "}\n"
+    )
+
+    return prompt
+
+
+class ConversationSupervisor:
+    """High-level interface for querying the conversation supervisor LLM."""
+
+    def __init__(self, config: MultiTurnSupervisorConfig):
+        self.config = config
+
+    async def decide(
+        self,
+        *,
+        conversation_state: Dict[str, Any],
+        persona_description: Optional[str],
+        service_context: Optional[str],
+    ) -> SupervisorDecision:
+        turns: Sequence[Dict[str, Any]] = conversation_state.get("turns", [])
+        prompt = build_supervisor_prompt(
+            turns=turns,
+            persona_description=persona_description,
+            service_context=service_context,
+            instructions=self.config.system_prompt,
+        )
+
+        provider = (self.config.provider or "openai").lower()
+        if provider == "mock":
+            return self._mock_decision()
+        if provider == "openai":
+            response_text = await self._call_openai(prompt)
+        else:
+            raise ValueError(f"Unsupported supervisor provider: {self.config.provider}")
+
+        return self._parse_decision(response_text)
+
+    def _mock_decision(self) -> SupervisorDecision:
+        """Deterministic supervisor used for tests or offline runs."""
+
+        metadata = self.config.metadata or {}
+        default_decision = metadata.get("mock_decision", "terminate")
+        decision: SupervisorDecisionType = (
+            default_decision if default_decision in {"continue", "terminate"} else "terminate"
+        )
+
+        if decision == "continue":
+            next_message = metadata.get(
+                "mock_next_user_message",
+                "Could you provide a bit more detail about that?",
+            )
+            return SupervisorDecision(
+                decision="continue",
+                next_user_message=next_message,
+                raw_response={"mock": True},
+            )
+
+        return SupervisorDecision(
+            decision="terminate",
+            termination_reason=metadata.get("mock_reason", "Supervisor mock termination"),
+            closing_user_message=metadata.get(
+                "mock_closing",
+                "Thanks for the help. I have no further questions.",
+            ),
+            raw_response={"mock": True},
+        )
+
+    async def _call_openai(self, prompt: str) -> str:
+        """Call OpenAI chat completions endpoint."""
+
+        api_key = (
+            self.config.api_key
+            or self.config.metadata.get("api_key")
+            or os.getenv("FLUXLOOP_SUPERVISOR_API_KEY")
+            or os.getenv("OPENAI_API_KEY")
+        )
+
+        if not api_key:
+            raise RuntimeError(
+                "OpenAI supervisor requires an API key. "
+                "Set multi_turn.supervisor.api_key or FLUXLOOP_SUPERVISOR_API_KEY/OPENAI_API_KEY."
+            )
+
+        messages: List[Dict[str, str]] = []
+        if self.config.system_prompt:
+            messages.append({"role": "system", "content": self.config.system_prompt})
+        messages.append({"role": "user", "content": prompt})
+
+        payload = {
+            "model": self.config.model,
+            "messages": messages,
+            "temperature": self.config.temperature,
+        }
+
+        async with httpx.AsyncClient(timeout=60) as client:
+            response = await client.post(
+                "https://api.openai.com/v1/chat/completions",
+                headers={
+                    "Authorization": f"Bearer {api_key}",
+                    "Content-Type": "application/json",
+                },
+                json=payload,
+            )
+
+        if response.status_code >= 400:
+            raise RuntimeError(
+                f"OpenAI supervisor request failed ({response.status_code}): {response.text}"
+            )
+
+        data = response.json()
+        choices = data.get("choices") or []
+        if not choices:
+            raise RuntimeError("OpenAI supervisor response contained no choices.")
+
+        message = choices[0].get("message") or {}
+        content = message.get("content")
+        if not content:
+            raise RuntimeError("OpenAI supervisor response missing content.")
+
+        return content
+
+    def _parse_decision(self, content: str) -> SupervisorDecision:
+        """Extract JSON payload from supervisor response and convert to decision."""
+
+        payload = self._extract_json(content)
+        decision_value = payload.get("decision")
+        if decision_value not in {"continue", "terminate"}:
+            raise ValueError(f"Supervisor returned invalid decision: {decision_value}")
+
+        decision: SupervisorDecisionType = decision_value  # type: ignore[assignment]
+        result = SupervisorDecision(
+            decision=decision,
+            next_user_message=payload.get("next_user_message"),
+            termination_reason=payload.get("termination_reason"),
+            closing_user_message=payload.get("closing_user_message"),
+            raw_response=payload,
+        )
+
+        if decision == "continue":
+            if not (result.next_user_message and result.next_user_message.strip()):
+                raise ValueError("Supervisor decided to continue but provided no next_user_message.")
+        return result
+
+    @staticmethod
+    def _extract_json(content: str) -> Dict[str, Any]:
+        """Attempt to parse a JSON object from the model output."""
+
+        content = content.strip()
+        try:
+            return json.loads(content)
+        except json.JSONDecodeError:
+            pass
+
+        # Attempt to find JSON substring
+        match = re.search(r"\{.*\}", content, flags=re.DOTALL)
+        if match:
+            try:
+                return json.loads(match.group(0))
+            except json.JSONDecodeError:
+                pass
+
+        raise ValueError(f"Supervisor response was not valid JSON: {content}")
+
+
+__all__ = ["ConversationSupervisor", "SupervisorDecision", "build_supervisor_prompt"]
+