fluxloop-cli 0.2.12__tar.gz

fluxloop_cli-0.2.12/PKG-INFO

@@ -0,0 +1,102 @@
+Metadata-Version: 2.4
+Name: fluxloop-cli
+Version: 0.2.12
+Summary: FluxLoop CLI for running agent simulations
+Author-email: FluxLoop Team <team@fluxloop.dev>
+License: Apache-2.0
+Project-URL: Homepage, https://github.com/chuckgu/fluxloop
+Project-URL: Documentation, https://docs.fluxloop.dev
+Project-URL: Repository, https://github.com/chuckgu/fluxloop
+Project-URL: Issues, https://github.com/chuckgu/fluxloop/issues
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: typer[all]>=0.9.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: httpx>=0.24.0
+Requires-Dist: rich>=13.0
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: fluxloop>=0.1.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: mypy>=1.0; extra == "dev"
+Requires-Dist: black>=23.0; extra == "dev"
+Provides-Extra: openai
+Requires-Dist: openai>=1.0.0; extra == "openai"
+Provides-Extra: anthropic
+Requires-Dist: anthropic>=0.7.0; extra == "anthropic"
+
+# FluxLoop CLI
+
+Command-line interface for running agent simulations.
+
+## Installation
+
+```
+pip install fluxloop-cli
+```
+
+## Configuration Overview (v0.2.0)
+
+FluxLoop CLI now stores experiment settings in four files under `configs/`:
+
+- `configs/project.yaml` – project metadata, collector defaults
+- `configs/input.yaml` – personas, base inputs, input generation options
+- `configs/simulation.yaml` – runtime parameters (iterations, runner, replay args)
+- `configs/evaluation.yaml` – evaluator definitions (rule-based, LLM judge, etc.)
+
+The legacy `setting.yaml` is still supported, but new projects created with
+`fluxloop init project` will generate the structured layout above.
+
+## Key Commands
+
+- `fluxloop init project` – scaffold a new project (configs, `.env`, examples)
+- `fluxloop generate inputs` – produce input variations for the active project
+- `fluxloop run experiment` – execute an experiment using `configs/simulation.yaml`
+- `fluxloop parse experiment` – convert experiment outputs into readable artifacts
+- `fluxloop config set-llm` – update LLM provider/model in `configs/input.yaml`
+- `fluxloop record enable|disable|status` – toggle recording mode across `.env` and simulation config
+
+Run `fluxloop --help` or `fluxloop <command> --help` for more detail.
+
+## Runner Integration Patterns
+
+Configure how FluxLoop calls your code in `configs/simulation.yaml`:
+
+- Module + function: `module_path`/`function_name` or `target: "module:function"`
+- Class.method (zero-arg ctor): `target: "module:Class.method"`
+- Module-scoped instance method: `target: "module:instance.method"`
+- Class.method with factory: add `factory: "module:make_instance"` (+ `factory_kwargs`)
+- Async generators: set `runner.stream_output_path` if your streamed event shape differs (default `message.delta`).
+
+See full examples: `packages/website/docs-cli/configuration/runner-targets.md`.
+
+## Developing
+
+Install dependencies and run tests:
+
+```
+python -m venv .venv
+source .venv/bin/activate
+pip install -e .[dev]
+pytest
+```
+
+To package the CLI:
+
+```
+./build.sh
+```
+

fluxloop_cli-0.2.12/README.md

@@ -0,0 +1,62 @@
+# FluxLoop CLI
+
+Command-line interface for running agent simulations.
+
+## Installation
+
+```
+pip install fluxloop-cli
+```
+
+## Configuration Overview (v0.2.0)
+
+FluxLoop CLI now stores experiment settings in four files under `configs/`:
+
+- `configs/project.yaml` – project metadata, collector defaults
+- `configs/input.yaml` – personas, base inputs, input generation options
+- `configs/simulation.yaml` – runtime parameters (iterations, runner, replay args)
+- `configs/evaluation.yaml` – evaluator definitions (rule-based, LLM judge, etc.)
+
+The legacy `setting.yaml` is still supported, but new projects created with
+`fluxloop init project` will generate the structured layout above.
+
+## Key Commands
+
+- `fluxloop init project` – scaffold a new project (configs, `.env`, examples)
+- `fluxloop generate inputs` – produce input variations for the active project
+- `fluxloop run experiment` – execute an experiment using `configs/simulation.yaml`
+- `fluxloop parse experiment` – convert experiment outputs into readable artifacts
+- `fluxloop config set-llm` – update LLM provider/model in `configs/input.yaml`
+- `fluxloop record enable|disable|status` – toggle recording mode across `.env` and simulation config
+
+Run `fluxloop --help` or `fluxloop <command> --help` for more detail.
+
+## Runner Integration Patterns
+
+Configure how FluxLoop calls your code in `configs/simulation.yaml`:
+
+- Module + function: `module_path`/`function_name` or `target: "module:function"`
+- Class.method (zero-arg ctor): `target: "module:Class.method"`
+- Module-scoped instance method: `target: "module:instance.method"`
+- Class.method with factory: add `factory: "module:make_instance"` (+ `factory_kwargs`)
+- Async generators: set `runner.stream_output_path` if your streamed event shape differs (default `message.delta`).
+
+See full examples: `packages/website/docs-cli/configuration/runner-targets.md`.
+
+## Developing
+
+Install dependencies and run tests:
+
+```
+python -m venv .venv
+source .venv/bin/activate
+pip install -e .[dev]
+pytest
+```
+
+To package the CLI:
+
+```
+./build.sh
+```
+

fluxloop_cli-0.2.12/fluxloop_cli/__init__.py

@@ -0,0 +1,9 @@
+"""
+FluxLoop CLI - Command-line interface for running agent simulations.
+"""
+
+__version__ = "0.2.12"
+
+from .main import app
+
+__all__ = ["app"]

fluxloop_cli-0.2.12/fluxloop_cli/arg_binder.py

@@ -0,0 +1,274 @@
+"""Argument binding utilities with replay support."""
+
+from __future__ import annotations
+
+import inspect
+import json
+from pathlib import Path
+from typing import Any, Callable, Dict, Optional
+
+
+class _AttrDict(dict):
+    """Dictionary that also supports attribute access for keys."""
+
+    def __getattr__(self, item: str) -> Any:  # type: ignore[override]
+        try:
+            return self[item]
+        except KeyError as exc:  # pragma: no cover
+            raise AttributeError(item) from exc
+
+    def __setattr__(self, key: str, value: Any) -> None:  # type: ignore[override]
+        self[key] = value
+
+    def __delattr__(self, item: str) -> None:  # type: ignore[override]
+        try:
+            del self[item]
+        except KeyError as exc:  # pragma: no cover
+            raise AttributeError(item) from exc
+
+from fluxloop.schemas import ExperimentConfig, ReplayArgsConfig
+
+
+class _AwaitableNone:
+    """Simple awaitable that resolves to ``None``."""
+
+    def __await__(self):  # type: ignore[override]
+        async def _noop() -> None:
+            return None
+
+        return _noop().__await__()
+
+
+class ArgBinder:
+    """Bind call arguments using replay data when configured."""
+
+    def __init__(self, experiment_config: ExperimentConfig) -> None:
+        self.config = experiment_config
+        self.replay_config: Optional[ReplayArgsConfig] = experiment_config.replay_args
+        self._recording: Optional[Dict[str, Any]] = None
+
+        if self.replay_config and self.replay_config.enabled:
+            self._load_recording()
+
+    def _load_recording(self) -> None:
+        replay = self.replay_config
+        assert replay is not None
+
+        if not replay.recording_file:
+            raise ValueError("replay_args.recording_file must be provided when replay is enabled")
+
+        file_path = Path(replay.recording_file)
+        if not file_path.is_absolute():
+            source_dir = self.config.get_source_dir()
+            if source_dir:
+                file_path = (source_dir / file_path).resolve()
+            else:
+                file_path = file_path.resolve()
+
+        if not file_path.exists():
+            raise FileNotFoundError(
+                f"Recording file not found: {file_path}. Make sure it is available locally."
+            )
+
+        last_line: Optional[str] = None
+        with file_path.open("r", encoding="utf-8") as fp:
+            for line in fp:
+                if line.strip():
+                    last_line = line
+
+        if not last_line:
+            raise ValueError(f"Recording file is empty: {file_path}")
+
+        try:
+            self._recording = json.loads(last_line)
+        except json.JSONDecodeError as exc:
+            raise ValueError(f"Invalid JSON in recording file {file_path}: {exc}")
+
+        recording_target = self._recording.get("target")
+        config_target = self._resolve_config_target()
+        if recording_target and recording_target != config_target:
+            print(
+                "⚠️  Recording target mismatch:"
+                f" recording={recording_target}, config={config_target}. Proceeding anyway."
+            )
+
+    def bind_call_args(
+        self,
+        func: Callable,
+        *,
+        runtime_input: str,
+        iteration: int = 0,
+    ) -> Dict[str, Any]:
+        """Construct kwargs for calling *func* based on replay or inspection."""
+
+        if self._recording:
+            kwargs = self._recording.get("kwargs", {}).copy()
+
+            replay = self.replay_config
+            assert replay is not None
+
+            if replay.override_param_path:
+                try:
+                    self._set_by_path(kwargs, replay.override_param_path, runtime_input)
+                except (KeyError, TypeError):
+                    # If path missing, fall back to plain binding
+                    return self._bind_by_signature(func, runtime_input)
+
+            self._restore_callables(kwargs, replay)
+            self._ensure_no_unmapped_callables(kwargs, replay)
+            return self._hydrate_structures(kwargs)
+
+        return self._bind_by_signature(func, runtime_input)
+
+    def _bind_by_signature(self, func: Callable, runtime_input: str) -> Dict[str, Any]:
+        signature = inspect.signature(func)
+        parameters = list(signature.parameters.values())
+
+        if parameters and parameters[0].name == "self":
+            parameters = parameters[1:]
+
+        candidate_names = [
+            "input",
+            "input_text",
+            "message",
+            "query",
+            "text",
+            "content",
+        ]
+
+        for param in parameters:
+            if param.name in candidate_names:
+                return {param.name: runtime_input}
+
+        if parameters:
+            return {parameters[0].name: runtime_input}
+
+        raise ValueError(
+            f"Cannot determine where to bind runtime input for function '{func.__name__}'."
+        )
+
+    def _restore_callables(self, kwargs: Dict[str, Any], replay: ReplayArgsConfig) -> None:
+        for param_name, provider in replay.callable_providers.items():
+            if param_name not in kwargs:
+                continue
+
+            marker = kwargs[param_name]
+            if isinstance(marker, str) and marker.startswith("<"):
+                kwargs[param_name] = self._resolve_builtin_callable(provider, marker)
+
+    def _ensure_no_unmapped_callables(self, kwargs: Dict[str, Any], replay: ReplayArgsConfig) -> None:
+        callable_markers = {
+            key: value
+            for key, value in kwargs.items()
+            if isinstance(value, str)
+            and value.startswith("<")
+            and not value.startswith("<repr:")
+        }
+
+        if not callable_markers:
+            return
+
+        configured = set(replay.callable_providers.keys())
+        missing = [key for key in callable_markers if key not in configured]
+        if missing:
+            raise ValueError(
+                "Missing callable providers for recorded parameters: "
+                f"{', '.join(missing)}. Configure them under replay_args.callable_providers."
+            )
+
+    def _hydrate_structures(self, payload: Dict[str, Any]) -> Dict[str, Any]:
+        return {key: self._hydrate_value(value) for key, value in payload.items()}
+
+    def _hydrate_value(self, value: Any) -> Any:
+        if callable(value):
+            return value
+        if isinstance(value, _AttrDict):
+            return value
+        if isinstance(value, dict):
+            return _AttrDict({k: self._hydrate_value(v) for k, v in value.items()})
+        if isinstance(value, list):
+            return [self._hydrate_value(item) for item in value]
+        return value
+
+    def _resolve_builtin_callable(self, provider: str, marker: str) -> Callable:
+        is_async = marker.endswith(":async>")
+
+        if provider == "builtin:collector.send":
+            messages: list = []
+
+            def _record(args: Any, kwargs: Any) -> None:
+                messages.append((args, kwargs))
+                pretty = args[0] if len(args) == 1 and not kwargs else {"args": args, "kwargs": kwargs}
+
+            def send(*args: Any, **kwargs: Any) -> _AwaitableNone:
+                _record(args, kwargs)
+                return _AwaitableNone()
+
+            async def send_async(*args: Any, **kwargs: Any) -> None:
+                _record(args, kwargs)
+
+            send.messages = messages
+            send_async.messages = messages
+            send.__fluxloop_builtin__ = "collector.send"
+            send_async.__fluxloop_builtin__ = "collector.send:async"
+            return send_async if is_async else send
+
+        if provider == "builtin:collector.error":
+            errors: list = []
+
+            def _record_error(args: Any, kwargs: Any) -> None:
+                errors.append((args, kwargs))
+                pretty = args[0] if len(args) == 1 and not kwargs else {"args": args, "kwargs": kwargs}
+                print(f"[ERROR] {pretty}")
+
+            def send_error(*args: Any, **kwargs: Any) -> _AwaitableNone:
+                _record_error(args, kwargs)
+                return _AwaitableNone()
+
+            async def send_error_async(*args: Any, **kwargs: Any) -> None:
+                _record_error(args, kwargs)
+
+            send_error.errors = errors
+            send_error_async.errors = errors
+            send_error.__fluxloop_builtin__ = "collector.error"
+            send_error_async.__fluxloop_builtin__ = "collector.error:async"
+            return send_error_async if is_async else send_error
+
+        raise ValueError(
+            f"Unknown callable provider '{provider}'. Supported providers: "
+            "builtin:collector.send, builtin:collector.error."
+        )
+
+    def _set_by_path(self, payload: Dict[str, Any], path: str, value: Any) -> None:
+        parts = path.split(".")
+        current: Any = payload
+
+        for key in parts[:-1]:
+            if isinstance(current, list):
+                index = self._coerce_list_index(key)
+                current = current[index]
+            else:
+                current = current[key]
+
+        final_key = parts[-1]
+        if isinstance(current, list):
+            index = self._coerce_list_index(final_key)
+            current[index] = value
+        else:
+            current[final_key] = value
+
+    @staticmethod
+    def _coerce_list_index(key: str) -> int:
+        try:
+            return int(key)
+        except ValueError as exc:
+            raise TypeError(
+                "List index segments in override_param_path must be integers"
+            ) from exc
+
+    def _resolve_config_target(self) -> Optional[str]:
+        runner = self.config.runner
+        if runner.target:
+            return runner.target
+        return f"{runner.module_path}:{runner.function_name}"
+

fluxloop_cli-0.2.12/fluxloop_cli/commands/__init__.py

@@ -0,0 +1,5 @@
+"""CLI commands."""
+
+from . import config, generate, init, parse, run, status, record
+
+__all__ = ["config", "generate", "init", "parse", "run", "status", "record"]