fluxloop-cli 0.3.9__tar.gz → 0.3.11__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/PKG-INFO +27 -1
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/README.md +26 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/__init__.py +1 -1
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/commands/bundles.py +9 -0
- fluxloop_cli-0.3.11/fluxloop_cli/commands/data.py +1316 -0
- fluxloop_cli-0.3.11/fluxloop_cli/commands/evaluate.py +353 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/commands/inputs.py +219 -7
- fluxloop_cli-0.3.11/fluxloop_cli/commands/personas.py +580 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/commands/projects.py +21 -3
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/commands/scenarios.py +46 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/commands/sync.py +43 -0
- fluxloop_cli-0.3.11/fluxloop_cli/language.py +17 -0
- fluxloop_cli-0.3.11/fluxloop_cli/progress.py +138 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli.egg-info/PKG-INFO +27 -1
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli.egg-info/SOURCES.txt +9 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/pyproject.toml +1 -1
- fluxloop_cli-0.3.11/tests/test_data_command.py +589 -0
- fluxloop_cli-0.3.11/tests/test_evaluate_server_decision.py +193 -0
- fluxloop_cli-0.3.11/tests/test_inputs_command.py +204 -0
- fluxloop_cli-0.3.11/tests/test_personas_command.py +244 -0
- fluxloop_cli-0.3.11/tests/test_projects_language.py +81 -0
- fluxloop_cli-0.3.11/tests/test_scenarios_language.py +252 -0
- fluxloop_cli-0.3.11/tests/test_sync_quick_gate_output.py +34 -0
- fluxloop_cli-0.3.9/fluxloop_cli/commands/data.py +0 -683
- fluxloop_cli-0.3.9/fluxloop_cli/commands/evaluate.py +0 -191
- fluxloop_cli-0.3.9/fluxloop_cli/commands/personas.py +0 -270
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/api_utils.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/arg_binder.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/auth_manager.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/commands/__init__.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/commands/apikeys.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/commands/auth.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/commands/config.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/commands/context.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/commands/criteria.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/commands/generate.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/commands/init.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/commands/local_context.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/commands/run.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/commands/status.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/commands/test.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/common_options.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/config_loader.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/config_schema.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/constants.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/context_manager.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/conversation_supervisor.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/environment.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/http_client.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/input_generator.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/llm_generator.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/main.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/project_paths.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/runner.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/target_loader.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/templates.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/testing/__init__.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/testing/pytest_plugin.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/testing/types.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/token_usage.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/turns.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli/validators.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli.egg-info/dependency_links.txt +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli.egg-info/entry_points.txt +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli.egg-info/requires.txt +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/fluxloop_cli.egg-info/top_level.txt +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/setup.cfg +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/tests/test_analysis_recommendations.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/tests/test_arg_binder.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/tests/test_config_command.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/tests/test_conversation_supervisor.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/tests/test_evaluate_command.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/tests/test_evaluation_llm.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/tests/test_init_pytest_template.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/tests/test_input_generator.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/tests/test_prompt_library.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/tests/test_pytest_plugin.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/tests/test_run_command.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/tests/test_runner_multi_turn.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/tests/test_success_criteria.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/tests/test_target_loader.py +0 -0
- {fluxloop_cli-0.3.9 → fluxloop_cli-0.3.11}/tests/test_testing_types.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fluxloop-cli
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.11
|
|
4
4
|
Summary: FluxLoop CLI for running agent simulations
|
|
5
5
|
Author-email: FluxLoop Team <team@fluxloop.dev>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -68,10 +68,36 @@ The legacy `setting.yaml` is still supported, but new projects created with
|
|
|
68
68
|
- `fluxloop parse experiment` – convert experiment outputs into readable artifacts and emit structured per-trace JSON at `per_trace_analysis/per_trace.jsonl`
|
|
69
69
|
- `fluxloop evaluate experiment` – run the LLM-driven evaluation pipeline (LLM-PT → rule aggregation → LLM-OV → HTML render). Requires the parsed per-trace file (or `--per-trace`) and writes an interactive report to `evaluation_report/report.html` by default.
|
|
70
70
|
- `fluxloop config set-llm` – update LLM provider/model in `configs/input.yaml`
|
|
71
|
+
- `fluxloop data push|bind|gt status` – upload/bind Ground Truth (Validation) datasets and inspect GT materialization status
|
|
71
72
|
- `fluxloop record enable|disable|status` – toggle recording mode across `.env` and simulation config
|
|
72
73
|
- `fluxloop doctor` – summarize Python, FluxLoop CLI/MCP, and MCP index state for the active environment
|
|
73
74
|
- `--yes/-y` (for `fluxloop run experiment`) – skip the interactive confirmation prompt, ideal for CI and the Pytest bridge
|
|
74
75
|
|
|
76
|
+
### Ground Truth Data Workflow
|
|
77
|
+
|
|
78
|
+
Use these commands to upload validation datasets, bind them to a scenario, and
|
|
79
|
+
check GT materialization progress:
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
# 1) Upload as Ground Truth + bind + materialize
|
|
83
|
+
fluxloop data push qa_pairs.csv \
|
|
84
|
+
--usage ground-truth \
|
|
85
|
+
--scenario <scenario_id> \
|
|
86
|
+
--split test \
|
|
87
|
+
--label-column expected_answer \
|
|
88
|
+
--row-filter "lang = 'ko'"
|
|
89
|
+
|
|
90
|
+
# 2) Bind existing data as validation (auto materialize by default)
|
|
91
|
+
fluxloop data bind <data_id> \
|
|
92
|
+
--scenario <scenario_id> \
|
|
93
|
+
--role validation \
|
|
94
|
+
--split test \
|
|
95
|
+
--label-column expected_answer
|
|
96
|
+
|
|
97
|
+
# 3) Check GT materialization status
|
|
98
|
+
fluxloop data gt status --scenario <scenario_id> [--data-id <data_id>]
|
|
99
|
+
```
|
|
100
|
+
|
|
75
101
|
### Multi-turn supervisor options
|
|
76
102
|
|
|
77
103
|
`fluxloop run experiment` supports multi-turn orchestration out of the box:
|
|
@@ -28,10 +28,36 @@ The legacy `setting.yaml` is still supported, but new projects created with
|
|
|
28
28
|
- `fluxloop parse experiment` – convert experiment outputs into readable artifacts and emit structured per-trace JSON at `per_trace_analysis/per_trace.jsonl`
|
|
29
29
|
- `fluxloop evaluate experiment` – run the LLM-driven evaluation pipeline (LLM-PT → rule aggregation → LLM-OV → HTML render). Requires the parsed per-trace file (or `--per-trace`) and writes an interactive report to `evaluation_report/report.html` by default.
|
|
30
30
|
- `fluxloop config set-llm` – update LLM provider/model in `configs/input.yaml`
|
|
31
|
+
- `fluxloop data push|bind|gt status` – upload/bind Ground Truth (Validation) datasets and inspect GT materialization status
|
|
31
32
|
- `fluxloop record enable|disable|status` – toggle recording mode across `.env` and simulation config
|
|
32
33
|
- `fluxloop doctor` – summarize Python, FluxLoop CLI/MCP, and MCP index state for the active environment
|
|
33
34
|
- `--yes/-y` (for `fluxloop run experiment`) – skip the interactive confirmation prompt, ideal for CI and the Pytest bridge
|
|
34
35
|
|
|
36
|
+
### Ground Truth Data Workflow
|
|
37
|
+
|
|
38
|
+
Use these commands to upload validation datasets, bind them to a scenario, and
|
|
39
|
+
check GT materialization progress:
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
# 1) Upload as Ground Truth + bind + materialize
|
|
43
|
+
fluxloop data push qa_pairs.csv \
|
|
44
|
+
--usage ground-truth \
|
|
45
|
+
--scenario <scenario_id> \
|
|
46
|
+
--split test \
|
|
47
|
+
--label-column expected_answer \
|
|
48
|
+
--row-filter "lang = 'ko'"
|
|
49
|
+
|
|
50
|
+
# 2) Bind existing data as validation (auto materialize by default)
|
|
51
|
+
fluxloop data bind <data_id> \
|
|
52
|
+
--scenario <scenario_id> \
|
|
53
|
+
--role validation \
|
|
54
|
+
--split test \
|
|
55
|
+
--label-column expected_answer
|
|
56
|
+
|
|
57
|
+
# 3) Check GT materialization status
|
|
58
|
+
fluxloop data gt status --scenario <scenario_id> [--data-id <data_id>]
|
|
59
|
+
```
|
|
60
|
+
|
|
35
61
|
### Multi-turn supervisor options
|
|
36
62
|
|
|
37
63
|
`fluxloop run experiment` supports multi-turn orchestration out of the box:
|
|
@@ -229,6 +229,9 @@ def list_bundles(
|
|
|
229
229
|
project_id: Optional[str] = typer.Option(
|
|
230
230
|
None, "--project-id", help="Filter by project ID"
|
|
231
231
|
),
|
|
232
|
+
format: str = typer.Option(
|
|
233
|
+
"table", "--format", help="Output format (table, json)"
|
|
234
|
+
),
|
|
232
235
|
api_url: Optional[str] = typer.Option(
|
|
233
236
|
None, "--api-url", help="FluxLoop API base URL"
|
|
234
237
|
),
|
|
@@ -275,6 +278,12 @@ def list_bundles(
|
|
|
275
278
|
)
|
|
276
279
|
return
|
|
277
280
|
|
|
281
|
+
if format == "json":
|
|
282
|
+
import json
|
|
283
|
+
|
|
284
|
+
console.print_json(json.dumps(bundles, ensure_ascii=False, default=str))
|
|
285
|
+
return
|
|
286
|
+
|
|
278
287
|
# Create table
|
|
279
288
|
table = Table(title="Bundles")
|
|
280
289
|
table.add_column("ID", style="cyan")
|