fluxloop-cli 0.2.25__tar.gz → 0.2.27__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/PKG-INFO +13 -3
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/README.md +12 -2
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/__init__.py +1 -1
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/commands/evaluate.py +20 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/commands/parse.py +108 -7
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/commands/run.py +48 -1
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/conversation_supervisor.py +20 -2
- fluxloop_cli-0.2.27/fluxloop_cli/evaluation/artifacts.py +74 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/engine/core.py +10 -23
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/engine/reporting/html.py +86 -5
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/rules.py +167 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/runner.py +38 -2
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/templates.py +4 -4
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli.egg-info/PKG-INFO +13 -3
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli.egg-info/SOURCES.txt +3 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/pyproject.toml +1 -1
- fluxloop_cli-0.2.27/tests/test_conversation_supervisor.py +39 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/tests/test_evaluate_command.py +97 -6
- fluxloop_cli-0.2.27/tests/test_run_command.py +133 -0
- fluxloop_cli-0.2.27/tests/test_runner_multi_turn.py +96 -0
- fluxloop_cli-0.2.25/tests/test_run_command.py +0 -72
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/arg_binder.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/commands/__init__.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/commands/config.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/commands/doctor.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/commands/generate.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/commands/init.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/commands/record.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/commands/status.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/config_loader.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/config_schema.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/constants.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/environment.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/__init__.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/config.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/engine/__init__.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/engine/analysis.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/engine/reporting/__init__.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/engine/reporting/markdown.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/engine/success.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/llm.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/prompts/__init__.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/prompts/base.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/prompts/information_completeness.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/prompts/intent_recognition.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/prompts/response_clarity.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/prompts/response_consistency.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/input_generator.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/llm_generator.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/main.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/project_paths.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/target_loader.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/validators.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli.egg-info/dependency_links.txt +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli.egg-info/entry_points.txt +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli.egg-info/requires.txt +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli.egg-info/top_level.txt +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/setup.cfg +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/tests/test_analysis_recommendations.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/tests/test_arg_binder.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/tests/test_config_command.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/tests/test_evaluation_llm.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/tests/test_input_generator.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/tests/test_prompt_library.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/tests/test_success_criteria.py +0 -0
- {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/tests/test_target_loader.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fluxloop-cli
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.27
|
|
4
4
|
Summary: FluxLoop CLI for running agent simulations
|
|
5
5
|
Author-email: FluxLoop Team <team@fluxloop.dev>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -66,8 +66,8 @@ The legacy `setting.yaml` is still supported, but new projects created with
|
|
|
66
66
|
- `fluxloop init project` – scaffold a new project (configs, `.env`, examples)
|
|
67
67
|
- `fluxloop generate inputs` – produce input variations for the active project
|
|
68
68
|
- `fluxloop run experiment` – execute an experiment using `configs/simulation.yaml`
|
|
69
|
-
- `fluxloop parse experiment` – convert experiment outputs into readable artifacts
|
|
70
|
-
- `fluxloop evaluate experiment` – score experiment outputs using rule-based and LLM evaluators, generate reports with success criteria, analysis, and customizable templates
|
|
69
|
+
- `fluxloop parse experiment` – convert experiment outputs into readable artifacts and emit structured per-trace JSON at `per_trace_analysis/per_trace.jsonl`
|
|
70
|
+
- `fluxloop evaluate experiment` – score experiment outputs using rule-based and LLM evaluators (requires `fluxloop parse` output or `--per-trace`), generate reports with success criteria, analysis, and customizable templates
|
|
71
71
|
- `fluxloop config set-llm` – update LLM provider/model in `configs/input.yaml`
|
|
72
72
|
- `fluxloop record enable|disable|status` – toggle recording mode across `.env` and simulation config
|
|
73
73
|
- `fluxloop doctor` – summarize Python, FluxLoop CLI/MCP, and MCP index state for the active environment
|
|
@@ -88,6 +88,16 @@ These flags override the values in `configs/simulation.yaml` (`multi_turn` block
|
|
|
88
88
|
|
|
89
89
|
Run `fluxloop --help` or `fluxloop <command> --help` for more detail.
|
|
90
90
|
|
|
91
|
+
## Evaluation Workflow
|
|
92
|
+
|
|
93
|
+
Evaluation now follows a two-step process so that multi-turn context is preserved:
|
|
94
|
+
|
|
95
|
+
1. `fluxloop run experiment` – produce `trace_summary.jsonl` (and optionally `observations.jsonl`).
|
|
96
|
+
2. `fluxloop parse experiment <experiment_dir>` – generate markdown summaries and a structured artifact at `per_trace_analysis/per_trace.jsonl`.
|
|
97
|
+
3. `fluxloop evaluate experiment <experiment_dir>` – consume the structured file to calculate scores and reports.
|
|
98
|
+
|
|
99
|
+
`fluxloop evaluate` exits early with guidance when the per-trace artifact is missing. If you relocate the file, supply an explicit path with `--per-trace /path/to/per_trace.jsonl`.
|
|
100
|
+
|
|
91
101
|
## Quick Setup Script
|
|
92
102
|
|
|
93
103
|
To prepare a fresh checkout (create `.venv`, install dependencies, and run diagnostics):
|
|
@@ -25,8 +25,8 @@ The legacy `setting.yaml` is still supported, but new projects created with
|
|
|
25
25
|
- `fluxloop init project` – scaffold a new project (configs, `.env`, examples)
|
|
26
26
|
- `fluxloop generate inputs` – produce input variations for the active project
|
|
27
27
|
- `fluxloop run experiment` – execute an experiment using `configs/simulation.yaml`
|
|
28
|
-
- `fluxloop parse experiment` – convert experiment outputs into readable artifacts
|
|
29
|
-
- `fluxloop evaluate experiment` – score experiment outputs using rule-based and LLM evaluators, generate reports with success criteria, analysis, and customizable templates
|
|
28
|
+
- `fluxloop parse experiment` – convert experiment outputs into readable artifacts and emit structured per-trace JSON at `per_trace_analysis/per_trace.jsonl`
|
|
29
|
+
- `fluxloop evaluate experiment` – score experiment outputs using rule-based and LLM evaluators (requires `fluxloop parse` output or `--per-trace`), generate reports with success criteria, analysis, and customizable templates
|
|
30
30
|
- `fluxloop config set-llm` – update LLM provider/model in `configs/input.yaml`
|
|
31
31
|
- `fluxloop record enable|disable|status` – toggle recording mode across `.env` and simulation config
|
|
32
32
|
- `fluxloop doctor` – summarize Python, FluxLoop CLI/MCP, and MCP index state for the active environment
|
|
@@ -47,6 +47,16 @@ These flags override the values in `configs/simulation.yaml` (`multi_turn` block
|
|
|
47
47
|
|
|
48
48
|
Run `fluxloop --help` or `fluxloop <command> --help` for more detail.
|
|
49
49
|
|
|
50
|
+
## Evaluation Workflow
|
|
51
|
+
|
|
52
|
+
Evaluation now follows a two-step process so that multi-turn context is preserved:
|
|
53
|
+
|
|
54
|
+
1. `fluxloop run experiment` – produce `trace_summary.jsonl` (and optionally `observations.jsonl`).
|
|
55
|
+
2. `fluxloop parse experiment <experiment_dir>` – generate markdown summaries and a structured artifact at `per_trace_analysis/per_trace.jsonl`.
|
|
56
|
+
3. `fluxloop evaluate experiment <experiment_dir>` – consume the structured file to calculate scores and reports.
|
|
57
|
+
|
|
58
|
+
`fluxloop evaluate` exits early with guidance when the per-trace artifact is missing. If you relocate the file, supply an explicit path with `--per-trace /path/to/per_trace.jsonl`.
|
|
59
|
+
|
|
50
60
|
## Quick Setup Script
|
|
51
61
|
|
|
52
62
|
To prepare a fresh checkout (create `.venv`, install dependencies, and run diagnostics):
|
|
@@ -81,6 +81,11 @@ def experiment(
|
|
|
81
81
|
file_okay=True,
|
|
82
82
|
dir_okay=False,
|
|
83
83
|
),
|
|
84
|
+
per_trace: Optional[Path] = typer.Option(
|
|
85
|
+
None,
|
|
86
|
+
"--per-trace",
|
|
87
|
+
help="Path to structured per-trace JSONL generated by `fluxloop parse`",
|
|
88
|
+
),
|
|
84
89
|
verbose: bool = typer.Option(
|
|
85
90
|
False,
|
|
86
91
|
"--verbose",
|
|
@@ -129,6 +134,19 @@ def experiment(
|
|
|
129
134
|
resolved_baseline = (Path.cwd() / resolved_baseline).resolve()
|
|
130
135
|
baseline_path = resolved_baseline
|
|
131
136
|
|
|
137
|
+
if per_trace is not None:
|
|
138
|
+
per_trace_path = per_trace
|
|
139
|
+
if not per_trace_path.is_absolute():
|
|
140
|
+
per_trace_path = (Path.cwd() / per_trace_path).resolve()
|
|
141
|
+
else:
|
|
142
|
+
per_trace_path = resolved_experiment_dir / "per_trace_analysis" / "per_trace.jsonl"
|
|
143
|
+
|
|
144
|
+
if not per_trace_path.exists():
|
|
145
|
+
raise typer.BadParameter(
|
|
146
|
+
f"Structured per-trace artifacts not found at {per_trace_path}. "
|
|
147
|
+
"Run `fluxloop parse` before evaluating or provide --per-trace."
|
|
148
|
+
)
|
|
149
|
+
|
|
132
150
|
try:
|
|
133
151
|
evaluation_config = load_evaluation_config(config_path)
|
|
134
152
|
except FileNotFoundError as exc:
|
|
@@ -164,6 +182,7 @@ def experiment(
|
|
|
164
182
|
report_format=report_format, # type: ignore[arg-type]
|
|
165
183
|
report_template=report_template_path,
|
|
166
184
|
baseline_path=baseline_path,
|
|
185
|
+
per_trace_path=per_trace_path,
|
|
167
186
|
)
|
|
168
187
|
|
|
169
188
|
effective_report_format = report_format or evaluation_config.report.output
|
|
@@ -183,6 +202,7 @@ def experiment(
|
|
|
183
202
|
f"⚙️ Config: [magenta]{config_path}[/magenta]",
|
|
184
203
|
f"📁 Output: [green]{output_dir}[/green]",
|
|
185
204
|
f"📝 Report: [yellow]{effective_report_format.upper()}[/yellow] (template: [cyan]{template_display}[/cyan])",
|
|
205
|
+
f"🧵 Per-trace data: [blue]{per_trace_path}[/blue]",
|
|
186
206
|
]
|
|
187
207
|
if baseline_display:
|
|
188
208
|
message_lines.append(f"📈 Baseline: [cyan]{baseline_display}[/cyan]")
|
|
@@ -7,7 +7,7 @@ from collections import defaultdict
|
|
|
7
7
|
from dataclasses import dataclass
|
|
8
8
|
from datetime import datetime
|
|
9
9
|
from pathlib import Path
|
|
10
|
-
from typing import Dict, Iterable, List, Literal, Optional
|
|
10
|
+
from typing import Any, Dict, Iterable, List, Literal, Optional
|
|
11
11
|
|
|
12
12
|
import typer
|
|
13
13
|
from rich.console import Console
|
|
@@ -44,6 +44,24 @@ class Observation:
|
|
|
44
44
|
except ValueError:
|
|
45
45
|
return None
|
|
46
46
|
|
|
47
|
+
def to_payload(self) -> dict:
|
|
48
|
+
"""Return a JSON-serialisable representation of the observation."""
|
|
49
|
+
|
|
50
|
+
payload = {
|
|
51
|
+
"trace_id": self.trace_id,
|
|
52
|
+
"type": self.type,
|
|
53
|
+
"name": self.name,
|
|
54
|
+
"start_time": self.start_time,
|
|
55
|
+
"end_time": self.end_time,
|
|
56
|
+
"level": self.level,
|
|
57
|
+
"duration_ms": self.duration_ms,
|
|
58
|
+
"input": self.input,
|
|
59
|
+
"output": self.output,
|
|
60
|
+
}
|
|
61
|
+
# Preserve original record for downstream consumers that need full context.
|
|
62
|
+
payload["raw"] = self.raw
|
|
63
|
+
return payload
|
|
64
|
+
|
|
47
65
|
|
|
48
66
|
@dataclass
|
|
49
67
|
class TraceSummary:
|
|
@@ -58,6 +76,20 @@ class TraceSummary:
|
|
|
58
76
|
success: bool
|
|
59
77
|
raw: dict
|
|
60
78
|
|
|
79
|
+
def to_payload(self) -> dict:
|
|
80
|
+
"""Return a JSON-serialisable representation of the summary entry."""
|
|
81
|
+
|
|
82
|
+
return {
|
|
83
|
+
"trace_id": self.trace_id,
|
|
84
|
+
"iteration": self.iteration,
|
|
85
|
+
"persona": self.persona,
|
|
86
|
+
"input": self.input_text,
|
|
87
|
+
"output": self.output_text,
|
|
88
|
+
"duration_ms": self.duration_ms,
|
|
89
|
+
"success": self.success,
|
|
90
|
+
"raw": self.raw,
|
|
91
|
+
}
|
|
92
|
+
|
|
61
93
|
|
|
62
94
|
def _load_observations(path: Path) -> Dict[str, List[Observation]]:
|
|
63
95
|
"""Load observations grouped by trace_id."""
|
|
@@ -155,10 +187,7 @@ def _format_markdown(
|
|
|
155
187
|
) -> str:
|
|
156
188
|
"""Create markdown visualization for a single trace."""
|
|
157
189
|
|
|
158
|
-
observations_sorted =
|
|
159
|
-
observations,
|
|
160
|
-
key=lambda obs: (obs.start_time or "", obs.end_time or ""),
|
|
161
|
-
)
|
|
190
|
+
observations_sorted = _sort_observations(observations)
|
|
162
191
|
|
|
163
192
|
header = (
|
|
164
193
|
"---\n"
|
|
@@ -212,6 +241,65 @@ def _format_markdown(
|
|
|
212
241
|
return header + summary_section + "".join(timeline_lines)
|
|
213
242
|
|
|
214
243
|
|
|
244
|
+
def _sort_observations(
|
|
245
|
+
observations: List[Observation],
|
|
246
|
+
) -> List[Observation]:
|
|
247
|
+
"""Return observations sorted for timeline display."""
|
|
248
|
+
|
|
249
|
+
return sorted(
|
|
250
|
+
observations,
|
|
251
|
+
key=lambda obs: (obs.start_time or "", obs.end_time or ""),
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def _build_timeline_payload(observations: List[Observation]) -> List[dict]:
|
|
256
|
+
"""Convert observations to structured timeline entries."""
|
|
257
|
+
|
|
258
|
+
return [item.to_payload() for item in _sort_observations(observations)]
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def _build_structured_record(
|
|
262
|
+
trace: TraceSummary,
|
|
263
|
+
observations: List[Observation],
|
|
264
|
+
) -> Dict[str, Any]:
|
|
265
|
+
"""Build structured per-trace data for downstream analysis."""
|
|
266
|
+
|
|
267
|
+
timeline = _build_timeline_payload(observations)
|
|
268
|
+
summary_payload = trace.to_payload()
|
|
269
|
+
|
|
270
|
+
record: Dict[str, Any] = {
|
|
271
|
+
"trace_id": summary_payload["trace_id"],
|
|
272
|
+
"iteration": summary_payload["iteration"],
|
|
273
|
+
"persona": summary_payload["persona"],
|
|
274
|
+
"input": summary_payload["input"],
|
|
275
|
+
"output": summary_payload["output"],
|
|
276
|
+
"final_output": summary_payload["output"],
|
|
277
|
+
"duration_ms": summary_payload["duration_ms"],
|
|
278
|
+
"success": summary_payload["success"],
|
|
279
|
+
"summary": summary_payload,
|
|
280
|
+
"timeline": timeline,
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
record["metrics"] = {
|
|
284
|
+
"observation_count": len(timeline),
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
return record
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def _write_structured_records(
|
|
291
|
+
output_dir: Path,
|
|
292
|
+
records: Iterable[Dict[str, Any]],
|
|
293
|
+
) -> Path:
|
|
294
|
+
"""Persist structured per-trace records as JSONL."""
|
|
295
|
+
|
|
296
|
+
target = output_dir / "per_trace.jsonl"
|
|
297
|
+
with target.open("w", encoding="utf-8") as handle:
|
|
298
|
+
for record in records:
|
|
299
|
+
handle.write(json.dumps(record, ensure_ascii=False) + "\n")
|
|
300
|
+
return target
|
|
301
|
+
|
|
302
|
+
|
|
215
303
|
def _slugify(name: str) -> str:
|
|
216
304
|
"""Create a filesystem-safe slug from a trace identifier."""
|
|
217
305
|
|
|
@@ -268,7 +356,13 @@ def experiment(
|
|
|
268
356
|
observations_path = experiment_dir / "observations.jsonl"
|
|
269
357
|
trace_summary_path = experiment_dir / "trace_summary.jsonl"
|
|
270
358
|
|
|
271
|
-
|
|
359
|
+
try:
|
|
360
|
+
observations = _load_observations(observations_path)
|
|
361
|
+
except FileNotFoundError:
|
|
362
|
+
console.print(
|
|
363
|
+
"[yellow]No observations.jsonl found. Structured timelines will omit observation data.[/yellow]"
|
|
364
|
+
)
|
|
365
|
+
observations = defaultdict(list)
|
|
272
366
|
summaries = list(_load_trace_summaries(trace_summary_path))
|
|
273
367
|
|
|
274
368
|
if not summaries:
|
|
@@ -276,9 +370,11 @@ def experiment(
|
|
|
276
370
|
raise typer.Exit(0)
|
|
277
371
|
|
|
278
372
|
console.print(
|
|
279
|
-
f"📝 Found {len(summaries)} trace summaries. Generating markdown..."
|
|
373
|
+
f"📝 Found {len(summaries)} trace summaries. Generating markdown and structured data..."
|
|
280
374
|
)
|
|
281
375
|
|
|
376
|
+
structured_records: List[Dict[str, Any]] = []
|
|
377
|
+
|
|
282
378
|
for summary in summaries:
|
|
283
379
|
trace_observations = observations.get(summary.trace_id, [])
|
|
284
380
|
content = _format_markdown(summary, trace_observations)
|
|
@@ -286,8 +382,13 @@ def experiment(
|
|
|
286
382
|
target_path = output_dir / file_name
|
|
287
383
|
target_path.write_text(content, encoding="utf-8")
|
|
288
384
|
|
|
385
|
+
structured_records.append(_build_structured_record(summary, trace_observations))
|
|
386
|
+
|
|
387
|
+
jsonl_path = _write_structured_records(output_dir, structured_records)
|
|
388
|
+
|
|
289
389
|
console.print(
|
|
290
390
|
f"✅ Generated {len(summaries)} files in: [green]{output_dir.resolve()}[/green]"
|
|
291
391
|
)
|
|
392
|
+
console.print(f"🗃️ Structured timeline saved to: [green]{jsonl_path}[/green]")
|
|
292
393
|
|
|
293
394
|
|
|
@@ -264,15 +264,62 @@ def experiment(
|
|
|
264
264
|
f"Running {config.name}",
|
|
265
265
|
total=total_runs,
|
|
266
266
|
)
|
|
267
|
+
multi_turn_total = (
|
|
268
|
+
config.multi_turn.max_turns
|
|
269
|
+
if config.multi_turn and config.multi_turn.max_turns
|
|
270
|
+
else None
|
|
271
|
+
)
|
|
272
|
+
turn_task = progress.add_task(
|
|
273
|
+
"[cyan]Turn 0/0",
|
|
274
|
+
total=multi_turn_total or 1,
|
|
275
|
+
visible=False,
|
|
276
|
+
)
|
|
267
277
|
|
|
268
278
|
def _progress_callback():
|
|
269
279
|
progress.advance(main_task)
|
|
270
280
|
|
|
281
|
+
def _turn_progress_callback(
|
|
282
|
+
current_turn: int,
|
|
283
|
+
total_turns: int,
|
|
284
|
+
message: Optional[str] = None,
|
|
285
|
+
) -> None:
|
|
286
|
+
total = total_turns or 1
|
|
287
|
+
clamped_turn = max(0, current_turn)
|
|
288
|
+
if message is None:
|
|
289
|
+
completed = min(clamped_turn, total)
|
|
290
|
+
description = f"[cyan]Turn {min(clamped_turn, total)}/{total}: complete"
|
|
291
|
+
progress.update(
|
|
292
|
+
turn_task,
|
|
293
|
+
total=total,
|
|
294
|
+
completed=completed,
|
|
295
|
+
description=description,
|
|
296
|
+
visible=False,
|
|
297
|
+
)
|
|
298
|
+
return
|
|
299
|
+
|
|
300
|
+
if not progress.tasks[turn_task].visible:
|
|
301
|
+
progress.update(turn_task, visible=True)
|
|
302
|
+
|
|
303
|
+
preview = message.replace("\n", " ")
|
|
304
|
+
if len(preview) > 40:
|
|
305
|
+
preview = preview[:37] + "..."
|
|
306
|
+
|
|
307
|
+
completed = max(0, min(clamped_turn - 1, total))
|
|
308
|
+
description = f"[cyan]Turn {min(clamped_turn, total)}/{total}: {preview}"
|
|
309
|
+
|
|
310
|
+
progress.update(
|
|
311
|
+
turn_task,
|
|
312
|
+
total=total,
|
|
313
|
+
completed=completed,
|
|
314
|
+
description=description,
|
|
315
|
+
)
|
|
316
|
+
|
|
271
317
|
# Run experiment
|
|
272
318
|
try:
|
|
273
319
|
results = asyncio.run(
|
|
274
320
|
runner.run_experiment(
|
|
275
|
-
progress_callback=_progress_callback
|
|
321
|
+
progress_callback=_progress_callback,
|
|
322
|
+
turn_progress_callback=_turn_progress_callback,
|
|
276
323
|
)
|
|
277
324
|
)
|
|
278
325
|
except KeyboardInterrupt:
|
|
@@ -37,13 +37,31 @@ class SupervisorDecision:
|
|
|
37
37
|
return self.decision == "continue"
|
|
38
38
|
|
|
39
39
|
|
|
40
|
+
def _coerce_text(value: Any) -> str:
|
|
41
|
+
"""Convert any value into a safe string representation."""
|
|
42
|
+
|
|
43
|
+
if value is None:
|
|
44
|
+
return ""
|
|
45
|
+
|
|
46
|
+
if isinstance(value, str):
|
|
47
|
+
return value
|
|
48
|
+
|
|
49
|
+
if isinstance(value, (dict, list, tuple, set)):
|
|
50
|
+
try:
|
|
51
|
+
return json.dumps(value, ensure_ascii=False)
|
|
52
|
+
except Exception:
|
|
53
|
+
return str(value)
|
|
54
|
+
|
|
55
|
+
return str(value)
|
|
56
|
+
|
|
57
|
+
|
|
40
58
|
def format_transcript(turns: Sequence[Dict[str, Any]]) -> str:
|
|
41
59
|
"""Render a conversation transcript into a bullet list for supervisor prompts."""
|
|
42
60
|
|
|
43
61
|
lines: List[str] = []
|
|
44
62
|
for idx, turn in enumerate(turns, start=1):
|
|
45
63
|
role = turn.get("role", "unknown").capitalize()
|
|
46
|
-
content = turn.get("content")
|
|
64
|
+
content = _coerce_text(turn.get("content"))
|
|
47
65
|
content = content.strip()
|
|
48
66
|
|
|
49
67
|
if turn.get("tool_calls"):
|
|
@@ -53,7 +71,7 @@ def format_transcript(turns: Sequence[Dict[str, Any]]) -> str:
|
|
|
53
71
|
for call in tool_calls:
|
|
54
72
|
name = call.get("name") or call.get("tool") or "tool"
|
|
55
73
|
args = call.get("arguments") or call.get("args") or {}
|
|
56
|
-
tool_summaries.append(f"{name}({
|
|
74
|
+
tool_summaries.append(f"{name}({_coerce_text(args)})")
|
|
57
75
|
if tool_summaries:
|
|
58
76
|
content = f"{content}\n [Tool Calls] " + "; ".join(tool_summaries)
|
|
59
77
|
lines.append(f"{idx}. {role}: {content}")
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Utilities for working with structured per-trace artifacts produced by `fluxloop parse`.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import json
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any, Dict, Iterable, List, Optional
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass
|
|
14
|
+
class PerTraceRecord:
|
|
15
|
+
"""Structured trace artifact consumed by the evaluation pipeline."""
|
|
16
|
+
|
|
17
|
+
data: Dict[str, Any]
|
|
18
|
+
|
|
19
|
+
@property
|
|
20
|
+
def trace(self) -> Dict[str, Any]:
|
|
21
|
+
"""Return the trace payload used for scoring."""
|
|
22
|
+
|
|
23
|
+
return self.data
|
|
24
|
+
|
|
25
|
+
@property
|
|
26
|
+
def trace_id(self) -> Optional[str]:
|
|
27
|
+
return self.data.get("trace_id")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _default_per_trace_path(experiment_dir: Path) -> Path:
|
|
31
|
+
return experiment_dir / "per_trace_analysis" / "per_trace.jsonl"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def load_per_trace_records(
|
|
35
|
+
experiment_dir: Path,
|
|
36
|
+
source_path: Optional[Path] = None,
|
|
37
|
+
) -> List[PerTraceRecord]:
|
|
38
|
+
"""
|
|
39
|
+
Load structured per-trace artifacts from disk.
|
|
40
|
+
|
|
41
|
+
Raises:
|
|
42
|
+
FileNotFoundError: When the per-trace file is missing.
|
|
43
|
+
ValueError: When the file contains invalid JSON or non-object entries.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
path = source_path or _default_per_trace_path(experiment_dir)
|
|
47
|
+
if not path.exists():
|
|
48
|
+
raise FileNotFoundError(
|
|
49
|
+
f"Structured per-trace artifacts not found at {path}. "
|
|
50
|
+
"Run `fluxloop parse` before evaluating."
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
records: List[PerTraceRecord] = []
|
|
54
|
+
with path.open("r", encoding="utf-8") as handle:
|
|
55
|
+
for line_no, line in enumerate(handle, start=1):
|
|
56
|
+
stripped = line.strip()
|
|
57
|
+
if not stripped:
|
|
58
|
+
continue
|
|
59
|
+
try:
|
|
60
|
+
payload = json.loads(stripped)
|
|
61
|
+
except json.JSONDecodeError as exc:
|
|
62
|
+
raise ValueError(f"Invalid JSON in {path} at line {line_no}: {exc}") from exc
|
|
63
|
+
if not isinstance(payload, dict):
|
|
64
|
+
raise ValueError(
|
|
65
|
+
f"Per-trace entry at line {line_no} must be a JSON object, "
|
|
66
|
+
f"but got {type(payload).__name__}"
|
|
67
|
+
)
|
|
68
|
+
records.append(PerTraceRecord(data=payload))
|
|
69
|
+
|
|
70
|
+
return records
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
__all__ = ["PerTraceRecord", "load_per_trace_records"]
|
|
74
|
+
|
|
@@ -21,6 +21,7 @@ from rich.progress import (
|
|
|
21
21
|
TimeRemainingColumn,
|
|
22
22
|
)
|
|
23
23
|
|
|
24
|
+
from ..artifacts import PerTraceRecord, load_per_trace_records
|
|
24
25
|
from ..config import AggregateConfig, EvaluationConfig, EvaluatorConfig
|
|
25
26
|
from ..llm import LLMEvaluationManager, LLMResult
|
|
26
27
|
from ..rules import RuleContext, RuleResult, evaluate_rule
|
|
@@ -44,6 +45,7 @@ class EvaluationOptions:
|
|
|
44
45
|
report_format: Optional[Literal["md", "html", "both"]] = None
|
|
45
46
|
report_template: Optional[Path] = None
|
|
46
47
|
baseline_path: Optional[Path] = None
|
|
48
|
+
per_trace_path: Optional[Path] = None
|
|
47
49
|
|
|
48
50
|
|
|
49
51
|
@dataclass
|
|
@@ -66,26 +68,6 @@ class TraceOutcome:
|
|
|
66
68
|
passed: bool
|
|
67
69
|
|
|
68
70
|
|
|
69
|
-
def _load_trace_summaries(path: Path) -> List[Dict[str, Any]]:
|
|
70
|
-
if not path.exists():
|
|
71
|
-
raise FileNotFoundError(f"trace_summary.jsonl not found: {path}")
|
|
72
|
-
|
|
73
|
-
traces: List[Dict[str, Any]] = []
|
|
74
|
-
with path.open("r", encoding="utf-8") as handle:
|
|
75
|
-
for line_no, line in enumerate(handle, start=1):
|
|
76
|
-
line = line.strip()
|
|
77
|
-
if not line:
|
|
78
|
-
continue
|
|
79
|
-
try:
|
|
80
|
-
payload = json.loads(line)
|
|
81
|
-
except json.JSONDecodeError as exc:
|
|
82
|
-
raise ValueError(
|
|
83
|
-
f"Invalid JSON in trace_summary.jsonl at line {line_no}: {exc}"
|
|
84
|
-
) from exc
|
|
85
|
-
traces.append(payload)
|
|
86
|
-
return traces
|
|
87
|
-
|
|
88
|
-
|
|
89
71
|
def _prepare_output_directory(path: Path, overwrite: bool) -> None:
|
|
90
72
|
if path.exists():
|
|
91
73
|
if not overwrite:
|
|
@@ -309,11 +291,16 @@ def run_evaluation(
|
|
|
309
291
|
if not experiment_dir.is_dir():
|
|
310
292
|
raise NotADirectoryError(f"Experiment directory not found: {experiment_dir}")
|
|
311
293
|
|
|
312
|
-
|
|
313
|
-
|
|
294
|
+
per_trace_records: List[PerTraceRecord] = load_per_trace_records(
|
|
295
|
+
experiment_dir,
|
|
296
|
+
options.per_trace_path,
|
|
297
|
+
)
|
|
298
|
+
traces = [record.trace for record in per_trace_records]
|
|
314
299
|
|
|
315
300
|
if not traces:
|
|
316
|
-
raise ValueError(
|
|
301
|
+
raise ValueError(
|
|
302
|
+
"No per-trace records available. Run `fluxloop parse` before evaluating."
|
|
303
|
+
)
|
|
317
304
|
|
|
318
305
|
_prepare_output_directory(options.output_dir, options.overwrite)
|
|
319
306
|
|
{fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/engine/reporting/html.py
RENAMED
|
@@ -40,6 +40,7 @@ DEFAULT_TEMPLATE = r"""<!DOCTYPE html>
|
|
|
40
40
|
<head>
|
|
41
41
|
<meta charset="utf-8" />
|
|
42
42
|
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
43
|
+
<meta name="report:generated_at" content="[[DATE]]" />
|
|
43
44
|
<title>[[TITLE]]</title>
|
|
44
45
|
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/tailwindcss@2.2.19/dist/tailwind.min.css">
|
|
45
46
|
<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.1/dist/chart.umd.min.js"></script>
|
|
@@ -47,9 +48,19 @@ DEFAULT_TEMPLATE = r"""<!DOCTYPE html>
|
|
|
47
48
|
<body class="bg-slate-950 text-slate-100">
|
|
48
49
|
<main class="max-w-6xl mx-auto p-8 space-y-6">
|
|
49
50
|
<header class="space-y-6">
|
|
50
|
-
<div>
|
|
51
|
-
|
|
52
|
-
|
|
51
|
+
<div class="flex flex-col gap-4 md:flex-row md:items-center md:justify-between">
|
|
52
|
+
<div>
|
|
53
|
+
<h1 class="text-3xl font-bold">[[TITLE]]</h1>
|
|
54
|
+
<p class="text-sm text-slate-300">[[SUBTITLE]]</p>
|
|
55
|
+
</div>
|
|
56
|
+
<a
|
|
57
|
+
href="https://fluxloop.io/"
|
|
58
|
+
class="inline-flex items-center gap-2 self-start rounded-full border border-sky-400 px-4 py-2 text-sm font-semibold text-sky-200 transition hover:bg-sky-500/10 focus:outline-none focus:ring-2 focus:ring-sky-400 focus:ring-offset-2 focus:ring-offset-slate-950"
|
|
59
|
+
target="_blank"
|
|
60
|
+
rel="noopener noreferrer"
|
|
61
|
+
>
|
|
62
|
+
<span>Built with FluxLoop</span>
|
|
63
|
+
</a>
|
|
53
64
|
</div>
|
|
54
65
|
<nav class="flex flex-wrap gap-2" role="tablist" aria-label="Report sections">
|
|
55
66
|
<button class="tab-button px-4 py-2 rounded-full bg-sky-500/20 border border-sky-400 text-sky-200 font-semibold" data-tab-button="summary">Executive Summary</button>
|
|
@@ -62,6 +73,11 @@ DEFAULT_TEMPLATE = r"""<!DOCTYPE html>
|
|
|
62
73
|
<section data-tab-panel="summary" class="tab-panel space-y-6">
|
|
63
74
|
<div id="summaryCards" class="grid gap-4 md:grid-cols-2 xl:grid-cols-4"></div>
|
|
64
75
|
|
|
76
|
+
<section class="space-y-4">
|
|
77
|
+
<h2 class="text-xl font-semibold">Evaluation Goal</h2>
|
|
78
|
+
<div id="evaluationGoal" class="whitespace-pre-wrap rounded-xl border border-slate-800 bg-slate-900 p-4 text-sm text-slate-300"></div>
|
|
79
|
+
</section>
|
|
80
|
+
|
|
65
81
|
<section class="space-y-4">
|
|
66
82
|
<h2 class="text-xl font-semibold">Success Criteria</h2>
|
|
67
83
|
<div id="criteriaList" class="space-y-3"></div>
|
|
@@ -214,6 +230,21 @@ DEFAULT_TEMPLATE = r"""<!DOCTYPE html>
|
|
|
214
230
|
.join("");
|
|
215
231
|
}
|
|
216
232
|
|
|
233
|
+
function renderEvaluationGoal() {
|
|
234
|
+
const container = document.getElementById("evaluationGoal");
|
|
235
|
+
if (!container) return;
|
|
236
|
+
const goal = summary?.evaluation_goal;
|
|
237
|
+
if (!goal) {
|
|
238
|
+
container.textContent = "No evaluation goal provided.";
|
|
239
|
+
container.classList.remove("text-slate-300");
|
|
240
|
+
container.classList.add("text-slate-400");
|
|
241
|
+
return;
|
|
242
|
+
}
|
|
243
|
+
container.classList.remove("text-slate-400");
|
|
244
|
+
container.classList.add("text-slate-300");
|
|
245
|
+
container.textContent = goal;
|
|
246
|
+
}
|
|
247
|
+
|
|
217
248
|
function renderCriteria() {
|
|
218
249
|
const container = document.getElementById("criteriaList");
|
|
219
250
|
if (!container || !criteria || !Object.keys(criteria).length) {
|
|
@@ -458,6 +489,7 @@ DEFAULT_TEMPLATE = r"""<!DOCTYPE html>
|
|
|
458
489
|
|
|
459
490
|
initTabs();
|
|
460
491
|
renderSummaryCards();
|
|
492
|
+
renderEvaluationGoal();
|
|
461
493
|
renderCriteria();
|
|
462
494
|
renderRecommendations();
|
|
463
495
|
renderScoreChart();
|
|
@@ -471,6 +503,45 @@ DEFAULT_TEMPLATE = r"""<!DOCTYPE html>
|
|
|
471
503
|
"""
|
|
472
504
|
|
|
473
505
|
|
|
506
|
+
def _resolve_experiment_label(summary: Dict[str, Any], output_path: Path) -> Optional[str]:
|
|
507
|
+
def _stringify(value: Any) -> Optional[str]:
|
|
508
|
+
if value is None:
|
|
509
|
+
return None
|
|
510
|
+
if isinstance(value, (list, tuple, set)):
|
|
511
|
+
parts = [str(item) for item in value if item not in (None, "")]
|
|
512
|
+
if not parts:
|
|
513
|
+
return None
|
|
514
|
+
return ", ".join(parts)
|
|
515
|
+
if isinstance(value, str):
|
|
516
|
+
return value.strip() or None
|
|
517
|
+
return str(value)
|
|
518
|
+
|
|
519
|
+
for key in ("experiment_id", "experiment_name"):
|
|
520
|
+
candidate = _stringify(summary.get(key))
|
|
521
|
+
if candidate:
|
|
522
|
+
return candidate
|
|
523
|
+
|
|
524
|
+
experiment_meta = summary.get("experiment")
|
|
525
|
+
if isinstance(experiment_meta, dict):
|
|
526
|
+
for key in ("name", "id", "label"):
|
|
527
|
+
candidate = _stringify(experiment_meta.get(key))
|
|
528
|
+
if candidate:
|
|
529
|
+
return candidate
|
|
530
|
+
|
|
531
|
+
output_dir = output_path.parent
|
|
532
|
+
if output_dir != output_path:
|
|
533
|
+
experiment_dir = output_dir.parent
|
|
534
|
+
if experiment_dir and experiment_dir != output_dir:
|
|
535
|
+
candidate = _stringify(experiment_dir.name)
|
|
536
|
+
if candidate:
|
|
537
|
+
return candidate
|
|
538
|
+
candidate = _stringify(output_dir.name)
|
|
539
|
+
if candidate:
|
|
540
|
+
return candidate
|
|
541
|
+
|
|
542
|
+
return None
|
|
543
|
+
|
|
544
|
+
|
|
474
545
|
def select_html_template(options: "EvaluationOptions", config: "EvaluationConfig") -> Tuple[str, Optional[str]]:
|
|
475
546
|
if options.report_template:
|
|
476
547
|
template_text = load_template_from_path(options.report_template)
|
|
@@ -497,9 +568,19 @@ def write_html_report(
|
|
|
497
568
|
success_criteria = summary.get("success_criteria_results") or {}
|
|
498
569
|
analysis = summary.get("analysis") or {}
|
|
499
570
|
|
|
571
|
+
generated_at = datetime.now(UTC)
|
|
572
|
+
generated_iso = generated_at.isoformat(timespec="seconds").replace("+00:00", "Z")
|
|
573
|
+
human_readable = generated_at.strftime("%Y-%m-%d %H:%M:%SZ")
|
|
574
|
+
experiment_label = _resolve_experiment_label(summary, output_path)
|
|
575
|
+
subtitle_parts = [f"Generated on {human_readable}"]
|
|
576
|
+
if experiment_label:
|
|
577
|
+
subtitle_parts.append(f"Experiment: {experiment_label}")
|
|
578
|
+
subtitle = " · ".join(subtitle_parts)
|
|
579
|
+
|
|
500
580
|
replacements = {
|
|
501
|
-
"[[TITLE]]":
|
|
502
|
-
"[[DATE]]":
|
|
581
|
+
"[[TITLE]]": "FluxLoop Evaluation Report",
|
|
582
|
+
"[[DATE]]": generated_iso,
|
|
583
|
+
"[[SUBTITLE]]": subtitle,
|
|
503
584
|
"[[SUMMARY_JSON]]": json.dumps(summary, ensure_ascii=False),
|
|
504
585
|
"[[PER_TRACE_JSON]]": json.dumps(per_trace_payload, ensure_ascii=False),
|
|
505
586
|
"[[CRITERIA_JSON]]": json.dumps(success_criteria, ensure_ascii=False),
|