fluxloop-cli 0.2.25__tar.gz → 0.2.27__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/PKG-INFO +13 -3
  2. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/README.md +12 -2
  3. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/__init__.py +1 -1
  4. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/commands/evaluate.py +20 -0
  5. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/commands/parse.py +108 -7
  6. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/commands/run.py +48 -1
  7. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/conversation_supervisor.py +20 -2
  8. fluxloop_cli-0.2.27/fluxloop_cli/evaluation/artifacts.py +74 -0
  9. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/engine/core.py +10 -23
  10. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/engine/reporting/html.py +86 -5
  11. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/rules.py +167 -0
  12. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/runner.py +38 -2
  13. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/templates.py +4 -4
  14. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli.egg-info/PKG-INFO +13 -3
  15. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli.egg-info/SOURCES.txt +3 -0
  16. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/pyproject.toml +1 -1
  17. fluxloop_cli-0.2.27/tests/test_conversation_supervisor.py +39 -0
  18. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/tests/test_evaluate_command.py +97 -6
  19. fluxloop_cli-0.2.27/tests/test_run_command.py +133 -0
  20. fluxloop_cli-0.2.27/tests/test_runner_multi_turn.py +96 -0
  21. fluxloop_cli-0.2.25/tests/test_run_command.py +0 -72
  22. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/arg_binder.py +0 -0
  23. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/commands/__init__.py +0 -0
  24. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/commands/config.py +0 -0
  25. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/commands/doctor.py +0 -0
  26. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/commands/generate.py +0 -0
  27. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/commands/init.py +0 -0
  28. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/commands/record.py +0 -0
  29. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/commands/status.py +0 -0
  30. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/config_loader.py +0 -0
  31. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/config_schema.py +0 -0
  32. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/constants.py +0 -0
  33. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/environment.py +0 -0
  34. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/__init__.py +0 -0
  35. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/config.py +0 -0
  36. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/engine/__init__.py +0 -0
  37. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/engine/analysis.py +0 -0
  38. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/engine/reporting/__init__.py +0 -0
  39. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/engine/reporting/markdown.py +0 -0
  40. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/engine/success.py +0 -0
  41. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/llm.py +0 -0
  42. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/prompts/__init__.py +0 -0
  43. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/prompts/base.py +0 -0
  44. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/prompts/information_completeness.py +0 -0
  45. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/prompts/intent_recognition.py +0 -0
  46. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/prompts/response_clarity.py +0 -0
  47. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/evaluation/prompts/response_consistency.py +0 -0
  48. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/input_generator.py +0 -0
  49. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/llm_generator.py +0 -0
  50. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/main.py +0 -0
  51. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/project_paths.py +0 -0
  52. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/target_loader.py +0 -0
  53. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli/validators.py +0 -0
  54. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli.egg-info/dependency_links.txt +0 -0
  55. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli.egg-info/entry_points.txt +0 -0
  56. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli.egg-info/requires.txt +0 -0
  57. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/fluxloop_cli.egg-info/top_level.txt +0 -0
  58. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/setup.cfg +0 -0
  59. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/tests/test_analysis_recommendations.py +0 -0
  60. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/tests/test_arg_binder.py +0 -0
  61. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/tests/test_config_command.py +0 -0
  62. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/tests/test_evaluation_llm.py +0 -0
  63. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/tests/test_input_generator.py +0 -0
  64. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/tests/test_prompt_library.py +0 -0
  65. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/tests/test_success_criteria.py +0 -0
  66. {fluxloop_cli-0.2.25 → fluxloop_cli-0.2.27}/tests/test_target_loader.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: fluxloop-cli
3
- Version: 0.2.25
3
+ Version: 0.2.27
4
4
  Summary: FluxLoop CLI for running agent simulations
5
5
  Author-email: FluxLoop Team <team@fluxloop.dev>
6
6
  License: Apache-2.0
@@ -66,8 +66,8 @@ The legacy `setting.yaml` is still supported, but new projects created with
66
66
  - `fluxloop init project` – scaffold a new project (configs, `.env`, examples)
67
67
  - `fluxloop generate inputs` – produce input variations for the active project
68
68
  - `fluxloop run experiment` – execute an experiment using `configs/simulation.yaml`
69
- - `fluxloop parse experiment` – convert experiment outputs into readable artifacts
70
- - `fluxloop evaluate experiment` – score experiment outputs using rule-based and LLM evaluators, generate reports with success criteria, analysis, and customizable templates
69
+ - `fluxloop parse experiment` – convert experiment outputs into readable artifacts and emit structured per-trace JSON at `per_trace_analysis/per_trace.jsonl`
70
+ - `fluxloop evaluate experiment` – score experiment outputs using rule-based and LLM evaluators (requires `fluxloop parse` output or `--per-trace`), generate reports with success criteria, analysis, and customizable templates
71
71
  - `fluxloop config set-llm` – update LLM provider/model in `configs/input.yaml`
72
72
  - `fluxloop record enable|disable|status` – toggle recording mode across `.env` and simulation config
73
73
  - `fluxloop doctor` – summarize Python, FluxLoop CLI/MCP, and MCP index state for the active environment
@@ -88,6 +88,16 @@ These flags override the values in `configs/simulation.yaml` (`multi_turn` block
88
88
 
89
89
  Run `fluxloop --help` or `fluxloop <command> --help` for more detail.
90
90
 
91
+ ## Evaluation Workflow
92
+
93
+ Evaluation now follows a two-step process so that multi-turn context is preserved:
94
+
95
+ 1. `fluxloop run experiment` – produce `trace_summary.jsonl` (and optionally `observations.jsonl`).
96
+ 2. `fluxloop parse experiment <experiment_dir>` – generate markdown summaries and a structured artifact at `per_trace_analysis/per_trace.jsonl`.
97
+ 3. `fluxloop evaluate experiment <experiment_dir>` – consume the structured file to calculate scores and reports.
98
+
99
+ `fluxloop evaluate` exits early with guidance when the per-trace artifact is missing. If you relocate the file, supply an explicit path with `--per-trace /path/to/per_trace.jsonl`.
100
+
91
101
  ## Quick Setup Script
92
102
 
93
103
  To prepare a fresh checkout (create `.venv`, install dependencies, and run diagnostics):
@@ -25,8 +25,8 @@ The legacy `setting.yaml` is still supported, but new projects created with
25
25
  - `fluxloop init project` – scaffold a new project (configs, `.env`, examples)
26
26
  - `fluxloop generate inputs` – produce input variations for the active project
27
27
  - `fluxloop run experiment` – execute an experiment using `configs/simulation.yaml`
28
- - `fluxloop parse experiment` – convert experiment outputs into readable artifacts
29
- - `fluxloop evaluate experiment` – score experiment outputs using rule-based and LLM evaluators, generate reports with success criteria, analysis, and customizable templates
28
+ - `fluxloop parse experiment` – convert experiment outputs into readable artifacts and emit structured per-trace JSON at `per_trace_analysis/per_trace.jsonl`
29
+ - `fluxloop evaluate experiment` – score experiment outputs using rule-based and LLM evaluators (requires `fluxloop parse` output or `--per-trace`), generate reports with success criteria, analysis, and customizable templates
30
30
  - `fluxloop config set-llm` – update LLM provider/model in `configs/input.yaml`
31
31
  - `fluxloop record enable|disable|status` – toggle recording mode across `.env` and simulation config
32
32
  - `fluxloop doctor` – summarize Python, FluxLoop CLI/MCP, and MCP index state for the active environment
@@ -47,6 +47,16 @@ These flags override the values in `configs/simulation.yaml` (`multi_turn` block
47
47
 
48
48
  Run `fluxloop --help` or `fluxloop <command> --help` for more detail.
49
49
 
50
+ ## Evaluation Workflow
51
+
52
+ Evaluation now follows a two-step process so that multi-turn context is preserved:
53
+
54
+ 1. `fluxloop run experiment` – produce `trace_summary.jsonl` (and optionally `observations.jsonl`).
55
+ 2. `fluxloop parse experiment <experiment_dir>` – generate markdown summaries and a structured artifact at `per_trace_analysis/per_trace.jsonl`.
56
+ 3. `fluxloop evaluate experiment <experiment_dir>` – consume the structured file to calculate scores and reports.
57
+
58
+ `fluxloop evaluate` exits early with guidance when the per-trace artifact is missing. If you relocate the file, supply an explicit path with `--per-trace /path/to/per_trace.jsonl`.
59
+
50
60
  ## Quick Setup Script
51
61
 
52
62
  To prepare a fresh checkout (create `.venv`, install dependencies, and run diagnostics):
@@ -2,7 +2,7 @@
2
2
  FluxLoop CLI - Command-line interface for running agent simulations.
3
3
  """
4
4
 
5
- __version__ = "0.2.25"
5
+ __version__ = "0.2.27"
6
6
 
7
7
  from .main import app
8
8
 
@@ -81,6 +81,11 @@ def experiment(
81
81
  file_okay=True,
82
82
  dir_okay=False,
83
83
  ),
84
+ per_trace: Optional[Path] = typer.Option(
85
+ None,
86
+ "--per-trace",
87
+ help="Path to structured per-trace JSONL generated by `fluxloop parse`",
88
+ ),
84
89
  verbose: bool = typer.Option(
85
90
  False,
86
91
  "--verbose",
@@ -129,6 +134,19 @@ def experiment(
129
134
  resolved_baseline = (Path.cwd() / resolved_baseline).resolve()
130
135
  baseline_path = resolved_baseline
131
136
 
137
+ if per_trace is not None:
138
+ per_trace_path = per_trace
139
+ if not per_trace_path.is_absolute():
140
+ per_trace_path = (Path.cwd() / per_trace_path).resolve()
141
+ else:
142
+ per_trace_path = resolved_experiment_dir / "per_trace_analysis" / "per_trace.jsonl"
143
+
144
+ if not per_trace_path.exists():
145
+ raise typer.BadParameter(
146
+ f"Structured per-trace artifacts not found at {per_trace_path}. "
147
+ "Run `fluxloop parse` before evaluating or provide --per-trace."
148
+ )
149
+
132
150
  try:
133
151
  evaluation_config = load_evaluation_config(config_path)
134
152
  except FileNotFoundError as exc:
@@ -164,6 +182,7 @@ def experiment(
164
182
  report_format=report_format, # type: ignore[arg-type]
165
183
  report_template=report_template_path,
166
184
  baseline_path=baseline_path,
185
+ per_trace_path=per_trace_path,
167
186
  )
168
187
 
169
188
  effective_report_format = report_format or evaluation_config.report.output
@@ -183,6 +202,7 @@ def experiment(
183
202
  f"⚙️ Config: [magenta]{config_path}[/magenta]",
184
203
  f"📁 Output: [green]{output_dir}[/green]",
185
204
  f"📝 Report: [yellow]{effective_report_format.upper()}[/yellow] (template: [cyan]{template_display}[/cyan])",
205
+ f"🧵 Per-trace data: [blue]{per_trace_path}[/blue]",
186
206
  ]
187
207
  if baseline_display:
188
208
  message_lines.append(f"📈 Baseline: [cyan]{baseline_display}[/cyan]")
@@ -7,7 +7,7 @@ from collections import defaultdict
7
7
  from dataclasses import dataclass
8
8
  from datetime import datetime
9
9
  from pathlib import Path
10
- from typing import Dict, Iterable, List, Literal, Optional
10
+ from typing import Any, Dict, Iterable, List, Literal, Optional
11
11
 
12
12
  import typer
13
13
  from rich.console import Console
@@ -44,6 +44,24 @@ class Observation:
44
44
  except ValueError:
45
45
  return None
46
46
 
47
+ def to_payload(self) -> dict:
48
+ """Return a JSON-serialisable representation of the observation."""
49
+
50
+ payload = {
51
+ "trace_id": self.trace_id,
52
+ "type": self.type,
53
+ "name": self.name,
54
+ "start_time": self.start_time,
55
+ "end_time": self.end_time,
56
+ "level": self.level,
57
+ "duration_ms": self.duration_ms,
58
+ "input": self.input,
59
+ "output": self.output,
60
+ }
61
+ # Preserve original record for downstream consumers that need full context.
62
+ payload["raw"] = self.raw
63
+ return payload
64
+
47
65
 
48
66
  @dataclass
49
67
  class TraceSummary:
@@ -58,6 +76,20 @@ class TraceSummary:
58
76
  success: bool
59
77
  raw: dict
60
78
 
79
+ def to_payload(self) -> dict:
80
+ """Return a JSON-serialisable representation of the summary entry."""
81
+
82
+ return {
83
+ "trace_id": self.trace_id,
84
+ "iteration": self.iteration,
85
+ "persona": self.persona,
86
+ "input": self.input_text,
87
+ "output": self.output_text,
88
+ "duration_ms": self.duration_ms,
89
+ "success": self.success,
90
+ "raw": self.raw,
91
+ }
92
+
61
93
 
62
94
  def _load_observations(path: Path) -> Dict[str, List[Observation]]:
63
95
  """Load observations grouped by trace_id."""
@@ -155,10 +187,7 @@ def _format_markdown(
155
187
  ) -> str:
156
188
  """Create markdown visualization for a single trace."""
157
189
 
158
- observations_sorted = sorted(
159
- observations,
160
- key=lambda obs: (obs.start_time or "", obs.end_time or ""),
161
- )
190
+ observations_sorted = _sort_observations(observations)
162
191
 
163
192
  header = (
164
193
  "---\n"
@@ -212,6 +241,65 @@ def _format_markdown(
212
241
  return header + summary_section + "".join(timeline_lines)
213
242
 
214
243
 
244
+ def _sort_observations(
245
+ observations: List[Observation],
246
+ ) -> List[Observation]:
247
+ """Return observations sorted for timeline display."""
248
+
249
+ return sorted(
250
+ observations,
251
+ key=lambda obs: (obs.start_time or "", obs.end_time or ""),
252
+ )
253
+
254
+
255
+ def _build_timeline_payload(observations: List[Observation]) -> List[dict]:
256
+ """Convert observations to structured timeline entries."""
257
+
258
+ return [item.to_payload() for item in _sort_observations(observations)]
259
+
260
+
261
+ def _build_structured_record(
262
+ trace: TraceSummary,
263
+ observations: List[Observation],
264
+ ) -> Dict[str, Any]:
265
+ """Build structured per-trace data for downstream analysis."""
266
+
267
+ timeline = _build_timeline_payload(observations)
268
+ summary_payload = trace.to_payload()
269
+
270
+ record: Dict[str, Any] = {
271
+ "trace_id": summary_payload["trace_id"],
272
+ "iteration": summary_payload["iteration"],
273
+ "persona": summary_payload["persona"],
274
+ "input": summary_payload["input"],
275
+ "output": summary_payload["output"],
276
+ "final_output": summary_payload["output"],
277
+ "duration_ms": summary_payload["duration_ms"],
278
+ "success": summary_payload["success"],
279
+ "summary": summary_payload,
280
+ "timeline": timeline,
281
+ }
282
+
283
+ record["metrics"] = {
284
+ "observation_count": len(timeline),
285
+ }
286
+
287
+ return record
288
+
289
+
290
+ def _write_structured_records(
291
+ output_dir: Path,
292
+ records: Iterable[Dict[str, Any]],
293
+ ) -> Path:
294
+ """Persist structured per-trace records as JSONL."""
295
+
296
+ target = output_dir / "per_trace.jsonl"
297
+ with target.open("w", encoding="utf-8") as handle:
298
+ for record in records:
299
+ handle.write(json.dumps(record, ensure_ascii=False) + "\n")
300
+ return target
301
+
302
+
215
303
  def _slugify(name: str) -> str:
216
304
  """Create a filesystem-safe slug from a trace identifier."""
217
305
 
@@ -268,7 +356,13 @@ def experiment(
268
356
  observations_path = experiment_dir / "observations.jsonl"
269
357
  trace_summary_path = experiment_dir / "trace_summary.jsonl"
270
358
 
271
- observations = _load_observations(observations_path)
359
+ try:
360
+ observations = _load_observations(observations_path)
361
+ except FileNotFoundError:
362
+ console.print(
363
+ "[yellow]No observations.jsonl found. Structured timelines will omit observation data.[/yellow]"
364
+ )
365
+ observations = defaultdict(list)
272
366
  summaries = list(_load_trace_summaries(trace_summary_path))
273
367
 
274
368
  if not summaries:
@@ -276,9 +370,11 @@ def experiment(
276
370
  raise typer.Exit(0)
277
371
 
278
372
  console.print(
279
- f"📝 Found {len(summaries)} trace summaries. Generating markdown..."
373
+ f"📝 Found {len(summaries)} trace summaries. Generating markdown and structured data..."
280
374
  )
281
375
 
376
+ structured_records: List[Dict[str, Any]] = []
377
+
282
378
  for summary in summaries:
283
379
  trace_observations = observations.get(summary.trace_id, [])
284
380
  content = _format_markdown(summary, trace_observations)
@@ -286,8 +382,13 @@ def experiment(
286
382
  target_path = output_dir / file_name
287
383
  target_path.write_text(content, encoding="utf-8")
288
384
 
385
+ structured_records.append(_build_structured_record(summary, trace_observations))
386
+
387
+ jsonl_path = _write_structured_records(output_dir, structured_records)
388
+
289
389
  console.print(
290
390
  f"✅ Generated {len(summaries)} files in: [green]{output_dir.resolve()}[/green]"
291
391
  )
392
+ console.print(f"🗃️ Structured timeline saved to: [green]{jsonl_path}[/green]")
292
393
 
293
394
 
@@ -264,15 +264,62 @@ def experiment(
264
264
  f"Running {config.name}",
265
265
  total=total_runs,
266
266
  )
267
+ multi_turn_total = (
268
+ config.multi_turn.max_turns
269
+ if config.multi_turn and config.multi_turn.max_turns
270
+ else None
271
+ )
272
+ turn_task = progress.add_task(
273
+ "[cyan]Turn 0/0",
274
+ total=multi_turn_total or 1,
275
+ visible=False,
276
+ )
267
277
 
268
278
  def _progress_callback():
269
279
  progress.advance(main_task)
270
280
 
281
+ def _turn_progress_callback(
282
+ current_turn: int,
283
+ total_turns: int,
284
+ message: Optional[str] = None,
285
+ ) -> None:
286
+ total = total_turns or 1
287
+ clamped_turn = max(0, current_turn)
288
+ if message is None:
289
+ completed = min(clamped_turn, total)
290
+ description = f"[cyan]Turn {min(clamped_turn, total)}/{total}: complete"
291
+ progress.update(
292
+ turn_task,
293
+ total=total,
294
+ completed=completed,
295
+ description=description,
296
+ visible=False,
297
+ )
298
+ return
299
+
300
+ if not progress.tasks[turn_task].visible:
301
+ progress.update(turn_task, visible=True)
302
+
303
+ preview = message.replace("\n", " ")
304
+ if len(preview) > 40:
305
+ preview = preview[:37] + "..."
306
+
307
+ completed = max(0, min(clamped_turn - 1, total))
308
+ description = f"[cyan]Turn {min(clamped_turn, total)}/{total}: {preview}"
309
+
310
+ progress.update(
311
+ turn_task,
312
+ total=total,
313
+ completed=completed,
314
+ description=description,
315
+ )
316
+
271
317
  # Run experiment
272
318
  try:
273
319
  results = asyncio.run(
274
320
  runner.run_experiment(
275
- progress_callback=_progress_callback
321
+ progress_callback=_progress_callback,
322
+ turn_progress_callback=_turn_progress_callback,
276
323
  )
277
324
  )
278
325
  except KeyboardInterrupt:
@@ -37,13 +37,31 @@ class SupervisorDecision:
37
37
  return self.decision == "continue"
38
38
 
39
39
 
40
+ def _coerce_text(value: Any) -> str:
41
+ """Convert any value into a safe string representation."""
42
+
43
+ if value is None:
44
+ return ""
45
+
46
+ if isinstance(value, str):
47
+ return value
48
+
49
+ if isinstance(value, (dict, list, tuple, set)):
50
+ try:
51
+ return json.dumps(value, ensure_ascii=False)
52
+ except Exception:
53
+ return str(value)
54
+
55
+ return str(value)
56
+
57
+
40
58
  def format_transcript(turns: Sequence[Dict[str, Any]]) -> str:
41
59
  """Render a conversation transcript into a bullet list for supervisor prompts."""
42
60
 
43
61
  lines: List[str] = []
44
62
  for idx, turn in enumerate(turns, start=1):
45
63
  role = turn.get("role", "unknown").capitalize()
46
- content = turn.get("content") or ""
64
+ content = _coerce_text(turn.get("content"))
47
65
  content = content.strip()
48
66
 
49
67
  if turn.get("tool_calls"):
@@ -53,7 +71,7 @@ def format_transcript(turns: Sequence[Dict[str, Any]]) -> str:
53
71
  for call in tool_calls:
54
72
  name = call.get("name") or call.get("tool") or "tool"
55
73
  args = call.get("arguments") or call.get("args") or {}
56
- tool_summaries.append(f"{name}({json.dumps(args, ensure_ascii=False)})")
74
+ tool_summaries.append(f"{name}({_coerce_text(args)})")
57
75
  if tool_summaries:
58
76
  content = f"{content}\n [Tool Calls] " + "; ".join(tool_summaries)
59
77
  lines.append(f"{idx}. {role}: {content}")
@@ -0,0 +1,74 @@
1
+ """
2
+ Utilities for working with structured per-trace artifacts produced by `fluxloop parse`.
3
+ """
4
+
5
+ from __future__ import annotations
6
+
7
+ import json
8
+ from dataclasses import dataclass
9
+ from pathlib import Path
10
+ from typing import Any, Dict, Iterable, List, Optional
11
+
12
+
13
+ @dataclass
14
+ class PerTraceRecord:
15
+ """Structured trace artifact consumed by the evaluation pipeline."""
16
+
17
+ data: Dict[str, Any]
18
+
19
+ @property
20
+ def trace(self) -> Dict[str, Any]:
21
+ """Return the trace payload used for scoring."""
22
+
23
+ return self.data
24
+
25
+ @property
26
+ def trace_id(self) -> Optional[str]:
27
+ return self.data.get("trace_id")
28
+
29
+
30
+ def _default_per_trace_path(experiment_dir: Path) -> Path:
31
+ return experiment_dir / "per_trace_analysis" / "per_trace.jsonl"
32
+
33
+
34
+ def load_per_trace_records(
35
+ experiment_dir: Path,
36
+ source_path: Optional[Path] = None,
37
+ ) -> List[PerTraceRecord]:
38
+ """
39
+ Load structured per-trace artifacts from disk.
40
+
41
+ Raises:
42
+ FileNotFoundError: When the per-trace file is missing.
43
+ ValueError: When the file contains invalid JSON or non-object entries.
44
+ """
45
+
46
+ path = source_path or _default_per_trace_path(experiment_dir)
47
+ if not path.exists():
48
+ raise FileNotFoundError(
49
+ f"Structured per-trace artifacts not found at {path}. "
50
+ "Run `fluxloop parse` before evaluating."
51
+ )
52
+
53
+ records: List[PerTraceRecord] = []
54
+ with path.open("r", encoding="utf-8") as handle:
55
+ for line_no, line in enumerate(handle, start=1):
56
+ stripped = line.strip()
57
+ if not stripped:
58
+ continue
59
+ try:
60
+ payload = json.loads(stripped)
61
+ except json.JSONDecodeError as exc:
62
+ raise ValueError(f"Invalid JSON in {path} at line {line_no}: {exc}") from exc
63
+ if not isinstance(payload, dict):
64
+ raise ValueError(
65
+ f"Per-trace entry at line {line_no} must be a JSON object, "
66
+ f"but got {type(payload).__name__}"
67
+ )
68
+ records.append(PerTraceRecord(data=payload))
69
+
70
+ return records
71
+
72
+
73
+ __all__ = ["PerTraceRecord", "load_per_trace_records"]
74
+
@@ -21,6 +21,7 @@ from rich.progress import (
21
21
  TimeRemainingColumn,
22
22
  )
23
23
 
24
+ from ..artifacts import PerTraceRecord, load_per_trace_records
24
25
  from ..config import AggregateConfig, EvaluationConfig, EvaluatorConfig
25
26
  from ..llm import LLMEvaluationManager, LLMResult
26
27
  from ..rules import RuleContext, RuleResult, evaluate_rule
@@ -44,6 +45,7 @@ class EvaluationOptions:
44
45
  report_format: Optional[Literal["md", "html", "both"]] = None
45
46
  report_template: Optional[Path] = None
46
47
  baseline_path: Optional[Path] = None
48
+ per_trace_path: Optional[Path] = None
47
49
 
48
50
 
49
51
  @dataclass
@@ -66,26 +68,6 @@ class TraceOutcome:
66
68
  passed: bool
67
69
 
68
70
 
69
- def _load_trace_summaries(path: Path) -> List[Dict[str, Any]]:
70
- if not path.exists():
71
- raise FileNotFoundError(f"trace_summary.jsonl not found: {path}")
72
-
73
- traces: List[Dict[str, Any]] = []
74
- with path.open("r", encoding="utf-8") as handle:
75
- for line_no, line in enumerate(handle, start=1):
76
- line = line.strip()
77
- if not line:
78
- continue
79
- try:
80
- payload = json.loads(line)
81
- except json.JSONDecodeError as exc:
82
- raise ValueError(
83
- f"Invalid JSON in trace_summary.jsonl at line {line_no}: {exc}"
84
- ) from exc
85
- traces.append(payload)
86
- return traces
87
-
88
-
89
71
  def _prepare_output_directory(path: Path, overwrite: bool) -> None:
90
72
  if path.exists():
91
73
  if not overwrite:
@@ -309,11 +291,16 @@ def run_evaluation(
309
291
  if not experiment_dir.is_dir():
310
292
  raise NotADirectoryError(f"Experiment directory not found: {experiment_dir}")
311
293
 
312
- trace_summary_path = experiment_dir / "trace_summary.jsonl"
313
- traces = _load_trace_summaries(trace_summary_path)
294
+ per_trace_records: List[PerTraceRecord] = load_per_trace_records(
295
+ experiment_dir,
296
+ options.per_trace_path,
297
+ )
298
+ traces = [record.trace for record in per_trace_records]
314
299
 
315
300
  if not traces:
316
- raise ValueError("No trace summaries found. Run an experiment before evaluating.")
301
+ raise ValueError(
302
+ "No per-trace records available. Run `fluxloop parse` before evaluating."
303
+ )
317
304
 
318
305
  _prepare_output_directory(options.output_dir, options.overwrite)
319
306
 
@@ -40,6 +40,7 @@ DEFAULT_TEMPLATE = r"""<!DOCTYPE html>
40
40
  <head>
41
41
  <meta charset="utf-8" />
42
42
  <meta name="viewport" content="width=device-width, initial-scale=1" />
43
+ <meta name="report:generated_at" content="[[DATE]]" />
43
44
  <title>[[TITLE]]</title>
44
45
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/tailwindcss@2.2.19/dist/tailwind.min.css">
45
46
  <script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.1/dist/chart.umd.min.js"></script>
@@ -47,9 +48,19 @@ DEFAULT_TEMPLATE = r"""<!DOCTYPE html>
47
48
  <body class="bg-slate-950 text-slate-100">
48
49
  <main class="max-w-6xl mx-auto p-8 space-y-6">
49
50
  <header class="space-y-6">
50
- <div>
51
- <h1 class="text-3xl font-bold">[[TITLE]]</h1>
52
- <p class="text-sm text-slate-300">Generated at [[DATE]]</p>
51
+ <div class="flex flex-col gap-4 md:flex-row md:items-center md:justify-between">
52
+ <div>
53
+ <h1 class="text-3xl font-bold">[[TITLE]]</h1>
54
+ <p class="text-sm text-slate-300">[[SUBTITLE]]</p>
55
+ </div>
56
+ <a
57
+ href="https://fluxloop.io/"
58
+ class="inline-flex items-center gap-2 self-start rounded-full border border-sky-400 px-4 py-2 text-sm font-semibold text-sky-200 transition hover:bg-sky-500/10 focus:outline-none focus:ring-2 focus:ring-sky-400 focus:ring-offset-2 focus:ring-offset-slate-950"
59
+ target="_blank"
60
+ rel="noopener noreferrer"
61
+ >
62
+ <span>Built with FluxLoop</span>
63
+ </a>
53
64
  </div>
54
65
  <nav class="flex flex-wrap gap-2" role="tablist" aria-label="Report sections">
55
66
  <button class="tab-button px-4 py-2 rounded-full bg-sky-500/20 border border-sky-400 text-sky-200 font-semibold" data-tab-button="summary">Executive Summary</button>
@@ -62,6 +73,11 @@ DEFAULT_TEMPLATE = r"""<!DOCTYPE html>
62
73
  <section data-tab-panel="summary" class="tab-panel space-y-6">
63
74
  <div id="summaryCards" class="grid gap-4 md:grid-cols-2 xl:grid-cols-4"></div>
64
75
 
76
+ <section class="space-y-4">
77
+ <h2 class="text-xl font-semibold">Evaluation Goal</h2>
78
+ <div id="evaluationGoal" class="whitespace-pre-wrap rounded-xl border border-slate-800 bg-slate-900 p-4 text-sm text-slate-300"></div>
79
+ </section>
80
+
65
81
  <section class="space-y-4">
66
82
  <h2 class="text-xl font-semibold">Success Criteria</h2>
67
83
  <div id="criteriaList" class="space-y-3"></div>
@@ -214,6 +230,21 @@ DEFAULT_TEMPLATE = r"""<!DOCTYPE html>
214
230
  .join("");
215
231
  }
216
232
 
233
+ function renderEvaluationGoal() {
234
+ const container = document.getElementById("evaluationGoal");
235
+ if (!container) return;
236
+ const goal = summary?.evaluation_goal;
237
+ if (!goal) {
238
+ container.textContent = "No evaluation goal provided.";
239
+ container.classList.remove("text-slate-300");
240
+ container.classList.add("text-slate-400");
241
+ return;
242
+ }
243
+ container.classList.remove("text-slate-400");
244
+ container.classList.add("text-slate-300");
245
+ container.textContent = goal;
246
+ }
247
+
217
248
  function renderCriteria() {
218
249
  const container = document.getElementById("criteriaList");
219
250
  if (!container || !criteria || !Object.keys(criteria).length) {
@@ -458,6 +489,7 @@ DEFAULT_TEMPLATE = r"""<!DOCTYPE html>
458
489
 
459
490
  initTabs();
460
491
  renderSummaryCards();
492
+ renderEvaluationGoal();
461
493
  renderCriteria();
462
494
  renderRecommendations();
463
495
  renderScoreChart();
@@ -471,6 +503,45 @@ DEFAULT_TEMPLATE = r"""<!DOCTYPE html>
471
503
  """
472
504
 
473
505
 
506
+ def _resolve_experiment_label(summary: Dict[str, Any], output_path: Path) -> Optional[str]:
507
+ def _stringify(value: Any) -> Optional[str]:
508
+ if value is None:
509
+ return None
510
+ if isinstance(value, (list, tuple, set)):
511
+ parts = [str(item) for item in value if item not in (None, "")]
512
+ if not parts:
513
+ return None
514
+ return ", ".join(parts)
515
+ if isinstance(value, str):
516
+ return value.strip() or None
517
+ return str(value)
518
+
519
+ for key in ("experiment_id", "experiment_name"):
520
+ candidate = _stringify(summary.get(key))
521
+ if candidate:
522
+ return candidate
523
+
524
+ experiment_meta = summary.get("experiment")
525
+ if isinstance(experiment_meta, dict):
526
+ for key in ("name", "id", "label"):
527
+ candidate = _stringify(experiment_meta.get(key))
528
+ if candidate:
529
+ return candidate
530
+
531
+ output_dir = output_path.parent
532
+ if output_dir != output_path:
533
+ experiment_dir = output_dir.parent
534
+ if experiment_dir and experiment_dir != output_dir:
535
+ candidate = _stringify(experiment_dir.name)
536
+ if candidate:
537
+ return candidate
538
+ candidate = _stringify(output_dir.name)
539
+ if candidate:
540
+ return candidate
541
+
542
+ return None
543
+
544
+
474
545
  def select_html_template(options: "EvaluationOptions", config: "EvaluationConfig") -> Tuple[str, Optional[str]]:
475
546
  if options.report_template:
476
547
  template_text = load_template_from_path(options.report_template)
@@ -497,9 +568,19 @@ def write_html_report(
497
568
  success_criteria = summary.get("success_criteria_results") or {}
498
569
  analysis = summary.get("analysis") or {}
499
570
 
571
+ generated_at = datetime.now(UTC)
572
+ generated_iso = generated_at.isoformat(timespec="seconds").replace("+00:00", "Z")
573
+ human_readable = generated_at.strftime("%Y-%m-%d %H:%M:%SZ")
574
+ experiment_label = _resolve_experiment_label(summary, output_path)
575
+ subtitle_parts = [f"Generated on {human_readable}"]
576
+ if experiment_label:
577
+ subtitle_parts.append(f"Experiment: {experiment_label}")
578
+ subtitle = " · ".join(subtitle_parts)
579
+
500
580
  replacements = {
501
- "[[TITLE]]": summary.get("evaluation_goal") or "FluxLoop Evaluation Report",
502
- "[[DATE]]": datetime.now(UTC).isoformat(timespec="seconds").replace("+00:00", "Z"),
581
+ "[[TITLE]]": "FluxLoop Evaluation Report",
582
+ "[[DATE]]": generated_iso,
583
+ "[[SUBTITLE]]": subtitle,
503
584
  "[[SUMMARY_JSON]]": json.dumps(summary, ensure_ascii=False),
504
585
  "[[PER_TRACE_JSON]]": json.dumps(per_trace_payload, ensure_ascii=False),
505
586
  "[[CRITERIA_JSON]]": json.dumps(success_criteria, ensure_ascii=False),