fluxloop-cli 0.2.27__tar.gz → 0.2.29__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/PKG-INFO +1 -1
  2. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/__init__.py +1 -1
  3. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/commands/parse.py +62 -1
  4. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/commands/run.py +115 -70
  5. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/engine/core.py +6 -0
  6. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/engine/reporting/html.py +283 -3
  7. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/runner.py +175 -20
  8. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli.egg-info/PKG-INFO +1 -1
  9. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/pyproject.toml +1 -1
  10. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/tests/test_evaluate_command.py +13 -1
  11. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/tests/test_evaluation_llm.py +1 -1
  12. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/tests/test_input_generator.py +1 -1
  13. fluxloop_cli-0.2.29/tests/test_runner_multi_turn.py +243 -0
  14. fluxloop_cli-0.2.27/tests/test_runner_multi_turn.py +0 -96
  15. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/README.md +0 -0
  16. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/arg_binder.py +0 -0
  17. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/commands/__init__.py +0 -0
  18. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/commands/config.py +0 -0
  19. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/commands/doctor.py +0 -0
  20. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/commands/evaluate.py +0 -0
  21. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/commands/generate.py +0 -0
  22. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/commands/init.py +0 -0
  23. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/commands/record.py +0 -0
  24. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/commands/status.py +0 -0
  25. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/config_loader.py +0 -0
  26. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/config_schema.py +0 -0
  27. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/constants.py +0 -0
  28. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/conversation_supervisor.py +0 -0
  29. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/environment.py +0 -0
  30. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/__init__.py +0 -0
  31. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/artifacts.py +0 -0
  32. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/config.py +0 -0
  33. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/engine/__init__.py +0 -0
  34. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/engine/analysis.py +0 -0
  35. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/engine/reporting/__init__.py +0 -0
  36. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/engine/reporting/markdown.py +0 -0
  37. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/engine/success.py +0 -0
  38. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/llm.py +0 -0
  39. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/prompts/__init__.py +0 -0
  40. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/prompts/base.py +0 -0
  41. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/prompts/information_completeness.py +0 -0
  42. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/prompts/intent_recognition.py +0 -0
  43. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/prompts/response_clarity.py +0 -0
  44. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/prompts/response_consistency.py +0 -0
  45. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/rules.py +0 -0
  46. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/input_generator.py +0 -0
  47. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/llm_generator.py +0 -0
  48. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/main.py +0 -0
  49. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/project_paths.py +0 -0
  50. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/target_loader.py +0 -0
  51. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/templates.py +0 -0
  52. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/validators.py +0 -0
  53. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli.egg-info/SOURCES.txt +0 -0
  54. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli.egg-info/dependency_links.txt +0 -0
  55. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli.egg-info/entry_points.txt +0 -0
  56. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli.egg-info/requires.txt +0 -0
  57. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli.egg-info/top_level.txt +0 -0
  58. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/setup.cfg +0 -0
  59. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/tests/test_analysis_recommendations.py +0 -0
  60. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/tests/test_arg_binder.py +0 -0
  61. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/tests/test_config_command.py +0 -0
  62. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/tests/test_conversation_supervisor.py +0 -0
  63. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/tests/test_prompt_library.py +0 -0
  64. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/tests/test_run_command.py +0 -0
  65. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/tests/test_success_criteria.py +0 -0
  66. {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/tests/test_target_loader.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: fluxloop-cli
3
- Version: 0.2.27
3
+ Version: 0.2.29
4
4
  Summary: FluxLoop CLI for running agent simulations
5
5
  Author-email: FluxLoop Team <team@fluxloop.dev>
6
6
  License: Apache-2.0
@@ -2,7 +2,7 @@
2
2
  FluxLoop CLI - Command-line interface for running agent simulations.
3
3
  """
4
4
 
5
- __version__ = "0.2.27"
5
+ __version__ = "0.2.29"
6
6
 
7
7
  from .main import app
8
8
 
@@ -75,6 +75,9 @@ class TraceSummary:
75
75
  duration_ms: float
76
76
  success: bool
77
77
  raw: dict
78
+ conversation: Optional[List[Dict[str, Any]]] = None
79
+ conversation_state: Optional[Dict[str, Any]] = None
80
+ termination_reason: Optional[str] = None
78
81
 
79
82
  def to_payload(self) -> dict:
80
83
  """Return a JSON-serialisable representation of the summary entry."""
@@ -88,6 +91,9 @@ class TraceSummary:
88
91
  "duration_ms": self.duration_ms,
89
92
  "success": self.success,
90
93
  "raw": self.raw,
94
+ "conversation": self.conversation,
95
+ "conversation_state": self.conversation_state,
96
+ "termination_reason": self.termination_reason,
91
97
  }
92
98
 
93
99
 
@@ -164,6 +170,9 @@ def _load_trace_summaries(path: Path) -> Iterable[TraceSummary]:
164
170
  output_text=payload.get("output"),
165
171
  duration_ms=payload.get("duration_ms", 0.0),
166
172
  success=payload.get("success", False),
173
+ conversation=payload.get("conversation"),
174
+ conversation_state=payload.get("conversation_state"),
175
+ termination_reason=payload.get("termination_reason"),
167
176
  raw=payload,
168
177
  )
169
178
 
@@ -214,6 +223,30 @@ def _format_markdown(
214
223
  f"{_format_json_block({'output': trace.output_text})}\n\n"
215
224
  )
216
225
 
226
+ conversation_section = ""
227
+ if trace.conversation:
228
+ conversation_lines = ["## Conversation\n"]
229
+ for entry in trace.conversation:
230
+ role = (entry.get("role") or "unknown").capitalize()
231
+ content = entry.get("content") or ""
232
+ source = entry.get("source")
233
+ metadata = entry.get("metadata") or {}
234
+ meta_bits: List[str] = []
235
+ if source:
236
+ meta_bits.append(f"source={source}")
237
+ actions = metadata.get("actions") or []
238
+ if actions:
239
+ meta_bits.append("actions=" + ", ".join(actions))
240
+ if metadata.get("closing"):
241
+ meta_bits.append("closing=true")
242
+ persona = metadata.get("persona")
243
+ if persona:
244
+ meta_bits.append(f"persona={persona}")
245
+ meta_suffix = f" _({' ; '.join(meta_bits)})_" if meta_bits else ""
246
+ conversation_lines.append(f"- **{role}**: {content}{meta_suffix}")
247
+ conversation_lines.append("")
248
+ conversation_section = "\n".join(conversation_lines)
249
+
217
250
  timeline_lines = ["## Timeline\n"]
218
251
 
219
252
  for index, obs in enumerate(observations_sorted, start=1):
@@ -238,7 +271,7 @@ def _format_markdown(
238
271
  if not observations_sorted:
239
272
  timeline_lines.append("(no observations recorded)\n")
240
273
 
241
- return header + summary_section + "".join(timeline_lines)
274
+ return header + summary_section + conversation_section + "".join(timeline_lines)
242
275
 
243
276
 
244
277
  def _sort_observations(
@@ -278,12 +311,40 @@ def _build_structured_record(
278
311
  "success": summary_payload["success"],
279
312
  "summary": summary_payload,
280
313
  "timeline": timeline,
314
+ "conversation": trace.conversation or summary_payload.get("conversation") or [],
281
315
  }
282
316
 
317
+ def _maybe_decode_content(value: Any) -> Any:
318
+ if isinstance(value, str):
319
+ text = value.strip()
320
+ if text and text[0] in "{[" and text[-1] in "}]" and len(text) >= 2:
321
+ try:
322
+ return json.loads(text)
323
+ except (json.JSONDecodeError, TypeError):
324
+ return value
325
+ return value
326
+
283
327
  record["metrics"] = {
284
328
  "observation_count": len(timeline),
285
329
  }
286
330
 
331
+ if trace.conversation_state:
332
+ record["conversation_state"] = trace.conversation_state
333
+ if trace.termination_reason:
334
+ record["termination_reason"] = trace.termination_reason
335
+ if not record["conversation"] and trace.raw.get("conversation"):
336
+ record["conversation"] = trace.raw.get("conversation")
337
+ if record["conversation"]:
338
+ normalized_conversation: List[Dict[str, Any]] = []
339
+ for entry in record["conversation"]:
340
+ if not isinstance(entry, dict):
341
+ normalized_conversation.append(entry)
342
+ continue
343
+ normalized_entry = dict(entry)
344
+ normalized_entry["content"] = _maybe_decode_content(entry.get("content"))
345
+ normalized_conversation.append(normalized_entry)
346
+ record["conversation"] = normalized_conversation
347
+
287
348
  return record
288
349
 
289
350
 
@@ -5,7 +5,7 @@ Run command for executing experiments and simulations.
5
5
  import asyncio
6
6
  import sys
7
7
  from pathlib import Path
8
- from typing import Optional
8
+ from typing import Callable, Optional
9
9
 
10
10
  import typer
11
11
  from rich.console import Console
@@ -113,6 +113,11 @@ def experiment(
113
113
  "--dry-run",
114
114
  help="Show what would be run without executing",
115
115
  ),
116
+ display: bool = typer.Option(
117
+ True,
118
+ "--display/--no-display",
119
+ help="Show rich console output (disable for plain log streaming)",
120
+ ),
116
121
  ):
117
122
  """
118
123
  Run an experiment based on configuration file.
@@ -227,7 +232,7 @@ def experiment(
227
232
  summary.add_row("Output", config.output_directory)
228
233
 
229
234
  console.print(summary)
230
-
235
+
231
236
  if dry_run:
232
237
  console.print("\n[yellow]Dry run mode - no execution will occur[/yellow]")
233
238
  return
@@ -250,87 +255,127 @@ def experiment(
250
255
  # Run experiment with progress tracking
251
256
  console.print("\n[bold green]▶️ Starting experiment...[/bold green]\n")
252
257
 
253
- with Progress(
254
- SpinnerColumn(),
255
- TextColumn("[progress.description]{task.description}"),
256
- BarColumn(),
257
- TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
258
- TimeRemainingColumn(),
259
- TextColumn("({task.completed} of {task.total})"),
260
- console=console,
261
- ) as progress:
262
- # Create main task
263
- main_task = progress.add_task(
264
- f"Running {config.name}",
265
- total=total_runs,
266
- )
267
- multi_turn_total = (
268
- config.multi_turn.max_turns
269
- if config.multi_turn and config.multi_turn.max_turns
270
- else None
271
- )
272
- turn_task = progress.add_task(
273
- "[cyan]Turn 0/0",
274
- total=multi_turn_total or 1,
275
- visible=False,
276
- )
258
+ def _execute_with_callbacks(
259
+ progress_callback: Optional[Callable[[], None]],
260
+ turn_progress_callback: Optional[Callable[[int, int, Optional[str]], None]],
261
+ ):
262
+ try:
263
+ return asyncio.run(
264
+ runner.run_experiment(
265
+ progress_callback=progress_callback,
266
+ turn_progress_callback=turn_progress_callback,
267
+ )
268
+ )
269
+ except KeyboardInterrupt:
270
+ console.print("\n[yellow]Experiment interrupted by user[/yellow]")
271
+ raise typer.Exit(1)
272
+ except Exception as e:
273
+ console.print(f"\n[red]Error during experiment:[/red] {e}")
274
+ if "--debug" in sys.argv:
275
+ console.print_exception()
276
+ raise typer.Exit(1)
277
277
 
278
- def _progress_callback():
279
- progress.advance(main_task)
278
+ if display and console.is_terminal:
279
+ with Progress(
280
+ SpinnerColumn(),
281
+ TextColumn("[progress.description]{task.description}"),
282
+ BarColumn(),
283
+ TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
284
+ TimeRemainingColumn(),
285
+ TextColumn("({task.completed} of {task.total})"),
286
+ console=console,
287
+ ) as progress:
288
+ main_task = progress.add_task(
289
+ f"Running {config.name}",
290
+ total=total_runs,
291
+ )
292
+ multi_turn_total = (
293
+ config.multi_turn.max_turns
294
+ if config.multi_turn and config.multi_turn.max_turns
295
+ else None
296
+ )
297
+ turn_task = progress.add_task(
298
+ "[cyan]Turn 0/0",
299
+ total=multi_turn_total or 1,
300
+ visible=False,
301
+ )
280
302
 
281
- def _turn_progress_callback(
282
- current_turn: int,
283
- total_turns: int,
284
- message: Optional[str] = None,
285
- ) -> None:
286
- total = total_turns or 1
287
- clamped_turn = max(0, current_turn)
288
- if message is None:
289
- completed = min(clamped_turn, total)
290
- description = f"[cyan]Turn {min(clamped_turn, total)}/{total}: complete"
303
+ def _progress_callback() -> None:
304
+ progress.advance(main_task)
305
+
306
+ def _turn_progress_callback(
307
+ current_turn: int,
308
+ total_turns: int,
309
+ message: Optional[str] = None,
310
+ ) -> None:
311
+ total = total_turns or 1
312
+ clamped_turn = max(0, current_turn)
313
+ if message is None:
314
+ completed = min(clamped_turn, total)
315
+ description = f"[cyan]Turn {min(clamped_turn, total)}/{total}: complete"
316
+ progress.update(
317
+ turn_task,
318
+ total=total,
319
+ completed=completed,
320
+ description=description,
321
+ visible=False,
322
+ )
323
+ return
324
+
325
+ if not progress.tasks[turn_task].visible:
326
+ progress.update(turn_task, visible=True)
327
+
328
+ preview = message.replace("\n", " ")
329
+ if len(preview) > 40:
330
+ preview = preview[:37] + "..."
331
+
332
+ completed = max(0, min(clamped_turn - 1, total))
333
+ description = f"[cyan]Turn {min(clamped_turn, total)}/{total}: {preview}"
291
334
  progress.update(
292
335
  turn_task,
293
336
  total=total,
294
337
  completed=completed,
295
338
  description=description,
296
- visible=False,
297
339
  )
298
- return
299
340
 
300
- if not progress.tasks[turn_task].visible:
301
- progress.update(turn_task, visible=True)
302
-
303
- preview = message.replace("\n", " ")
304
- if len(preview) > 40:
305
- preview = preview[:37] + "..."
306
-
307
- completed = max(0, min(clamped_turn - 1, total))
308
- description = f"[cyan]Turn {min(clamped_turn, total)}/{total}: {preview}"
341
+ results = _execute_with_callbacks(_progress_callback, _turn_progress_callback)
342
+ else:
343
+ completed_runs = 0
309
344
 
310
- progress.update(
311
- turn_task,
312
- total=total,
313
- completed=completed,
314
- description=description,
345
+ def _progress_callback() -> None:
346
+ nonlocal completed_runs
347
+ completed_runs += 1
348
+ print(
349
+ f"[fluxloop] run progress {completed_runs}/{total_runs}",
350
+ flush=True,
315
351
  )
316
352
 
317
- # Run experiment
318
- try:
319
- results = asyncio.run(
320
- runner.run_experiment(
321
- progress_callback=_progress_callback,
322
- turn_progress_callback=_turn_progress_callback,
353
+ def _turn_progress_callback(
354
+ current_turn: int,
355
+ total_turns: int,
356
+ message: Optional[str] = None,
357
+ ) -> None:
358
+ total = total_turns or 1
359
+ clamped_turn = max(0, current_turn)
360
+ if message:
361
+ preview = message.replace("\n", " ")
362
+ if len(preview) > 80:
363
+ preview = preview[:77] + "..."
364
+ print(
365
+ f"[fluxloop] turn progress {min(clamped_turn, total)}/{total}: {preview}",
366
+ flush=True,
323
367
  )
324
- )
325
- except KeyboardInterrupt:
326
- console.print("\n[yellow]Experiment interrupted by user[/yellow]")
327
- raise typer.Exit(1)
328
- except Exception as e:
329
- console.print(f"\n[red]Error during experiment:[/red] {e}")
330
- # Debug mode - show full traceback
331
- if "--debug" in sys.argv:
332
- console.print_exception()
333
- raise typer.Exit(1)
368
+ else:
369
+ print(
370
+ f"[fluxloop] turn progress {min(clamped_turn, total)}/{total}: complete",
371
+ flush=True,
372
+ )
373
+
374
+ print(
375
+ f"[fluxloop] running {config.name} ({total_runs} total runs)",
376
+ flush=True,
377
+ )
378
+ results = _execute_with_callbacks(_progress_callback, _turn_progress_callback)
334
379
 
335
380
  config.set_resolved_input_count(results.get("input_count", config.get_input_count()))
336
381
 
@@ -217,6 +217,12 @@ def _write_per_trace(results: List[TraceOutcome], path: Path) -> None:
217
217
  if outcome.trace.get(key) is not None
218
218
  },
219
219
  }
220
+ if outcome.trace.get("conversation") is not None:
221
+ payload["conversation"] = outcome.trace.get("conversation")
222
+ if outcome.trace.get("conversation_state") is not None:
223
+ payload["conversation_state"] = outcome.trace.get("conversation_state")
224
+ if outcome.trace.get("termination_reason") is not None:
225
+ payload["termination_reason"] = outcome.trace.get("termination_reason")
220
226
  handle.write(json.dumps(payload, ensure_ascii=False) + "\n")
221
227
 
222
228