fluxloop-cli 0.2.26__tar.gz → 0.2.28__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/PKG-INFO +1 -1
  2. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/__init__.py +1 -1
  3. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/commands/parse.py +62 -1
  4. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/commands/run.py +65 -0
  5. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/engine/core.py +6 -0
  6. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/engine/reporting/html.py +364 -3
  7. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/rules.py +167 -0
  8. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/runner.py +175 -20
  9. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/templates.py +4 -4
  10. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli.egg-info/PKG-INFO +1 -1
  11. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/pyproject.toml +1 -1
  12. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/tests/test_evaluate_command.py +51 -7
  13. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/tests/test_evaluation_llm.py +1 -1
  14. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/tests/test_input_generator.py +1 -1
  15. fluxloop_cli-0.2.28/tests/test_runner_multi_turn.py +243 -0
  16. fluxloop_cli-0.2.26/tests/test_runner_multi_turn.py +0 -96
  17. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/README.md +0 -0
  18. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/arg_binder.py +0 -0
  19. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/commands/__init__.py +0 -0
  20. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/commands/config.py +0 -0
  21. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/commands/doctor.py +0 -0
  22. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/commands/evaluate.py +0 -0
  23. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/commands/generate.py +0 -0
  24. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/commands/init.py +0 -0
  25. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/commands/record.py +0 -0
  26. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/commands/status.py +0 -0
  27. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/config_loader.py +0 -0
  28. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/config_schema.py +0 -0
  29. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/constants.py +0 -0
  30. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/conversation_supervisor.py +0 -0
  31. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/environment.py +0 -0
  32. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/__init__.py +0 -0
  33. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/artifacts.py +0 -0
  34. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/config.py +0 -0
  35. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/engine/__init__.py +0 -0
  36. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/engine/analysis.py +0 -0
  37. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/engine/reporting/__init__.py +0 -0
  38. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/engine/reporting/markdown.py +0 -0
  39. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/engine/success.py +0 -0
  40. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/llm.py +0 -0
  41. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/prompts/__init__.py +0 -0
  42. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/prompts/base.py +0 -0
  43. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/prompts/information_completeness.py +0 -0
  44. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/prompts/intent_recognition.py +0 -0
  45. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/prompts/response_clarity.py +0 -0
  46. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/prompts/response_consistency.py +0 -0
  47. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/input_generator.py +0 -0
  48. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/llm_generator.py +0 -0
  49. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/main.py +0 -0
  50. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/project_paths.py +0 -0
  51. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/target_loader.py +0 -0
  52. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/validators.py +0 -0
  53. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli.egg-info/SOURCES.txt +0 -0
  54. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli.egg-info/dependency_links.txt +0 -0
  55. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli.egg-info/entry_points.txt +0 -0
  56. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli.egg-info/requires.txt +0 -0
  57. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli.egg-info/top_level.txt +0 -0
  58. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/setup.cfg +0 -0
  59. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/tests/test_analysis_recommendations.py +0 -0
  60. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/tests/test_arg_binder.py +0 -0
  61. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/tests/test_config_command.py +0 -0
  62. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/tests/test_conversation_supervisor.py +0 -0
  63. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/tests/test_prompt_library.py +0 -0
  64. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/tests/test_run_command.py +0 -0
  65. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/tests/test_success_criteria.py +0 -0
  66. {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/tests/test_target_loader.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: fluxloop-cli
3
- Version: 0.2.26
3
+ Version: 0.2.28
4
4
  Summary: FluxLoop CLI for running agent simulations
5
5
  Author-email: FluxLoop Team <team@fluxloop.dev>
6
6
  License: Apache-2.0
@@ -2,7 +2,7 @@
2
2
  FluxLoop CLI - Command-line interface for running agent simulations.
3
3
  """
4
4
 
5
- __version__ = "0.2.26"
5
+ __version__ = "0.2.28"
6
6
 
7
7
  from .main import app
8
8
 
@@ -75,6 +75,9 @@ class TraceSummary:
75
75
  duration_ms: float
76
76
  success: bool
77
77
  raw: dict
78
+ conversation: Optional[List[Dict[str, Any]]] = None
79
+ conversation_state: Optional[Dict[str, Any]] = None
80
+ termination_reason: Optional[str] = None
78
81
 
79
82
  def to_payload(self) -> dict:
80
83
  """Return a JSON-serialisable representation of the summary entry."""
@@ -88,6 +91,9 @@ class TraceSummary:
88
91
  "duration_ms": self.duration_ms,
89
92
  "success": self.success,
90
93
  "raw": self.raw,
94
+ "conversation": self.conversation,
95
+ "conversation_state": self.conversation_state,
96
+ "termination_reason": self.termination_reason,
91
97
  }
92
98
 
93
99
 
@@ -164,6 +170,9 @@ def _load_trace_summaries(path: Path) -> Iterable[TraceSummary]:
164
170
  output_text=payload.get("output"),
165
171
  duration_ms=payload.get("duration_ms", 0.0),
166
172
  success=payload.get("success", False),
173
+ conversation=payload.get("conversation"),
174
+ conversation_state=payload.get("conversation_state"),
175
+ termination_reason=payload.get("termination_reason"),
167
176
  raw=payload,
168
177
  )
169
178
 
@@ -214,6 +223,30 @@ def _format_markdown(
214
223
  f"{_format_json_block({'output': trace.output_text})}\n\n"
215
224
  )
216
225
 
226
+ conversation_section = ""
227
+ if trace.conversation:
228
+ conversation_lines = ["## Conversation\n"]
229
+ for entry in trace.conversation:
230
+ role = (entry.get("role") or "unknown").capitalize()
231
+ content = entry.get("content") or ""
232
+ source = entry.get("source")
233
+ metadata = entry.get("metadata") or {}
234
+ meta_bits: List[str] = []
235
+ if source:
236
+ meta_bits.append(f"source={source}")
237
+ actions = metadata.get("actions") or []
238
+ if actions:
239
+ meta_bits.append("actions=" + ", ".join(actions))
240
+ if metadata.get("closing"):
241
+ meta_bits.append("closing=true")
242
+ persona = metadata.get("persona")
243
+ if persona:
244
+ meta_bits.append(f"persona={persona}")
245
+ meta_suffix = f" _({' ; '.join(meta_bits)})_" if meta_bits else ""
246
+ conversation_lines.append(f"- **{role}**: {content}{meta_suffix}")
247
+ conversation_lines.append("")
248
+ conversation_section = "\n".join(conversation_lines)
249
+
217
250
  timeline_lines = ["## Timeline\n"]
218
251
 
219
252
  for index, obs in enumerate(observations_sorted, start=1):
@@ -238,7 +271,7 @@ def _format_markdown(
238
271
  if not observations_sorted:
239
272
  timeline_lines.append("(no observations recorded)\n")
240
273
 
241
- return header + summary_section + "".join(timeline_lines)
274
+ return header + summary_section + conversation_section + "".join(timeline_lines)
242
275
 
243
276
 
244
277
  def _sort_observations(
@@ -278,12 +311,40 @@ def _build_structured_record(
278
311
  "success": summary_payload["success"],
279
312
  "summary": summary_payload,
280
313
  "timeline": timeline,
314
+ "conversation": trace.conversation or summary_payload.get("conversation") or [],
281
315
  }
282
316
 
317
+ def _maybe_decode_content(value: Any) -> Any:
318
+ if isinstance(value, str):
319
+ text = value.strip()
320
+ if text and text[0] in "{[" and text[-1] in "}]" and len(text) >= 2:
321
+ try:
322
+ return json.loads(text)
323
+ except (json.JSONDecodeError, TypeError):
324
+ return value
325
+ return value
326
+
283
327
  record["metrics"] = {
284
328
  "observation_count": len(timeline),
285
329
  }
286
330
 
331
+ if trace.conversation_state:
332
+ record["conversation_state"] = trace.conversation_state
333
+ if trace.termination_reason:
334
+ record["termination_reason"] = trace.termination_reason
335
+ if not record["conversation"] and trace.raw.get("conversation"):
336
+ record["conversation"] = trace.raw.get("conversation")
337
+ if record["conversation"]:
338
+ normalized_conversation: List[Dict[str, Any]] = []
339
+ for entry in record["conversation"]:
340
+ if not isinstance(entry, dict):
341
+ normalized_conversation.append(entry)
342
+ continue
343
+ normalized_entry = dict(entry)
344
+ normalized_entry["content"] = _maybe_decode_content(entry.get("content"))
345
+ normalized_conversation.append(normalized_entry)
346
+ record["conversation"] = normalized_conversation
347
+
287
348
  return record
288
349
 
289
350
 
@@ -3,6 +3,7 @@ Run command for executing experiments and simulations.
3
3
  """
4
4
 
5
5
  import asyncio
6
+ import json
6
7
  import sys
7
8
  from pathlib import Path
8
9
  from typing import Optional
@@ -27,6 +28,16 @@ app = typer.Typer()
27
28
  console = Console()
28
29
 
29
30
 
31
+ def _emit_progress_event(event: str, **payload: object) -> None:
32
+ """Emit a structured progress event to stderr for machine consumption."""
33
+ data = {"event": event, **payload}
34
+ try:
35
+ print(json.dumps(data, default=str), file=sys.stderr, flush=True)
36
+ except Exception:
37
+ # Fall back to repr if serialization fails
38
+ print({"event": event, **{k: repr(v) for k, v in payload.items()}}, file=sys.stderr, flush=True)
39
+
40
+
30
41
  @app.command()
31
42
  def experiment(
32
43
  config_file: Path = typer.Option(
@@ -227,9 +238,20 @@ def experiment(
227
238
  summary.add_row("Output", config.output_directory)
228
239
 
229
240
  console.print(summary)
241
+
242
+ _emit_progress_event(
243
+ "experiment_summary",
244
+ name=config.name,
245
+ iterations=config.iterations,
246
+ personas=len(config.personas),
247
+ total_runs=total_runs,
248
+ multi_turn_enabled=bool(config.multi_turn and config.multi_turn.enabled),
249
+ output_dir=config.output_directory,
250
+ )
230
251
 
231
252
  if dry_run:
232
253
  console.print("\n[yellow]Dry run mode - no execution will occur[/yellow]")
254
+ _emit_progress_event("experiment_status", status="dry_run", name=config.name)
233
255
  return
234
256
 
235
257
  # Confirm execution
@@ -243,13 +265,22 @@ def experiment(
243
265
  f"\nThis will execute {total_runs} runs."
244
266
  )
245
267
 
268
+ _emit_progress_event(
269
+ "experiment_ready",
270
+ name=config.name,
271
+ total_runs=total_runs,
272
+ )
273
+
246
274
  if not typer.confirm("Continue?"):
275
+ _emit_progress_event("experiment_status", status="aborted", reason="user_declined", name=config.name)
247
276
  raise typer.Abort()
248
277
 
249
278
  # Create runner
250
279
  # Run experiment with progress tracking
251
280
  console.print("\n[bold green]▶️ Starting experiment...[/bold green]\n")
252
281
 
282
+ _emit_progress_event("experiment_started", name=config.name, total_runs=total_runs)
283
+
253
284
  with Progress(
254
285
  SpinnerColumn(),
255
286
  TextColumn("[progress.description]{task.description}"),
@@ -277,6 +308,14 @@ def experiment(
277
308
 
278
309
  def _progress_callback():
279
310
  progress.advance(main_task)
311
+ task = progress.tasks[main_task]
312
+ _emit_progress_event(
313
+ "run_progress",
314
+ completed=int(task.completed),
315
+ total=int(task.total or 0),
316
+ description=task.description,
317
+ experiment=config.name,
318
+ )
280
319
 
281
320
  def _turn_progress_callback(
282
321
  current_turn: int,
@@ -295,6 +334,13 @@ def experiment(
295
334
  description=description,
296
335
  visible=False,
297
336
  )
337
+ _emit_progress_event(
338
+ "turn_progress",
339
+ state="complete",
340
+ current=min(clamped_turn, total),
341
+ total=total,
342
+ experiment=config.name,
343
+ )
298
344
  return
299
345
 
300
346
  if not progress.tasks[turn_task].visible:
@@ -313,6 +359,14 @@ def experiment(
313
359
  completed=completed,
314
360
  description=description,
315
361
  )
362
+ _emit_progress_event(
363
+ "turn_progress",
364
+ state="in_progress",
365
+ current=clamped_turn,
366
+ total=total,
367
+ preview=preview,
368
+ experiment=config.name,
369
+ )
316
370
 
317
371
  # Run experiment
318
372
  try:
@@ -324,12 +378,14 @@ def experiment(
324
378
  )
325
379
  except KeyboardInterrupt:
326
380
  console.print("\n[yellow]Experiment interrupted by user[/yellow]")
381
+ _emit_progress_event("experiment_status", status="interrupted", name=config.name)
327
382
  raise typer.Exit(1)
328
383
  except Exception as e:
329
384
  console.print(f"\n[red]Error during experiment:[/red] {e}")
330
385
  # Debug mode - show full traceback
331
386
  if "--debug" in sys.argv:
332
387
  console.print_exception()
388
+ _emit_progress_event("experiment_status", status="error", error=str(e), name=config.name)
333
389
  raise typer.Exit(1)
334
390
 
335
391
  config.set_resolved_input_count(results.get("input_count", config.get_input_count()))
@@ -343,6 +399,15 @@ def experiment(
343
399
  )
344
400
 
345
401
  _display_results(results)
402
+ _emit_progress_event(
403
+ "experiment_status",
404
+ status="completed",
405
+ name=config.name,
406
+ total_runs=results.get("total_runs", total_runs),
407
+ successful=results.get("successful"),
408
+ failed=results.get("failed"),
409
+ success_rate=results.get("success_rate"),
410
+ )
346
411
 
347
412
 
348
413
  @app.command()
@@ -217,6 +217,12 @@ def _write_per_trace(results: List[TraceOutcome], path: Path) -> None:
217
217
  if outcome.trace.get(key) is not None
218
218
  },
219
219
  }
220
+ if outcome.trace.get("conversation") is not None:
221
+ payload["conversation"] = outcome.trace.get("conversation")
222
+ if outcome.trace.get("conversation_state") is not None:
223
+ payload["conversation_state"] = outcome.trace.get("conversation_state")
224
+ if outcome.trace.get("termination_reason") is not None:
225
+ payload["termination_reason"] = outcome.trace.get("termination_reason")
220
226
  handle.write(json.dumps(payload, ensure_ascii=False) + "\n")
221
227
 
222
228