fluxloop-cli 0.2.27__tar.gz → 0.2.29__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/PKG-INFO +1 -1
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/__init__.py +1 -1
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/commands/parse.py +62 -1
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/commands/run.py +115 -70
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/engine/core.py +6 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/engine/reporting/html.py +283 -3
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/runner.py +175 -20
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli.egg-info/PKG-INFO +1 -1
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/pyproject.toml +1 -1
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/tests/test_evaluate_command.py +13 -1
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/tests/test_evaluation_llm.py +1 -1
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/tests/test_input_generator.py +1 -1
- fluxloop_cli-0.2.29/tests/test_runner_multi_turn.py +243 -0
- fluxloop_cli-0.2.27/tests/test_runner_multi_turn.py +0 -96
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/README.md +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/arg_binder.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/commands/__init__.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/commands/config.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/commands/doctor.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/commands/evaluate.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/commands/generate.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/commands/init.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/commands/record.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/commands/status.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/config_loader.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/config_schema.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/constants.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/conversation_supervisor.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/environment.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/__init__.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/artifacts.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/config.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/engine/__init__.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/engine/analysis.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/engine/reporting/__init__.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/engine/reporting/markdown.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/engine/success.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/llm.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/prompts/__init__.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/prompts/base.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/prompts/information_completeness.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/prompts/intent_recognition.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/prompts/response_clarity.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/prompts/response_consistency.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/evaluation/rules.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/input_generator.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/llm_generator.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/main.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/project_paths.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/target_loader.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/templates.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli/validators.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli.egg-info/SOURCES.txt +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli.egg-info/dependency_links.txt +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli.egg-info/entry_points.txt +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli.egg-info/requires.txt +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/fluxloop_cli.egg-info/top_level.txt +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/setup.cfg +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/tests/test_analysis_recommendations.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/tests/test_arg_binder.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/tests/test_config_command.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/tests/test_conversation_supervisor.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/tests/test_prompt_library.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/tests/test_run_command.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/tests/test_success_criteria.py +0 -0
- {fluxloop_cli-0.2.27 → fluxloop_cli-0.2.29}/tests/test_target_loader.py +0 -0
|
@@ -75,6 +75,9 @@ class TraceSummary:
|
|
|
75
75
|
duration_ms: float
|
|
76
76
|
success: bool
|
|
77
77
|
raw: dict
|
|
78
|
+
conversation: Optional[List[Dict[str, Any]]] = None
|
|
79
|
+
conversation_state: Optional[Dict[str, Any]] = None
|
|
80
|
+
termination_reason: Optional[str] = None
|
|
78
81
|
|
|
79
82
|
def to_payload(self) -> dict:
|
|
80
83
|
"""Return a JSON-serialisable representation of the summary entry."""
|
|
@@ -88,6 +91,9 @@ class TraceSummary:
|
|
|
88
91
|
"duration_ms": self.duration_ms,
|
|
89
92
|
"success": self.success,
|
|
90
93
|
"raw": self.raw,
|
|
94
|
+
"conversation": self.conversation,
|
|
95
|
+
"conversation_state": self.conversation_state,
|
|
96
|
+
"termination_reason": self.termination_reason,
|
|
91
97
|
}
|
|
92
98
|
|
|
93
99
|
|
|
@@ -164,6 +170,9 @@ def _load_trace_summaries(path: Path) -> Iterable[TraceSummary]:
|
|
|
164
170
|
output_text=payload.get("output"),
|
|
165
171
|
duration_ms=payload.get("duration_ms", 0.0),
|
|
166
172
|
success=payload.get("success", False),
|
|
173
|
+
conversation=payload.get("conversation"),
|
|
174
|
+
conversation_state=payload.get("conversation_state"),
|
|
175
|
+
termination_reason=payload.get("termination_reason"),
|
|
167
176
|
raw=payload,
|
|
168
177
|
)
|
|
169
178
|
|
|
@@ -214,6 +223,30 @@ def _format_markdown(
|
|
|
214
223
|
f"{_format_json_block({'output': trace.output_text})}\n\n"
|
|
215
224
|
)
|
|
216
225
|
|
|
226
|
+
conversation_section = ""
|
|
227
|
+
if trace.conversation:
|
|
228
|
+
conversation_lines = ["## Conversation\n"]
|
|
229
|
+
for entry in trace.conversation:
|
|
230
|
+
role = (entry.get("role") or "unknown").capitalize()
|
|
231
|
+
content = entry.get("content") or ""
|
|
232
|
+
source = entry.get("source")
|
|
233
|
+
metadata = entry.get("metadata") or {}
|
|
234
|
+
meta_bits: List[str] = []
|
|
235
|
+
if source:
|
|
236
|
+
meta_bits.append(f"source={source}")
|
|
237
|
+
actions = metadata.get("actions") or []
|
|
238
|
+
if actions:
|
|
239
|
+
meta_bits.append("actions=" + ", ".join(actions))
|
|
240
|
+
if metadata.get("closing"):
|
|
241
|
+
meta_bits.append("closing=true")
|
|
242
|
+
persona = metadata.get("persona")
|
|
243
|
+
if persona:
|
|
244
|
+
meta_bits.append(f"persona={persona}")
|
|
245
|
+
meta_suffix = f" _({' ; '.join(meta_bits)})_" if meta_bits else ""
|
|
246
|
+
conversation_lines.append(f"- **{role}**: {content}{meta_suffix}")
|
|
247
|
+
conversation_lines.append("")
|
|
248
|
+
conversation_section = "\n".join(conversation_lines)
|
|
249
|
+
|
|
217
250
|
timeline_lines = ["## Timeline\n"]
|
|
218
251
|
|
|
219
252
|
for index, obs in enumerate(observations_sorted, start=1):
|
|
@@ -238,7 +271,7 @@ def _format_markdown(
|
|
|
238
271
|
if not observations_sorted:
|
|
239
272
|
timeline_lines.append("(no observations recorded)\n")
|
|
240
273
|
|
|
241
|
-
return header + summary_section + "".join(timeline_lines)
|
|
274
|
+
return header + summary_section + conversation_section + "".join(timeline_lines)
|
|
242
275
|
|
|
243
276
|
|
|
244
277
|
def _sort_observations(
|
|
@@ -278,12 +311,40 @@ def _build_structured_record(
|
|
|
278
311
|
"success": summary_payload["success"],
|
|
279
312
|
"summary": summary_payload,
|
|
280
313
|
"timeline": timeline,
|
|
314
|
+
"conversation": trace.conversation or summary_payload.get("conversation") or [],
|
|
281
315
|
}
|
|
282
316
|
|
|
317
|
+
def _maybe_decode_content(value: Any) -> Any:
|
|
318
|
+
if isinstance(value, str):
|
|
319
|
+
text = value.strip()
|
|
320
|
+
if text and text[0] in "{[" and text[-1] in "}]" and len(text) >= 2:
|
|
321
|
+
try:
|
|
322
|
+
return json.loads(text)
|
|
323
|
+
except (json.JSONDecodeError, TypeError):
|
|
324
|
+
return value
|
|
325
|
+
return value
|
|
326
|
+
|
|
283
327
|
record["metrics"] = {
|
|
284
328
|
"observation_count": len(timeline),
|
|
285
329
|
}
|
|
286
330
|
|
|
331
|
+
if trace.conversation_state:
|
|
332
|
+
record["conversation_state"] = trace.conversation_state
|
|
333
|
+
if trace.termination_reason:
|
|
334
|
+
record["termination_reason"] = trace.termination_reason
|
|
335
|
+
if not record["conversation"] and trace.raw.get("conversation"):
|
|
336
|
+
record["conversation"] = trace.raw.get("conversation")
|
|
337
|
+
if record["conversation"]:
|
|
338
|
+
normalized_conversation: List[Dict[str, Any]] = []
|
|
339
|
+
for entry in record["conversation"]:
|
|
340
|
+
if not isinstance(entry, dict):
|
|
341
|
+
normalized_conversation.append(entry)
|
|
342
|
+
continue
|
|
343
|
+
normalized_entry = dict(entry)
|
|
344
|
+
normalized_entry["content"] = _maybe_decode_content(entry.get("content"))
|
|
345
|
+
normalized_conversation.append(normalized_entry)
|
|
346
|
+
record["conversation"] = normalized_conversation
|
|
347
|
+
|
|
287
348
|
return record
|
|
288
349
|
|
|
289
350
|
|
|
@@ -5,7 +5,7 @@ Run command for executing experiments and simulations.
|
|
|
5
5
|
import asyncio
|
|
6
6
|
import sys
|
|
7
7
|
from pathlib import Path
|
|
8
|
-
from typing import Optional
|
|
8
|
+
from typing import Callable, Optional
|
|
9
9
|
|
|
10
10
|
import typer
|
|
11
11
|
from rich.console import Console
|
|
@@ -113,6 +113,11 @@ def experiment(
|
|
|
113
113
|
"--dry-run",
|
|
114
114
|
help="Show what would be run without executing",
|
|
115
115
|
),
|
|
116
|
+
display: bool = typer.Option(
|
|
117
|
+
True,
|
|
118
|
+
"--display/--no-display",
|
|
119
|
+
help="Show rich console output (disable for plain log streaming)",
|
|
120
|
+
),
|
|
116
121
|
):
|
|
117
122
|
"""
|
|
118
123
|
Run an experiment based on configuration file.
|
|
@@ -227,7 +232,7 @@ def experiment(
|
|
|
227
232
|
summary.add_row("Output", config.output_directory)
|
|
228
233
|
|
|
229
234
|
console.print(summary)
|
|
230
|
-
|
|
235
|
+
|
|
231
236
|
if dry_run:
|
|
232
237
|
console.print("\n[yellow]Dry run mode - no execution will occur[/yellow]")
|
|
233
238
|
return
|
|
@@ -250,87 +255,127 @@ def experiment(
|
|
|
250
255
|
# Run experiment with progress tracking
|
|
251
256
|
console.print("\n[bold green]▶️ Starting experiment...[/bold green]\n")
|
|
252
257
|
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
if
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
turn_task = progress.add_task(
|
|
273
|
-
"[cyan]Turn 0/0",
|
|
274
|
-
total=multi_turn_total or 1,
|
|
275
|
-
visible=False,
|
|
276
|
-
)
|
|
258
|
+
def _execute_with_callbacks(
|
|
259
|
+
progress_callback: Optional[Callable[[], None]],
|
|
260
|
+
turn_progress_callback: Optional[Callable[[int, int, Optional[str]], None]],
|
|
261
|
+
):
|
|
262
|
+
try:
|
|
263
|
+
return asyncio.run(
|
|
264
|
+
runner.run_experiment(
|
|
265
|
+
progress_callback=progress_callback,
|
|
266
|
+
turn_progress_callback=turn_progress_callback,
|
|
267
|
+
)
|
|
268
|
+
)
|
|
269
|
+
except KeyboardInterrupt:
|
|
270
|
+
console.print("\n[yellow]Experiment interrupted by user[/yellow]")
|
|
271
|
+
raise typer.Exit(1)
|
|
272
|
+
except Exception as e:
|
|
273
|
+
console.print(f"\n[red]Error during experiment:[/red] {e}")
|
|
274
|
+
if "--debug" in sys.argv:
|
|
275
|
+
console.print_exception()
|
|
276
|
+
raise typer.Exit(1)
|
|
277
277
|
|
|
278
|
-
|
|
279
|
-
|
|
278
|
+
if display and console.is_terminal:
|
|
279
|
+
with Progress(
|
|
280
|
+
SpinnerColumn(),
|
|
281
|
+
TextColumn("[progress.description]{task.description}"),
|
|
282
|
+
BarColumn(),
|
|
283
|
+
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
|
|
284
|
+
TimeRemainingColumn(),
|
|
285
|
+
TextColumn("({task.completed} of {task.total})"),
|
|
286
|
+
console=console,
|
|
287
|
+
) as progress:
|
|
288
|
+
main_task = progress.add_task(
|
|
289
|
+
f"Running {config.name}",
|
|
290
|
+
total=total_runs,
|
|
291
|
+
)
|
|
292
|
+
multi_turn_total = (
|
|
293
|
+
config.multi_turn.max_turns
|
|
294
|
+
if config.multi_turn and config.multi_turn.max_turns
|
|
295
|
+
else None
|
|
296
|
+
)
|
|
297
|
+
turn_task = progress.add_task(
|
|
298
|
+
"[cyan]Turn 0/0",
|
|
299
|
+
total=multi_turn_total or 1,
|
|
300
|
+
visible=False,
|
|
301
|
+
)
|
|
280
302
|
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
303
|
+
def _progress_callback() -> None:
|
|
304
|
+
progress.advance(main_task)
|
|
305
|
+
|
|
306
|
+
def _turn_progress_callback(
|
|
307
|
+
current_turn: int,
|
|
308
|
+
total_turns: int,
|
|
309
|
+
message: Optional[str] = None,
|
|
310
|
+
) -> None:
|
|
311
|
+
total = total_turns or 1
|
|
312
|
+
clamped_turn = max(0, current_turn)
|
|
313
|
+
if message is None:
|
|
314
|
+
completed = min(clamped_turn, total)
|
|
315
|
+
description = f"[cyan]Turn {min(clamped_turn, total)}/{total}: complete"
|
|
316
|
+
progress.update(
|
|
317
|
+
turn_task,
|
|
318
|
+
total=total,
|
|
319
|
+
completed=completed,
|
|
320
|
+
description=description,
|
|
321
|
+
visible=False,
|
|
322
|
+
)
|
|
323
|
+
return
|
|
324
|
+
|
|
325
|
+
if not progress.tasks[turn_task].visible:
|
|
326
|
+
progress.update(turn_task, visible=True)
|
|
327
|
+
|
|
328
|
+
preview = message.replace("\n", " ")
|
|
329
|
+
if len(preview) > 40:
|
|
330
|
+
preview = preview[:37] + "..."
|
|
331
|
+
|
|
332
|
+
completed = max(0, min(clamped_turn - 1, total))
|
|
333
|
+
description = f"[cyan]Turn {min(clamped_turn, total)}/{total}: {preview}"
|
|
291
334
|
progress.update(
|
|
292
335
|
turn_task,
|
|
293
336
|
total=total,
|
|
294
337
|
completed=completed,
|
|
295
338
|
description=description,
|
|
296
|
-
visible=False,
|
|
297
339
|
)
|
|
298
|
-
return
|
|
299
340
|
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
preview = message.replace("\n", " ")
|
|
304
|
-
if len(preview) > 40:
|
|
305
|
-
preview = preview[:37] + "..."
|
|
306
|
-
|
|
307
|
-
completed = max(0, min(clamped_turn - 1, total))
|
|
308
|
-
description = f"[cyan]Turn {min(clamped_turn, total)}/{total}: {preview}"
|
|
341
|
+
results = _execute_with_callbacks(_progress_callback, _turn_progress_callback)
|
|
342
|
+
else:
|
|
343
|
+
completed_runs = 0
|
|
309
344
|
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
345
|
+
def _progress_callback() -> None:
|
|
346
|
+
nonlocal completed_runs
|
|
347
|
+
completed_runs += 1
|
|
348
|
+
print(
|
|
349
|
+
f"[fluxloop] run progress {completed_runs}/{total_runs}",
|
|
350
|
+
flush=True,
|
|
315
351
|
)
|
|
316
352
|
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
353
|
+
def _turn_progress_callback(
|
|
354
|
+
current_turn: int,
|
|
355
|
+
total_turns: int,
|
|
356
|
+
message: Optional[str] = None,
|
|
357
|
+
) -> None:
|
|
358
|
+
total = total_turns or 1
|
|
359
|
+
clamped_turn = max(0, current_turn)
|
|
360
|
+
if message:
|
|
361
|
+
preview = message.replace("\n", " ")
|
|
362
|
+
if len(preview) > 80:
|
|
363
|
+
preview = preview[:77] + "..."
|
|
364
|
+
print(
|
|
365
|
+
f"[fluxloop] turn progress {min(clamped_turn, total)}/{total}: {preview}",
|
|
366
|
+
flush=True,
|
|
323
367
|
)
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
368
|
+
else:
|
|
369
|
+
print(
|
|
370
|
+
f"[fluxloop] turn progress {min(clamped_turn, total)}/{total}: complete",
|
|
371
|
+
flush=True,
|
|
372
|
+
)
|
|
373
|
+
|
|
374
|
+
print(
|
|
375
|
+
f"[fluxloop] running {config.name} ({total_runs} total runs)",
|
|
376
|
+
flush=True,
|
|
377
|
+
)
|
|
378
|
+
results = _execute_with_callbacks(_progress_callback, _turn_progress_callback)
|
|
334
379
|
|
|
335
380
|
config.set_resolved_input_count(results.get("input_count", config.get_input_count()))
|
|
336
381
|
|
|
@@ -217,6 +217,12 @@ def _write_per_trace(results: List[TraceOutcome], path: Path) -> None:
|
|
|
217
217
|
if outcome.trace.get(key) is not None
|
|
218
218
|
},
|
|
219
219
|
}
|
|
220
|
+
if outcome.trace.get("conversation") is not None:
|
|
221
|
+
payload["conversation"] = outcome.trace.get("conversation")
|
|
222
|
+
if outcome.trace.get("conversation_state") is not None:
|
|
223
|
+
payload["conversation_state"] = outcome.trace.get("conversation_state")
|
|
224
|
+
if outcome.trace.get("termination_reason") is not None:
|
|
225
|
+
payload["termination_reason"] = outcome.trace.get("termination_reason")
|
|
220
226
|
handle.write(json.dumps(payload, ensure_ascii=False) + "\n")
|
|
221
227
|
|
|
222
228
|
|