fluxloop-cli 0.2.26__tar.gz → 0.2.28__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/PKG-INFO +1 -1
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/__init__.py +1 -1
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/commands/parse.py +62 -1
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/commands/run.py +65 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/engine/core.py +6 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/engine/reporting/html.py +364 -3
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/rules.py +167 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/runner.py +175 -20
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/templates.py +4 -4
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli.egg-info/PKG-INFO +1 -1
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/pyproject.toml +1 -1
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/tests/test_evaluate_command.py +51 -7
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/tests/test_evaluation_llm.py +1 -1
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/tests/test_input_generator.py +1 -1
- fluxloop_cli-0.2.28/tests/test_runner_multi_turn.py +243 -0
- fluxloop_cli-0.2.26/tests/test_runner_multi_turn.py +0 -96
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/README.md +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/arg_binder.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/commands/__init__.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/commands/config.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/commands/doctor.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/commands/evaluate.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/commands/generate.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/commands/init.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/commands/record.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/commands/status.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/config_loader.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/config_schema.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/constants.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/conversation_supervisor.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/environment.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/__init__.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/artifacts.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/config.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/engine/__init__.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/engine/analysis.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/engine/reporting/__init__.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/engine/reporting/markdown.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/engine/success.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/llm.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/prompts/__init__.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/prompts/base.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/prompts/information_completeness.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/prompts/intent_recognition.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/prompts/response_clarity.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/evaluation/prompts/response_consistency.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/input_generator.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/llm_generator.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/main.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/project_paths.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/target_loader.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli/validators.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli.egg-info/SOURCES.txt +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli.egg-info/dependency_links.txt +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli.egg-info/entry_points.txt +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli.egg-info/requires.txt +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/fluxloop_cli.egg-info/top_level.txt +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/setup.cfg +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/tests/test_analysis_recommendations.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/tests/test_arg_binder.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/tests/test_config_command.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/tests/test_conversation_supervisor.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/tests/test_prompt_library.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/tests/test_run_command.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/tests/test_success_criteria.py +0 -0
- {fluxloop_cli-0.2.26 → fluxloop_cli-0.2.28}/tests/test_target_loader.py +0 -0
|
@@ -75,6 +75,9 @@ class TraceSummary:
|
|
|
75
75
|
duration_ms: float
|
|
76
76
|
success: bool
|
|
77
77
|
raw: dict
|
|
78
|
+
conversation: Optional[List[Dict[str, Any]]] = None
|
|
79
|
+
conversation_state: Optional[Dict[str, Any]] = None
|
|
80
|
+
termination_reason: Optional[str] = None
|
|
78
81
|
|
|
79
82
|
def to_payload(self) -> dict:
|
|
80
83
|
"""Return a JSON-serialisable representation of the summary entry."""
|
|
@@ -88,6 +91,9 @@ class TraceSummary:
|
|
|
88
91
|
"duration_ms": self.duration_ms,
|
|
89
92
|
"success": self.success,
|
|
90
93
|
"raw": self.raw,
|
|
94
|
+
"conversation": self.conversation,
|
|
95
|
+
"conversation_state": self.conversation_state,
|
|
96
|
+
"termination_reason": self.termination_reason,
|
|
91
97
|
}
|
|
92
98
|
|
|
93
99
|
|
|
@@ -164,6 +170,9 @@ def _load_trace_summaries(path: Path) -> Iterable[TraceSummary]:
|
|
|
164
170
|
output_text=payload.get("output"),
|
|
165
171
|
duration_ms=payload.get("duration_ms", 0.0),
|
|
166
172
|
success=payload.get("success", False),
|
|
173
|
+
conversation=payload.get("conversation"),
|
|
174
|
+
conversation_state=payload.get("conversation_state"),
|
|
175
|
+
termination_reason=payload.get("termination_reason"),
|
|
167
176
|
raw=payload,
|
|
168
177
|
)
|
|
169
178
|
|
|
@@ -214,6 +223,30 @@ def _format_markdown(
|
|
|
214
223
|
f"{_format_json_block({'output': trace.output_text})}\n\n"
|
|
215
224
|
)
|
|
216
225
|
|
|
226
|
+
conversation_section = ""
|
|
227
|
+
if trace.conversation:
|
|
228
|
+
conversation_lines = ["## Conversation\n"]
|
|
229
|
+
for entry in trace.conversation:
|
|
230
|
+
role = (entry.get("role") or "unknown").capitalize()
|
|
231
|
+
content = entry.get("content") or ""
|
|
232
|
+
source = entry.get("source")
|
|
233
|
+
metadata = entry.get("metadata") or {}
|
|
234
|
+
meta_bits: List[str] = []
|
|
235
|
+
if source:
|
|
236
|
+
meta_bits.append(f"source={source}")
|
|
237
|
+
actions = metadata.get("actions") or []
|
|
238
|
+
if actions:
|
|
239
|
+
meta_bits.append("actions=" + ", ".join(actions))
|
|
240
|
+
if metadata.get("closing"):
|
|
241
|
+
meta_bits.append("closing=true")
|
|
242
|
+
persona = metadata.get("persona")
|
|
243
|
+
if persona:
|
|
244
|
+
meta_bits.append(f"persona={persona}")
|
|
245
|
+
meta_suffix = f" _({' ; '.join(meta_bits)})_" if meta_bits else ""
|
|
246
|
+
conversation_lines.append(f"- **{role}**: {content}{meta_suffix}")
|
|
247
|
+
conversation_lines.append("")
|
|
248
|
+
conversation_section = "\n".join(conversation_lines)
|
|
249
|
+
|
|
217
250
|
timeline_lines = ["## Timeline\n"]
|
|
218
251
|
|
|
219
252
|
for index, obs in enumerate(observations_sorted, start=1):
|
|
@@ -238,7 +271,7 @@ def _format_markdown(
|
|
|
238
271
|
if not observations_sorted:
|
|
239
272
|
timeline_lines.append("(no observations recorded)\n")
|
|
240
273
|
|
|
241
|
-
return header + summary_section + "".join(timeline_lines)
|
|
274
|
+
return header + summary_section + conversation_section + "".join(timeline_lines)
|
|
242
275
|
|
|
243
276
|
|
|
244
277
|
def _sort_observations(
|
|
@@ -278,12 +311,40 @@ def _build_structured_record(
|
|
|
278
311
|
"success": summary_payload["success"],
|
|
279
312
|
"summary": summary_payload,
|
|
280
313
|
"timeline": timeline,
|
|
314
|
+
"conversation": trace.conversation or summary_payload.get("conversation") or [],
|
|
281
315
|
}
|
|
282
316
|
|
|
317
|
+
def _maybe_decode_content(value: Any) -> Any:
|
|
318
|
+
if isinstance(value, str):
|
|
319
|
+
text = value.strip()
|
|
320
|
+
if text and text[0] in "{[" and text[-1] in "}]" and len(text) >= 2:
|
|
321
|
+
try:
|
|
322
|
+
return json.loads(text)
|
|
323
|
+
except (json.JSONDecodeError, TypeError):
|
|
324
|
+
return value
|
|
325
|
+
return value
|
|
326
|
+
|
|
283
327
|
record["metrics"] = {
|
|
284
328
|
"observation_count": len(timeline),
|
|
285
329
|
}
|
|
286
330
|
|
|
331
|
+
if trace.conversation_state:
|
|
332
|
+
record["conversation_state"] = trace.conversation_state
|
|
333
|
+
if trace.termination_reason:
|
|
334
|
+
record["termination_reason"] = trace.termination_reason
|
|
335
|
+
if not record["conversation"] and trace.raw.get("conversation"):
|
|
336
|
+
record["conversation"] = trace.raw.get("conversation")
|
|
337
|
+
if record["conversation"]:
|
|
338
|
+
normalized_conversation: List[Dict[str, Any]] = []
|
|
339
|
+
for entry in record["conversation"]:
|
|
340
|
+
if not isinstance(entry, dict):
|
|
341
|
+
normalized_conversation.append(entry)
|
|
342
|
+
continue
|
|
343
|
+
normalized_entry = dict(entry)
|
|
344
|
+
normalized_entry["content"] = _maybe_decode_content(entry.get("content"))
|
|
345
|
+
normalized_conversation.append(normalized_entry)
|
|
346
|
+
record["conversation"] = normalized_conversation
|
|
347
|
+
|
|
287
348
|
return record
|
|
288
349
|
|
|
289
350
|
|
|
@@ -3,6 +3,7 @@ Run command for executing experiments and simulations.
|
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
5
|
import asyncio
|
|
6
|
+
import json
|
|
6
7
|
import sys
|
|
7
8
|
from pathlib import Path
|
|
8
9
|
from typing import Optional
|
|
@@ -27,6 +28,16 @@ app = typer.Typer()
|
|
|
27
28
|
console = Console()
|
|
28
29
|
|
|
29
30
|
|
|
31
|
+
def _emit_progress_event(event: str, **payload: object) -> None:
|
|
32
|
+
"""Emit a structured progress event to stderr for machine consumption."""
|
|
33
|
+
data = {"event": event, **payload}
|
|
34
|
+
try:
|
|
35
|
+
print(json.dumps(data, default=str), file=sys.stderr, flush=True)
|
|
36
|
+
except Exception:
|
|
37
|
+
# Fall back to repr if serialization fails
|
|
38
|
+
print({"event": event, **{k: repr(v) for k, v in payload.items()}}, file=sys.stderr, flush=True)
|
|
39
|
+
|
|
40
|
+
|
|
30
41
|
@app.command()
|
|
31
42
|
def experiment(
|
|
32
43
|
config_file: Path = typer.Option(
|
|
@@ -227,9 +238,20 @@ def experiment(
|
|
|
227
238
|
summary.add_row("Output", config.output_directory)
|
|
228
239
|
|
|
229
240
|
console.print(summary)
|
|
241
|
+
|
|
242
|
+
_emit_progress_event(
|
|
243
|
+
"experiment_summary",
|
|
244
|
+
name=config.name,
|
|
245
|
+
iterations=config.iterations,
|
|
246
|
+
personas=len(config.personas),
|
|
247
|
+
total_runs=total_runs,
|
|
248
|
+
multi_turn_enabled=bool(config.multi_turn and config.multi_turn.enabled),
|
|
249
|
+
output_dir=config.output_directory,
|
|
250
|
+
)
|
|
230
251
|
|
|
231
252
|
if dry_run:
|
|
232
253
|
console.print("\n[yellow]Dry run mode - no execution will occur[/yellow]")
|
|
254
|
+
_emit_progress_event("experiment_status", status="dry_run", name=config.name)
|
|
233
255
|
return
|
|
234
256
|
|
|
235
257
|
# Confirm execution
|
|
@@ -243,13 +265,22 @@ def experiment(
|
|
|
243
265
|
f"\nThis will execute {total_runs} runs."
|
|
244
266
|
)
|
|
245
267
|
|
|
268
|
+
_emit_progress_event(
|
|
269
|
+
"experiment_ready",
|
|
270
|
+
name=config.name,
|
|
271
|
+
total_runs=total_runs,
|
|
272
|
+
)
|
|
273
|
+
|
|
246
274
|
if not typer.confirm("Continue?"):
|
|
275
|
+
_emit_progress_event("experiment_status", status="aborted", reason="user_declined", name=config.name)
|
|
247
276
|
raise typer.Abort()
|
|
248
277
|
|
|
249
278
|
# Create runner
|
|
250
279
|
# Run experiment with progress tracking
|
|
251
280
|
console.print("\n[bold green]▶️ Starting experiment...[/bold green]\n")
|
|
252
281
|
|
|
282
|
+
_emit_progress_event("experiment_started", name=config.name, total_runs=total_runs)
|
|
283
|
+
|
|
253
284
|
with Progress(
|
|
254
285
|
SpinnerColumn(),
|
|
255
286
|
TextColumn("[progress.description]{task.description}"),
|
|
@@ -277,6 +308,14 @@ def experiment(
|
|
|
277
308
|
|
|
278
309
|
def _progress_callback():
|
|
279
310
|
progress.advance(main_task)
|
|
311
|
+
task = progress.tasks[main_task]
|
|
312
|
+
_emit_progress_event(
|
|
313
|
+
"run_progress",
|
|
314
|
+
completed=int(task.completed),
|
|
315
|
+
total=int(task.total or 0),
|
|
316
|
+
description=task.description,
|
|
317
|
+
experiment=config.name,
|
|
318
|
+
)
|
|
280
319
|
|
|
281
320
|
def _turn_progress_callback(
|
|
282
321
|
current_turn: int,
|
|
@@ -295,6 +334,13 @@ def experiment(
|
|
|
295
334
|
description=description,
|
|
296
335
|
visible=False,
|
|
297
336
|
)
|
|
337
|
+
_emit_progress_event(
|
|
338
|
+
"turn_progress",
|
|
339
|
+
state="complete",
|
|
340
|
+
current=min(clamped_turn, total),
|
|
341
|
+
total=total,
|
|
342
|
+
experiment=config.name,
|
|
343
|
+
)
|
|
298
344
|
return
|
|
299
345
|
|
|
300
346
|
if not progress.tasks[turn_task].visible:
|
|
@@ -313,6 +359,14 @@ def experiment(
|
|
|
313
359
|
completed=completed,
|
|
314
360
|
description=description,
|
|
315
361
|
)
|
|
362
|
+
_emit_progress_event(
|
|
363
|
+
"turn_progress",
|
|
364
|
+
state="in_progress",
|
|
365
|
+
current=clamped_turn,
|
|
366
|
+
total=total,
|
|
367
|
+
preview=preview,
|
|
368
|
+
experiment=config.name,
|
|
369
|
+
)
|
|
316
370
|
|
|
317
371
|
# Run experiment
|
|
318
372
|
try:
|
|
@@ -324,12 +378,14 @@ def experiment(
|
|
|
324
378
|
)
|
|
325
379
|
except KeyboardInterrupt:
|
|
326
380
|
console.print("\n[yellow]Experiment interrupted by user[/yellow]")
|
|
381
|
+
_emit_progress_event("experiment_status", status="interrupted", name=config.name)
|
|
327
382
|
raise typer.Exit(1)
|
|
328
383
|
except Exception as e:
|
|
329
384
|
console.print(f"\n[red]Error during experiment:[/red] {e}")
|
|
330
385
|
# Debug mode - show full traceback
|
|
331
386
|
if "--debug" in sys.argv:
|
|
332
387
|
console.print_exception()
|
|
388
|
+
_emit_progress_event("experiment_status", status="error", error=str(e), name=config.name)
|
|
333
389
|
raise typer.Exit(1)
|
|
334
390
|
|
|
335
391
|
config.set_resolved_input_count(results.get("input_count", config.get_input_count()))
|
|
@@ -343,6 +399,15 @@ def experiment(
|
|
|
343
399
|
)
|
|
344
400
|
|
|
345
401
|
_display_results(results)
|
|
402
|
+
_emit_progress_event(
|
|
403
|
+
"experiment_status",
|
|
404
|
+
status="completed",
|
|
405
|
+
name=config.name,
|
|
406
|
+
total_runs=results.get("total_runs", total_runs),
|
|
407
|
+
successful=results.get("successful"),
|
|
408
|
+
failed=results.get("failed"),
|
|
409
|
+
success_rate=results.get("success_rate"),
|
|
410
|
+
)
|
|
346
411
|
|
|
347
412
|
|
|
348
413
|
@app.command()
|
|
@@ -217,6 +217,12 @@ def _write_per_trace(results: List[TraceOutcome], path: Path) -> None:
|
|
|
217
217
|
if outcome.trace.get(key) is not None
|
|
218
218
|
},
|
|
219
219
|
}
|
|
220
|
+
if outcome.trace.get("conversation") is not None:
|
|
221
|
+
payload["conversation"] = outcome.trace.get("conversation")
|
|
222
|
+
if outcome.trace.get("conversation_state") is not None:
|
|
223
|
+
payload["conversation_state"] = outcome.trace.get("conversation_state")
|
|
224
|
+
if outcome.trace.get("termination_reason") is not None:
|
|
225
|
+
payload["termination_reason"] = outcome.trace.get("termination_reason")
|
|
220
226
|
handle.write(json.dumps(payload, ensure_ascii=False) + "\n")
|
|
221
227
|
|
|
222
228
|
|