synkro 0.4.36__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synkro might be problematic. Click here for more details.
- synkro/__init__.py +331 -0
- synkro/advanced.py +184 -0
- synkro/cli.py +156 -0
- synkro/core/__init__.py +7 -0
- synkro/core/checkpoint.py +250 -0
- synkro/core/dataset.py +432 -0
- synkro/core/policy.py +337 -0
- synkro/errors.py +178 -0
- synkro/examples/__init__.py +148 -0
- synkro/factory.py +291 -0
- synkro/formatters/__init__.py +18 -0
- synkro/formatters/chatml.py +121 -0
- synkro/formatters/langfuse.py +98 -0
- synkro/formatters/langsmith.py +98 -0
- synkro/formatters/qa.py +112 -0
- synkro/formatters/sft.py +90 -0
- synkro/formatters/tool_call.py +127 -0
- synkro/generation/__init__.py +9 -0
- synkro/generation/follow_ups.py +134 -0
- synkro/generation/generator.py +314 -0
- synkro/generation/golden_responses.py +269 -0
- synkro/generation/golden_scenarios.py +333 -0
- synkro/generation/golden_tool_responses.py +791 -0
- synkro/generation/logic_extractor.py +126 -0
- synkro/generation/multiturn_responses.py +177 -0
- synkro/generation/planner.py +131 -0
- synkro/generation/responses.py +189 -0
- synkro/generation/scenarios.py +90 -0
- synkro/generation/tool_responses.py +625 -0
- synkro/generation/tool_simulator.py +114 -0
- synkro/interactive/__init__.py +16 -0
- synkro/interactive/hitl_session.py +205 -0
- synkro/interactive/intent_classifier.py +94 -0
- synkro/interactive/logic_map_editor.py +176 -0
- synkro/interactive/rich_ui.py +459 -0
- synkro/interactive/scenario_editor.py +198 -0
- synkro/llm/__init__.py +7 -0
- synkro/llm/client.py +309 -0
- synkro/llm/rate_limits.py +99 -0
- synkro/models/__init__.py +50 -0
- synkro/models/anthropic.py +26 -0
- synkro/models/google.py +19 -0
- synkro/models/local.py +104 -0
- synkro/models/openai.py +31 -0
- synkro/modes/__init__.py +13 -0
- synkro/modes/config.py +66 -0
- synkro/modes/conversation.py +35 -0
- synkro/modes/tool_call.py +18 -0
- synkro/parsers.py +442 -0
- synkro/pipeline/__init__.py +20 -0
- synkro/pipeline/phases.py +592 -0
- synkro/pipeline/runner.py +769 -0
- synkro/pipelines.py +136 -0
- synkro/prompts/__init__.py +57 -0
- synkro/prompts/base.py +167 -0
- synkro/prompts/golden_templates.py +533 -0
- synkro/prompts/interactive_templates.py +198 -0
- synkro/prompts/multiturn_templates.py +156 -0
- synkro/prompts/templates.py +281 -0
- synkro/prompts/tool_templates.py +318 -0
- synkro/quality/__init__.py +14 -0
- synkro/quality/golden_refiner.py +163 -0
- synkro/quality/grader.py +153 -0
- synkro/quality/multiturn_grader.py +150 -0
- synkro/quality/refiner.py +137 -0
- synkro/quality/tool_grader.py +126 -0
- synkro/quality/tool_refiner.py +128 -0
- synkro/quality/verifier.py +228 -0
- synkro/reporting.py +464 -0
- synkro/schemas.py +521 -0
- synkro/types/__init__.py +43 -0
- synkro/types/core.py +153 -0
- synkro/types/dataset_type.py +33 -0
- synkro/types/logic_map.py +348 -0
- synkro/types/tool.py +94 -0
- synkro-0.4.36.data/data/examples/__init__.py +148 -0
- synkro-0.4.36.dist-info/METADATA +507 -0
- synkro-0.4.36.dist-info/RECORD +81 -0
- synkro-0.4.36.dist-info/WHEEL +4 -0
- synkro-0.4.36.dist-info/entry_points.txt +2 -0
- synkro-0.4.36.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,459 @@
|
|
|
1
|
+
"""Rich UI components for Human-in-the-Loop interaction."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import TYPE_CHECKING
|
|
6
|
+
|
|
7
|
+
if TYPE_CHECKING:
|
|
8
|
+
from synkro.types.logic_map import LogicMap, GoldenScenario
|
|
9
|
+
from synkro.types.core import Plan
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class LogicMapDisplay:
|
|
13
|
+
"""Rich-based display for Logic Maps."""
|
|
14
|
+
|
|
15
|
+
def __init__(self) -> None:
|
|
16
|
+
from rich.console import Console
|
|
17
|
+
|
|
18
|
+
self.console = Console()
|
|
19
|
+
|
|
20
|
+
def display_full(self, logic_map: "LogicMap") -> None:
|
|
21
|
+
"""Display the complete Logic Map with all details."""
|
|
22
|
+
from rich.panel import Panel
|
|
23
|
+
from rich.tree import Tree
|
|
24
|
+
|
|
25
|
+
# Build tree view of rules by category
|
|
26
|
+
tree = Tree("[bold cyan]Logic Map[/bold cyan]")
|
|
27
|
+
|
|
28
|
+
# Group rules by category
|
|
29
|
+
categories: dict[str, list] = {}
|
|
30
|
+
for rule in logic_map.rules:
|
|
31
|
+
cat = rule.category.value if hasattr(rule.category, "value") else str(rule.category)
|
|
32
|
+
if cat not in categories:
|
|
33
|
+
categories[cat] = []
|
|
34
|
+
categories[cat].append(rule)
|
|
35
|
+
|
|
36
|
+
# Add each category as a branch
|
|
37
|
+
for category, rules in sorted(categories.items()):
|
|
38
|
+
branch = tree.add(f"[bold]{category}[/bold] ({len(rules)} rules)")
|
|
39
|
+
for rule in rules:
|
|
40
|
+
rule_text = f"[cyan]{rule.rule_id}[/cyan]: {rule.text[:60]}..."
|
|
41
|
+
if rule.dependencies:
|
|
42
|
+
rule_text += f" [dim]→ {', '.join(rule.dependencies)}[/dim]"
|
|
43
|
+
branch.add(rule_text)
|
|
44
|
+
|
|
45
|
+
# Show root rules
|
|
46
|
+
root_info = f"[dim]Root rules: {', '.join(logic_map.root_rules)}[/dim]"
|
|
47
|
+
|
|
48
|
+
self.console.print()
|
|
49
|
+
self.console.print(
|
|
50
|
+
Panel(
|
|
51
|
+
tree,
|
|
52
|
+
title=f"[bold]📜 Logic Map ({len(logic_map.rules)} rules)[/bold]",
|
|
53
|
+
subtitle=root_info,
|
|
54
|
+
border_style="cyan",
|
|
55
|
+
)
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
def display_diff(self, before: "LogicMap", after: "LogicMap") -> None:
|
|
59
|
+
"""Display all rules with changes highlighted in different colors."""
|
|
60
|
+
from rich.panel import Panel
|
|
61
|
+
from rich.tree import Tree
|
|
62
|
+
|
|
63
|
+
before_ids = {r.rule_id for r in before.rules}
|
|
64
|
+
after_ids = {r.rule_id for r in after.rules}
|
|
65
|
+
|
|
66
|
+
added = after_ids - before_ids
|
|
67
|
+
removed = before_ids - after_ids
|
|
68
|
+
common = before_ids & after_ids
|
|
69
|
+
|
|
70
|
+
# Check for modifications in common rules
|
|
71
|
+
modified: set[str] = set()
|
|
72
|
+
before_map = {r.rule_id: r for r in before.rules}
|
|
73
|
+
after_map = {r.rule_id: r for r in after.rules}
|
|
74
|
+
|
|
75
|
+
for rule_id in common:
|
|
76
|
+
if before_map[rule_id].text != after_map[rule_id].text:
|
|
77
|
+
modified.add(rule_id)
|
|
78
|
+
elif before_map[rule_id].dependencies != after_map[rule_id].dependencies:
|
|
79
|
+
modified.add(rule_id)
|
|
80
|
+
|
|
81
|
+
if not added and not removed and not modified:
|
|
82
|
+
self.console.print("[dim]No changes detected[/dim]")
|
|
83
|
+
return
|
|
84
|
+
|
|
85
|
+
# Build tree view of rules by category (like display_full but with colors)
|
|
86
|
+
tree = Tree("[bold cyan]Logic Map[/bold cyan]")
|
|
87
|
+
|
|
88
|
+
# Group rules by category
|
|
89
|
+
categories: dict[str, list] = {}
|
|
90
|
+
for rule in after.rules:
|
|
91
|
+
cat = rule.category.value if hasattr(rule.category, "value") else str(rule.category)
|
|
92
|
+
if cat not in categories:
|
|
93
|
+
categories[cat] = []
|
|
94
|
+
categories[cat].append(rule)
|
|
95
|
+
|
|
96
|
+
# Add each category as a branch
|
|
97
|
+
for category, rules in sorted(categories.items()):
|
|
98
|
+
branch = tree.add(f"[bold]{category}[/bold] ({len(rules)} rules)")
|
|
99
|
+
for rule in rules:
|
|
100
|
+
# Determine style based on change type
|
|
101
|
+
if rule.rule_id in added:
|
|
102
|
+
prefix = "[green]+ "
|
|
103
|
+
style_close = "[/green]"
|
|
104
|
+
id_style = "[green]"
|
|
105
|
+
elif rule.rule_id in modified:
|
|
106
|
+
prefix = "[yellow]~ "
|
|
107
|
+
style_close = "[/yellow]"
|
|
108
|
+
id_style = "[yellow]"
|
|
109
|
+
else:
|
|
110
|
+
prefix = ""
|
|
111
|
+
style_close = ""
|
|
112
|
+
id_style = "[cyan]"
|
|
113
|
+
|
|
114
|
+
rule_text = f"{prefix}{id_style}{rule.rule_id}[/]: {rule.text[:60]}...{style_close}"
|
|
115
|
+
if rule.dependencies:
|
|
116
|
+
rule_text += f" [dim]→ {', '.join(rule.dependencies)}[/dim]"
|
|
117
|
+
branch.add(rule_text)
|
|
118
|
+
|
|
119
|
+
# Add removed rules section at the bottom
|
|
120
|
+
if removed:
|
|
121
|
+
removed_branch = tree.add("[red][bold]REMOVED[/bold][/red]")
|
|
122
|
+
for rule_id in sorted(removed):
|
|
123
|
+
rule = before_map[rule_id]
|
|
124
|
+
removed_branch.add(f"[red][strike]- {rule_id}: {rule.text[:60]}...[/strike][/red]")
|
|
125
|
+
|
|
126
|
+
# Build legend
|
|
127
|
+
legend = "[dim]Legend: [green]+ Added[/green] | [yellow]~ Modified[/yellow] | [red][strike]- Removed[/strike][/red][/dim]"
|
|
128
|
+
|
|
129
|
+
self.console.print()
|
|
130
|
+
self.console.print(
|
|
131
|
+
Panel(
|
|
132
|
+
tree,
|
|
133
|
+
title=f"[bold]📜 Logic Map ({len(after.rules)} rules)[/bold]",
|
|
134
|
+
subtitle=legend,
|
|
135
|
+
border_style="cyan",
|
|
136
|
+
)
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
def display_rule(self, rule_id: str, logic_map: "LogicMap") -> None:
|
|
140
|
+
"""Display details of a specific rule."""
|
|
141
|
+
from rich.panel import Panel
|
|
142
|
+
|
|
143
|
+
rule = logic_map.get_rule(rule_id)
|
|
144
|
+
if not rule:
|
|
145
|
+
self.console.print(f"[red]Rule {rule_id} not found[/red]")
|
|
146
|
+
return
|
|
147
|
+
|
|
148
|
+
content = f"""[bold]ID:[/bold] {rule.rule_id}
|
|
149
|
+
[bold]Text:[/bold] {rule.text}
|
|
150
|
+
[bold]Category:[/bold] {rule.category}
|
|
151
|
+
[bold]Condition:[/bold] {rule.condition or 'N/A'}
|
|
152
|
+
[bold]Action:[/bold] {rule.action or 'N/A'}
|
|
153
|
+
[bold]Dependencies:[/bold] {', '.join(rule.dependencies) if rule.dependencies else 'None (root rule)'}"""
|
|
154
|
+
|
|
155
|
+
self.console.print(Panel(content, title=f"Rule {rule_id}", border_style="cyan"))
|
|
156
|
+
|
|
157
|
+
def show_error(self, message: str) -> None:
|
|
158
|
+
"""Display an error message."""
|
|
159
|
+
self.console.print(f"[red]Error:[/red] {message}")
|
|
160
|
+
|
|
161
|
+
def show_success(self, message: str) -> None:
|
|
162
|
+
"""Display a success message."""
|
|
163
|
+
self.console.print(f"[green]✓[/green] {message}")
|
|
164
|
+
|
|
165
|
+
def spinner(self, message: str = "Processing..."):
|
|
166
|
+
"""Context manager that shows a loading spinner.
|
|
167
|
+
|
|
168
|
+
Usage:
|
|
169
|
+
with display.spinner("Applying changes..."):
|
|
170
|
+
await some_llm_call()
|
|
171
|
+
"""
|
|
172
|
+
from rich.status import Status
|
|
173
|
+
self.console.print() # Add space above spinner
|
|
174
|
+
return Status(f"[cyan]{message}[/cyan]", spinner="dots", console=self.console)
|
|
175
|
+
|
|
176
|
+
def display_session_state(
|
|
177
|
+
self,
|
|
178
|
+
plan: "Plan",
|
|
179
|
+
logic_map: "LogicMap",
|
|
180
|
+
current_turns: int,
|
|
181
|
+
) -> None:
|
|
182
|
+
"""Display both conversation settings and logic map together."""
|
|
183
|
+
from rich.panel import Panel
|
|
184
|
+
from rich.table import Table
|
|
185
|
+
|
|
186
|
+
# Conversation settings panel
|
|
187
|
+
turns_table = Table(show_header=False, box=None)
|
|
188
|
+
turns_table.add_row(
|
|
189
|
+
"[dim]Complexity:[/dim]",
|
|
190
|
+
f"[cyan]{plan.complexity_level.title()}[/cyan]",
|
|
191
|
+
)
|
|
192
|
+
turns_table.add_row(
|
|
193
|
+
"[dim]Turns:[/dim]",
|
|
194
|
+
f"[green]{current_turns}[/green]",
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
self.console.print()
|
|
198
|
+
self.console.print(
|
|
199
|
+
Panel(
|
|
200
|
+
turns_table,
|
|
201
|
+
title="[cyan]Conversation Settings[/cyan]",
|
|
202
|
+
border_style="cyan",
|
|
203
|
+
)
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
# Then display logic map (existing method)
|
|
207
|
+
self.display_full(logic_map)
|
|
208
|
+
|
|
209
|
+
def display_scenarios(
|
|
210
|
+
self,
|
|
211
|
+
scenarios: list["GoldenScenario"],
|
|
212
|
+
distribution: dict[str, int] | None = None,
|
|
213
|
+
) -> None:
|
|
214
|
+
"""Display all scenarios with S1, S2... IDs grouped by type."""
|
|
215
|
+
from rich.panel import Panel
|
|
216
|
+
from rich.tree import Tree
|
|
217
|
+
|
|
218
|
+
# Scenario tree view grouped by type
|
|
219
|
+
tree = Tree("[bold cyan]Scenarios[/bold cyan]")
|
|
220
|
+
|
|
221
|
+
# Group by type
|
|
222
|
+
by_type: dict[str, list[tuple[int, "GoldenScenario"]]] = {}
|
|
223
|
+
for i, scenario in enumerate(scenarios, start=1):
|
|
224
|
+
type_key = scenario.scenario_type.value
|
|
225
|
+
if type_key not in by_type:
|
|
226
|
+
by_type[type_key] = []
|
|
227
|
+
by_type[type_key].append((i, scenario))
|
|
228
|
+
|
|
229
|
+
# Add each type as a branch
|
|
230
|
+
for type_name in ["positive", "negative", "edge_case", "irrelevant"]:
|
|
231
|
+
if type_name in by_type:
|
|
232
|
+
type_display = type_name.replace("_", " ").title()
|
|
233
|
+
branch = tree.add(f"[bold]{type_display}[/bold] ({len(by_type[type_name])})")
|
|
234
|
+
for idx, scenario in by_type[type_name]:
|
|
235
|
+
desc = scenario.description[:50] + "..." if len(scenario.description) > 50 else scenario.description
|
|
236
|
+
rules = ", ".join(scenario.target_rule_ids[:3]) if scenario.target_rule_ids else "None"
|
|
237
|
+
if len(scenario.target_rule_ids) > 3:
|
|
238
|
+
rules += "..."
|
|
239
|
+
branch.add(f"[cyan]S{idx}[/cyan]: {desc} [dim]→ {rules}[/dim]")
|
|
240
|
+
|
|
241
|
+
self.console.print()
|
|
242
|
+
self.console.print(
|
|
243
|
+
Panel(
|
|
244
|
+
tree,
|
|
245
|
+
title=f"[bold]📋 Scenarios ({len(scenarios)} total)[/bold]",
|
|
246
|
+
border_style="green",
|
|
247
|
+
)
|
|
248
|
+
)
|
|
249
|
+
|
|
250
|
+
def display_scenario(
|
|
251
|
+
self,
|
|
252
|
+
scenario_id: str,
|
|
253
|
+
scenarios: list["GoldenScenario"],
|
|
254
|
+
) -> None:
|
|
255
|
+
"""Display details of a specific scenario."""
|
|
256
|
+
from rich.panel import Panel
|
|
257
|
+
|
|
258
|
+
# Parse S1, S2, etc. to index
|
|
259
|
+
try:
|
|
260
|
+
idx = int(scenario_id.upper().replace("S", "")) - 1
|
|
261
|
+
if idx < 0 or idx >= len(scenarios):
|
|
262
|
+
self.console.print(f"[red]Scenario {scenario_id} not found (valid: S1-S{len(scenarios)})[/red]")
|
|
263
|
+
return
|
|
264
|
+
except ValueError:
|
|
265
|
+
self.console.print(f"[red]Invalid scenario ID: {scenario_id}[/red]")
|
|
266
|
+
return
|
|
267
|
+
|
|
268
|
+
scenario = scenarios[idx]
|
|
269
|
+
content = f"""[bold]ID:[/bold] S{idx + 1}
|
|
270
|
+
[bold]Type:[/bold] {scenario.scenario_type.value.replace('_', ' ').title()}
|
|
271
|
+
[bold]Description:[/bold] {scenario.description}
|
|
272
|
+
[bold]Context:[/bold] {scenario.context or 'N/A'}
|
|
273
|
+
[bold]Target Rules:[/bold] {', '.join(scenario.target_rule_ids) if scenario.target_rule_ids else 'None'}
|
|
274
|
+
[bold]Expected Outcome:[/bold] {scenario.expected_outcome}"""
|
|
275
|
+
|
|
276
|
+
self.console.print(Panel(content, title=f"Scenario S{idx + 1}", border_style="green"))
|
|
277
|
+
|
|
278
|
+
def display_scenario_diff(
|
|
279
|
+
self,
|
|
280
|
+
before: list["GoldenScenario"],
|
|
281
|
+
after: list["GoldenScenario"],
|
|
282
|
+
) -> None:
|
|
283
|
+
"""Display all scenarios with changes highlighted in different colors."""
|
|
284
|
+
from rich.panel import Panel
|
|
285
|
+
from rich.tree import Tree
|
|
286
|
+
|
|
287
|
+
# Create simple ID comparison
|
|
288
|
+
before_descs = {s.description for s in before}
|
|
289
|
+
after_descs = {s.description for s in after}
|
|
290
|
+
|
|
291
|
+
added_descs = after_descs - before_descs
|
|
292
|
+
removed_descs = before_descs - after_descs
|
|
293
|
+
|
|
294
|
+
if not added_descs and not removed_descs:
|
|
295
|
+
self.console.print("[dim]No changes detected[/dim]")
|
|
296
|
+
return
|
|
297
|
+
|
|
298
|
+
# Build tree view grouped by type
|
|
299
|
+
tree = Tree("[bold cyan]Scenarios[/bold cyan]")
|
|
300
|
+
|
|
301
|
+
# Group by type
|
|
302
|
+
by_type: dict[str, list[tuple[int, "GoldenScenario", bool]]] = {}
|
|
303
|
+
for i, scenario in enumerate(after, start=1):
|
|
304
|
+
type_key = scenario.scenario_type.value
|
|
305
|
+
if type_key not in by_type:
|
|
306
|
+
by_type[type_key] = []
|
|
307
|
+
is_added = scenario.description in added_descs
|
|
308
|
+
by_type[type_key].append((i, scenario, is_added))
|
|
309
|
+
|
|
310
|
+
# Add each type as a branch
|
|
311
|
+
for type_name in ["positive", "negative", "edge_case", "irrelevant"]:
|
|
312
|
+
if type_name in by_type:
|
|
313
|
+
type_display = type_name.replace("_", " ").title()
|
|
314
|
+
branch = tree.add(f"[bold]{type_display}[/bold] ({len(by_type[type_name])})")
|
|
315
|
+
for idx, scenario, is_added in by_type[type_name]:
|
|
316
|
+
desc = scenario.description[:50] + "..." if len(scenario.description) > 50 else scenario.description
|
|
317
|
+
|
|
318
|
+
if is_added:
|
|
319
|
+
prefix = "[green]+ "
|
|
320
|
+
style_close = "[/green]"
|
|
321
|
+
id_style = "[green]"
|
|
322
|
+
else:
|
|
323
|
+
prefix = ""
|
|
324
|
+
style_close = ""
|
|
325
|
+
id_style = "[cyan]"
|
|
326
|
+
|
|
327
|
+
rules = ", ".join(scenario.target_rule_ids[:3]) if scenario.target_rule_ids else "None"
|
|
328
|
+
branch.add(f"{prefix}{id_style}S{idx}[/]: {desc}{style_close} [dim]→ {rules}[/dim]")
|
|
329
|
+
|
|
330
|
+
# Add removed scenarios at bottom
|
|
331
|
+
if removed_descs:
|
|
332
|
+
removed_branch = tree.add("[red][bold]REMOVED[/bold][/red]")
|
|
333
|
+
for scenario in before:
|
|
334
|
+
if scenario.description in removed_descs:
|
|
335
|
+
desc = scenario.description[:50] + "..." if len(scenario.description) > 50 else scenario.description
|
|
336
|
+
removed_branch.add(f"[red][strike]- {desc}[/strike][/red]")
|
|
337
|
+
|
|
338
|
+
# Build legend
|
|
339
|
+
legend = "[dim]Legend: [green]+ Added[/green] | [red][strike]- Removed[/strike][/red][/dim]"
|
|
340
|
+
|
|
341
|
+
self.console.print()
|
|
342
|
+
self.console.print(
|
|
343
|
+
Panel(
|
|
344
|
+
tree,
|
|
345
|
+
title=f"[bold]📋 Scenarios ({len(after)} total)[/bold]",
|
|
346
|
+
subtitle=legend,
|
|
347
|
+
border_style="green",
|
|
348
|
+
)
|
|
349
|
+
)
|
|
350
|
+
|
|
351
|
+
def display_full_session_state(
|
|
352
|
+
self,
|
|
353
|
+
plan: "Plan",
|
|
354
|
+
logic_map: "LogicMap",
|
|
355
|
+
current_turns: int,
|
|
356
|
+
scenarios: list["GoldenScenario"] | None,
|
|
357
|
+
distribution: dict[str, int] | None,
|
|
358
|
+
) -> None:
|
|
359
|
+
"""Display logic map, scenarios, and session details (at bottom)."""
|
|
360
|
+
from rich.panel import Panel
|
|
361
|
+
|
|
362
|
+
# Display logic map first
|
|
363
|
+
self.display_full(logic_map)
|
|
364
|
+
|
|
365
|
+
# Display scenarios if available
|
|
366
|
+
if scenarios:
|
|
367
|
+
self.display_scenarios(scenarios)
|
|
368
|
+
|
|
369
|
+
# Session Details panel at bottom (instructions + settings)
|
|
370
|
+
session_details = f"""[bold]Commands:[/bold] [cyan]done[/cyan] | [cyan]undo[/cyan] | [cyan]reset[/cyan] | [cyan]show R001[/cyan] | [cyan]show S3[/cyan] | [cyan]help[/cyan]
|
|
371
|
+
|
|
372
|
+
[bold]Feedback:[/bold] [yellow]"shorter"[/yellow] [yellow]"5 turns"[/yellow] [yellow]"remove R005"[/yellow] [yellow]"add scenario for..."[/yellow] [yellow]"delete S3"[/yellow]
|
|
373
|
+
|
|
374
|
+
[dim]Complexity:[/dim] [cyan]{plan.complexity_level.title()}[/cyan] [dim]Turns:[/dim] [green]{current_turns}[/green]"""
|
|
375
|
+
|
|
376
|
+
self.console.print()
|
|
377
|
+
self.console.print(
|
|
378
|
+
Panel(
|
|
379
|
+
session_details,
|
|
380
|
+
title="[bold cyan]Session Details[/bold cyan]",
|
|
381
|
+
border_style="cyan",
|
|
382
|
+
)
|
|
383
|
+
)
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
class InteractivePrompt:
|
|
387
|
+
"""Handles user input for HITL sessions."""
|
|
388
|
+
|
|
389
|
+
def __init__(self) -> None:
|
|
390
|
+
from rich.console import Console
|
|
391
|
+
|
|
392
|
+
self.console = Console()
|
|
393
|
+
|
|
394
|
+
def show_instructions(self) -> None:
|
|
395
|
+
"""Display instructions for the HITL session."""
|
|
396
|
+
from rich.panel import Panel
|
|
397
|
+
|
|
398
|
+
instructions = """[bold]Commands:[/bold]
|
|
399
|
+
• Type feedback to modify the Logic Map (e.g., "add a rule for...", "remove R005")
|
|
400
|
+
• [cyan]done[/cyan] - Continue with current Logic Map
|
|
401
|
+
• [cyan]undo[/cyan] - Revert last change
|
|
402
|
+
• [cyan]reset[/cyan] - Restore original Logic Map
|
|
403
|
+
• [cyan]show R001[/cyan] - Show details of a specific rule
|
|
404
|
+
• [cyan]help[/cyan] - Show this message"""
|
|
405
|
+
|
|
406
|
+
self.console.print()
|
|
407
|
+
self.console.print(
|
|
408
|
+
Panel(
|
|
409
|
+
instructions,
|
|
410
|
+
title="[bold cyan]Interactive Logic Map Editor[/bold cyan]",
|
|
411
|
+
border_style="cyan",
|
|
412
|
+
)
|
|
413
|
+
)
|
|
414
|
+
|
|
415
|
+
def show_unified_instructions(self) -> None:
|
|
416
|
+
"""Display instructions for the unified HITL session (turns + rules + scenarios)."""
|
|
417
|
+
from rich.panel import Panel
|
|
418
|
+
|
|
419
|
+
instructions = """[bold]Commands:[/bold]
|
|
420
|
+
• [cyan]done[/cyan] - Continue with current settings
|
|
421
|
+
• [cyan]undo[/cyan] - Revert last change
|
|
422
|
+
• [cyan]reset[/cyan] - Restore original state
|
|
423
|
+
• [cyan]show R001[/cyan] - Show details of a specific rule
|
|
424
|
+
• [cyan]show S3[/cyan] - Show details of a specific scenario
|
|
425
|
+
• [cyan]help[/cyan] - Show this message
|
|
426
|
+
|
|
427
|
+
[bold]Feedback examples:[/bold]
|
|
428
|
+
[dim]Turns:[/dim]
|
|
429
|
+
• [yellow]"shorter conversations"[/yellow] - Adjust conversation turns
|
|
430
|
+
• [yellow]"I want 5 turns"[/yellow] - Set specific turn count
|
|
431
|
+
[dim]Rules:[/dim]
|
|
432
|
+
• [yellow]"remove R005"[/yellow] - Delete a rule
|
|
433
|
+
• [yellow]"add rule for late submissions"[/yellow] - Add a new rule
|
|
434
|
+
[dim]Scenarios:[/dim]
|
|
435
|
+
• [yellow]"add scenario for expenses at $50 limit"[/yellow] - Add edge case
|
|
436
|
+
• [yellow]"delete S3"[/yellow] - Remove a scenario
|
|
437
|
+
• [yellow]"more negative scenarios"[/yellow] - Adjust distribution"""
|
|
438
|
+
|
|
439
|
+
self.console.print()
|
|
440
|
+
self.console.print(
|
|
441
|
+
Panel(
|
|
442
|
+
instructions,
|
|
443
|
+
title="[bold cyan]Interactive Session[/bold cyan]",
|
|
444
|
+
border_style="cyan",
|
|
445
|
+
)
|
|
446
|
+
)
|
|
447
|
+
|
|
448
|
+
def get_feedback(self) -> str:
|
|
449
|
+
"""Prompt user for feedback on the Logic Map."""
|
|
450
|
+
from rich.prompt import Prompt
|
|
451
|
+
|
|
452
|
+
self.console.print()
|
|
453
|
+
return Prompt.ask("[cyan]Enter feedback[/cyan]")
|
|
454
|
+
|
|
455
|
+
def confirm_continue(self) -> bool:
|
|
456
|
+
"""Ask user if they want to continue with current Logic Map."""
|
|
457
|
+
from rich.prompt import Confirm
|
|
458
|
+
|
|
459
|
+
return Confirm.ask("Continue with this Logic Map?", default=True)
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
"""Scenario Editor - LLM-powered interactive refinement of golden scenarios."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import TYPE_CHECKING
|
|
6
|
+
|
|
7
|
+
from synkro.llm.client import LLM
|
|
8
|
+
from synkro.models import Model, OpenAI
|
|
9
|
+
from synkro.schemas import RefinedScenariosOutput
|
|
10
|
+
from synkro.types.logic_map import LogicMap, GoldenScenario, ScenarioType
|
|
11
|
+
from synkro.prompts.interactive_templates import SCENARIO_REFINEMENT_PROMPT
|
|
12
|
+
|
|
13
|
+
if TYPE_CHECKING:
|
|
14
|
+
pass
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class ScenarioEditor:
|
|
18
|
+
"""
|
|
19
|
+
LLM-powered scenario editor that interprets natural language feedback.
|
|
20
|
+
|
|
21
|
+
The editor takes user feedback in natural language (e.g., "add a scenario for...",
|
|
22
|
+
"delete S3", "more edge cases") and uses an LLM to interpret and apply
|
|
23
|
+
the changes to the scenario list.
|
|
24
|
+
|
|
25
|
+
Examples:
|
|
26
|
+
>>> editor = ScenarioEditor(llm=LLM(model=OpenAI.GPT_4O))
|
|
27
|
+
>>> new_scenarios, distribution, summary = await editor.refine(
|
|
28
|
+
... scenarios=current_scenarios,
|
|
29
|
+
... distribution=current_distribution,
|
|
30
|
+
... user_feedback="Add a scenario for expenses at exactly $50",
|
|
31
|
+
... policy_text=policy.text,
|
|
32
|
+
... logic_map=logic_map,
|
|
33
|
+
... )
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
def __init__(
|
|
37
|
+
self,
|
|
38
|
+
llm: LLM | None = None,
|
|
39
|
+
model: Model = OpenAI.GPT_4O,
|
|
40
|
+
):
|
|
41
|
+
"""
|
|
42
|
+
Initialize the Scenario Editor.
|
|
43
|
+
|
|
44
|
+
Args:
|
|
45
|
+
llm: LLM client to use (creates one if not provided)
|
|
46
|
+
model: Model to use if creating LLM (default: GPT-4O for accuracy)
|
|
47
|
+
"""
|
|
48
|
+
self.llm = llm or LLM(model=model, temperature=0.3)
|
|
49
|
+
|
|
50
|
+
async def refine(
|
|
51
|
+
self,
|
|
52
|
+
scenarios: list[GoldenScenario],
|
|
53
|
+
distribution: dict[str, int],
|
|
54
|
+
user_feedback: str,
|
|
55
|
+
policy_text: str,
|
|
56
|
+
logic_map: LogicMap,
|
|
57
|
+
conversation_history: str = "No previous feedback in this session.",
|
|
58
|
+
) -> tuple[list[GoldenScenario], dict[str, int], str]:
|
|
59
|
+
"""
|
|
60
|
+
Refine scenarios based on natural language feedback.
|
|
61
|
+
|
|
62
|
+
Args:
|
|
63
|
+
scenarios: Current list of golden scenarios
|
|
64
|
+
distribution: Current type distribution (e.g., {"positive": 5, "negative": 3})
|
|
65
|
+
user_feedback: Natural language instruction from user
|
|
66
|
+
policy_text: Original policy text for context
|
|
67
|
+
logic_map: Logic Map for rule references
|
|
68
|
+
conversation_history: Formatted history of previous feedback in this session
|
|
69
|
+
|
|
70
|
+
Returns:
|
|
71
|
+
Tuple of (refined scenarios, updated distribution, changes summary)
|
|
72
|
+
"""
|
|
73
|
+
# Format current scenarios for prompt
|
|
74
|
+
scenarios_str = self._format_scenarios_for_prompt(scenarios)
|
|
75
|
+
distribution_str = self._format_distribution(distribution)
|
|
76
|
+
logic_map_str = self._format_logic_map_for_prompt(logic_map)
|
|
77
|
+
|
|
78
|
+
# Format the prompt
|
|
79
|
+
prompt = SCENARIO_REFINEMENT_PROMPT.format(
|
|
80
|
+
logic_map=logic_map_str,
|
|
81
|
+
scenarios_formatted=scenarios_str,
|
|
82
|
+
distribution=distribution_str,
|
|
83
|
+
policy_text=policy_text,
|
|
84
|
+
user_feedback=user_feedback,
|
|
85
|
+
conversation_history=conversation_history,
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
# Generate structured output
|
|
89
|
+
result = await self.llm.generate_structured(prompt, RefinedScenariosOutput)
|
|
90
|
+
|
|
91
|
+
# Convert to domain model
|
|
92
|
+
refined_scenarios = self._convert_to_scenarios(result.scenarios)
|
|
93
|
+
|
|
94
|
+
# Calculate new distribution
|
|
95
|
+
new_distribution = self._calculate_distribution(refined_scenarios)
|
|
96
|
+
|
|
97
|
+
return refined_scenarios, new_distribution, result.changes_summary
|
|
98
|
+
|
|
99
|
+
def _format_scenarios_for_prompt(self, scenarios: list[GoldenScenario]) -> str:
|
|
100
|
+
"""Format scenarios with S1, S2... IDs for the LLM prompt."""
|
|
101
|
+
lines = []
|
|
102
|
+
for i, scenario in enumerate(scenarios, start=1):
|
|
103
|
+
lines.append(f"S{i}:")
|
|
104
|
+
lines.append(f" Description: {scenario.description}")
|
|
105
|
+
lines.append(f" Type: {scenario.scenario_type.value}")
|
|
106
|
+
lines.append(f" Target Rules: {', '.join(scenario.target_rule_ids) or 'None'}")
|
|
107
|
+
lines.append(f" Expected Outcome: {scenario.expected_outcome}")
|
|
108
|
+
if scenario.context:
|
|
109
|
+
lines.append(f" Context: {scenario.context}")
|
|
110
|
+
lines.append("")
|
|
111
|
+
return "\n".join(lines)
|
|
112
|
+
|
|
113
|
+
def _format_distribution(self, distribution: dict[str, int]) -> str:
|
|
114
|
+
"""Format distribution as a string."""
|
|
115
|
+
lines = []
|
|
116
|
+
for type_name, count in sorted(distribution.items()):
|
|
117
|
+
lines.append(f" {type_name}: {count}")
|
|
118
|
+
return "\n".join(lines)
|
|
119
|
+
|
|
120
|
+
def _format_logic_map_for_prompt(self, logic_map: LogicMap) -> str:
|
|
121
|
+
"""Format Logic Map rules for context."""
|
|
122
|
+
lines = []
|
|
123
|
+
lines.append(f"Total Rules: {len(logic_map.rules)}")
|
|
124
|
+
lines.append("Rules:")
|
|
125
|
+
for rule in logic_map.rules:
|
|
126
|
+
lines.append(f" {rule.rule_id}: {rule.text}")
|
|
127
|
+
return "\n".join(lines)
|
|
128
|
+
|
|
129
|
+
def _convert_to_scenarios(self, schema_scenarios: list) -> list[GoldenScenario]:
|
|
130
|
+
"""Convert schema output to domain model."""
|
|
131
|
+
scenarios = []
|
|
132
|
+
for s in schema_scenarios:
|
|
133
|
+
# Handle scenario_type as string or enum
|
|
134
|
+
if isinstance(s.scenario_type, str):
|
|
135
|
+
scenario_type = ScenarioType(s.scenario_type)
|
|
136
|
+
else:
|
|
137
|
+
scenario_type = s.scenario_type
|
|
138
|
+
|
|
139
|
+
scenario = GoldenScenario(
|
|
140
|
+
description=s.description,
|
|
141
|
+
context=getattr(s, 'context', ''),
|
|
142
|
+
category=getattr(s, 'category', ''),
|
|
143
|
+
scenario_type=scenario_type,
|
|
144
|
+
target_rule_ids=s.target_rule_ids,
|
|
145
|
+
expected_outcome=s.expected_outcome,
|
|
146
|
+
)
|
|
147
|
+
scenarios.append(scenario)
|
|
148
|
+
return scenarios
|
|
149
|
+
|
|
150
|
+
def _calculate_distribution(self, scenarios: list[GoldenScenario]) -> dict[str, int]:
|
|
151
|
+
"""Calculate type distribution from scenarios."""
|
|
152
|
+
distribution: dict[str, int] = {
|
|
153
|
+
"positive": 0,
|
|
154
|
+
"negative": 0,
|
|
155
|
+
"edge_case": 0,
|
|
156
|
+
"irrelevant": 0,
|
|
157
|
+
}
|
|
158
|
+
for scenario in scenarios:
|
|
159
|
+
type_key = scenario.scenario_type.value
|
|
160
|
+
distribution[type_key] = distribution.get(type_key, 0) + 1
|
|
161
|
+
return distribution
|
|
162
|
+
|
|
163
|
+
def validate_scenarios(
|
|
164
|
+
self,
|
|
165
|
+
scenarios: list[GoldenScenario],
|
|
166
|
+
logic_map: LogicMap,
|
|
167
|
+
) -> tuple[bool, list[str]]:
|
|
168
|
+
"""
|
|
169
|
+
Validate scenarios against the Logic Map.
|
|
170
|
+
|
|
171
|
+
Args:
|
|
172
|
+
scenarios: List of scenarios to validate
|
|
173
|
+
logic_map: Logic Map for rule reference validation
|
|
174
|
+
|
|
175
|
+
Returns:
|
|
176
|
+
Tuple of (is_valid, list of issue descriptions)
|
|
177
|
+
"""
|
|
178
|
+
issues = []
|
|
179
|
+
rule_ids = {r.rule_id for r in logic_map.rules}
|
|
180
|
+
|
|
181
|
+
for i, scenario in enumerate(scenarios, start=1):
|
|
182
|
+
# Check target_rule_ids reference existing rules
|
|
183
|
+
for rule_id in scenario.target_rule_ids:
|
|
184
|
+
if rule_id not in rule_ids:
|
|
185
|
+
issues.append(f"S{i} references non-existent rule {rule_id}")
|
|
186
|
+
|
|
187
|
+
# Check scenario_type is valid
|
|
188
|
+
if scenario.scenario_type.value not in ["positive", "negative", "edge_case", "irrelevant"]:
|
|
189
|
+
issues.append(f"S{i} has invalid scenario_type: {scenario.scenario_type}")
|
|
190
|
+
|
|
191
|
+
# Check expected_outcome is not empty
|
|
192
|
+
if not scenario.expected_outcome.strip():
|
|
193
|
+
issues.append(f"S{i} has empty expected_outcome")
|
|
194
|
+
|
|
195
|
+
return len(issues) == 0, issues
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
__all__ = ["ScenarioEditor"]
|