synkro 0.4.36__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of synkro might be problematic. Click here for more details.

Files changed (81) hide show
  1. synkro/__init__.py +331 -0
  2. synkro/advanced.py +184 -0
  3. synkro/cli.py +156 -0
  4. synkro/core/__init__.py +7 -0
  5. synkro/core/checkpoint.py +250 -0
  6. synkro/core/dataset.py +432 -0
  7. synkro/core/policy.py +337 -0
  8. synkro/errors.py +178 -0
  9. synkro/examples/__init__.py +148 -0
  10. synkro/factory.py +291 -0
  11. synkro/formatters/__init__.py +18 -0
  12. synkro/formatters/chatml.py +121 -0
  13. synkro/formatters/langfuse.py +98 -0
  14. synkro/formatters/langsmith.py +98 -0
  15. synkro/formatters/qa.py +112 -0
  16. synkro/formatters/sft.py +90 -0
  17. synkro/formatters/tool_call.py +127 -0
  18. synkro/generation/__init__.py +9 -0
  19. synkro/generation/follow_ups.py +134 -0
  20. synkro/generation/generator.py +314 -0
  21. synkro/generation/golden_responses.py +269 -0
  22. synkro/generation/golden_scenarios.py +333 -0
  23. synkro/generation/golden_tool_responses.py +791 -0
  24. synkro/generation/logic_extractor.py +126 -0
  25. synkro/generation/multiturn_responses.py +177 -0
  26. synkro/generation/planner.py +131 -0
  27. synkro/generation/responses.py +189 -0
  28. synkro/generation/scenarios.py +90 -0
  29. synkro/generation/tool_responses.py +625 -0
  30. synkro/generation/tool_simulator.py +114 -0
  31. synkro/interactive/__init__.py +16 -0
  32. synkro/interactive/hitl_session.py +205 -0
  33. synkro/interactive/intent_classifier.py +94 -0
  34. synkro/interactive/logic_map_editor.py +176 -0
  35. synkro/interactive/rich_ui.py +459 -0
  36. synkro/interactive/scenario_editor.py +198 -0
  37. synkro/llm/__init__.py +7 -0
  38. synkro/llm/client.py +309 -0
  39. synkro/llm/rate_limits.py +99 -0
  40. synkro/models/__init__.py +50 -0
  41. synkro/models/anthropic.py +26 -0
  42. synkro/models/google.py +19 -0
  43. synkro/models/local.py +104 -0
  44. synkro/models/openai.py +31 -0
  45. synkro/modes/__init__.py +13 -0
  46. synkro/modes/config.py +66 -0
  47. synkro/modes/conversation.py +35 -0
  48. synkro/modes/tool_call.py +18 -0
  49. synkro/parsers.py +442 -0
  50. synkro/pipeline/__init__.py +20 -0
  51. synkro/pipeline/phases.py +592 -0
  52. synkro/pipeline/runner.py +769 -0
  53. synkro/pipelines.py +136 -0
  54. synkro/prompts/__init__.py +57 -0
  55. synkro/prompts/base.py +167 -0
  56. synkro/prompts/golden_templates.py +533 -0
  57. synkro/prompts/interactive_templates.py +198 -0
  58. synkro/prompts/multiturn_templates.py +156 -0
  59. synkro/prompts/templates.py +281 -0
  60. synkro/prompts/tool_templates.py +318 -0
  61. synkro/quality/__init__.py +14 -0
  62. synkro/quality/golden_refiner.py +163 -0
  63. synkro/quality/grader.py +153 -0
  64. synkro/quality/multiturn_grader.py +150 -0
  65. synkro/quality/refiner.py +137 -0
  66. synkro/quality/tool_grader.py +126 -0
  67. synkro/quality/tool_refiner.py +128 -0
  68. synkro/quality/verifier.py +228 -0
  69. synkro/reporting.py +464 -0
  70. synkro/schemas.py +521 -0
  71. synkro/types/__init__.py +43 -0
  72. synkro/types/core.py +153 -0
  73. synkro/types/dataset_type.py +33 -0
  74. synkro/types/logic_map.py +348 -0
  75. synkro/types/tool.py +94 -0
  76. synkro-0.4.36.data/data/examples/__init__.py +148 -0
  77. synkro-0.4.36.dist-info/METADATA +507 -0
  78. synkro-0.4.36.dist-info/RECORD +81 -0
  79. synkro-0.4.36.dist-info/WHEEL +4 -0
  80. synkro-0.4.36.dist-info/entry_points.txt +2 -0
  81. synkro-0.4.36.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,459 @@
1
+ """Rich UI components for Human-in-the-Loop interaction."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import TYPE_CHECKING
6
+
7
+ if TYPE_CHECKING:
8
+ from synkro.types.logic_map import LogicMap, GoldenScenario
9
+ from synkro.types.core import Plan
10
+
11
+
12
+ class LogicMapDisplay:
13
+ """Rich-based display for Logic Maps."""
14
+
15
+ def __init__(self) -> None:
16
+ from rich.console import Console
17
+
18
+ self.console = Console()
19
+
20
+ def display_full(self, logic_map: "LogicMap") -> None:
21
+ """Display the complete Logic Map with all details."""
22
+ from rich.panel import Panel
23
+ from rich.tree import Tree
24
+
25
+ # Build tree view of rules by category
26
+ tree = Tree("[bold cyan]Logic Map[/bold cyan]")
27
+
28
+ # Group rules by category
29
+ categories: dict[str, list] = {}
30
+ for rule in logic_map.rules:
31
+ cat = rule.category.value if hasattr(rule.category, "value") else str(rule.category)
32
+ if cat not in categories:
33
+ categories[cat] = []
34
+ categories[cat].append(rule)
35
+
36
+ # Add each category as a branch
37
+ for category, rules in sorted(categories.items()):
38
+ branch = tree.add(f"[bold]{category}[/bold] ({len(rules)} rules)")
39
+ for rule in rules:
40
+ rule_text = f"[cyan]{rule.rule_id}[/cyan]: {rule.text[:60]}..."
41
+ if rule.dependencies:
42
+ rule_text += f" [dim]→ {', '.join(rule.dependencies)}[/dim]"
43
+ branch.add(rule_text)
44
+
45
+ # Show root rules
46
+ root_info = f"[dim]Root rules: {', '.join(logic_map.root_rules)}[/dim]"
47
+
48
+ self.console.print()
49
+ self.console.print(
50
+ Panel(
51
+ tree,
52
+ title=f"[bold]📜 Logic Map ({len(logic_map.rules)} rules)[/bold]",
53
+ subtitle=root_info,
54
+ border_style="cyan",
55
+ )
56
+ )
57
+
58
+ def display_diff(self, before: "LogicMap", after: "LogicMap") -> None:
59
+ """Display all rules with changes highlighted in different colors."""
60
+ from rich.panel import Panel
61
+ from rich.tree import Tree
62
+
63
+ before_ids = {r.rule_id for r in before.rules}
64
+ after_ids = {r.rule_id for r in after.rules}
65
+
66
+ added = after_ids - before_ids
67
+ removed = before_ids - after_ids
68
+ common = before_ids & after_ids
69
+
70
+ # Check for modifications in common rules
71
+ modified: set[str] = set()
72
+ before_map = {r.rule_id: r for r in before.rules}
73
+ after_map = {r.rule_id: r for r in after.rules}
74
+
75
+ for rule_id in common:
76
+ if before_map[rule_id].text != after_map[rule_id].text:
77
+ modified.add(rule_id)
78
+ elif before_map[rule_id].dependencies != after_map[rule_id].dependencies:
79
+ modified.add(rule_id)
80
+
81
+ if not added and not removed and not modified:
82
+ self.console.print("[dim]No changes detected[/dim]")
83
+ return
84
+
85
+ # Build tree view of rules by category (like display_full but with colors)
86
+ tree = Tree("[bold cyan]Logic Map[/bold cyan]")
87
+
88
+ # Group rules by category
89
+ categories: dict[str, list] = {}
90
+ for rule in after.rules:
91
+ cat = rule.category.value if hasattr(rule.category, "value") else str(rule.category)
92
+ if cat not in categories:
93
+ categories[cat] = []
94
+ categories[cat].append(rule)
95
+
96
+ # Add each category as a branch
97
+ for category, rules in sorted(categories.items()):
98
+ branch = tree.add(f"[bold]{category}[/bold] ({len(rules)} rules)")
99
+ for rule in rules:
100
+ # Determine style based on change type
101
+ if rule.rule_id in added:
102
+ prefix = "[green]+ "
103
+ style_close = "[/green]"
104
+ id_style = "[green]"
105
+ elif rule.rule_id in modified:
106
+ prefix = "[yellow]~ "
107
+ style_close = "[/yellow]"
108
+ id_style = "[yellow]"
109
+ else:
110
+ prefix = ""
111
+ style_close = ""
112
+ id_style = "[cyan]"
113
+
114
+ rule_text = f"{prefix}{id_style}{rule.rule_id}[/]: {rule.text[:60]}...{style_close}"
115
+ if rule.dependencies:
116
+ rule_text += f" [dim]→ {', '.join(rule.dependencies)}[/dim]"
117
+ branch.add(rule_text)
118
+
119
+ # Add removed rules section at the bottom
120
+ if removed:
121
+ removed_branch = tree.add("[red][bold]REMOVED[/bold][/red]")
122
+ for rule_id in sorted(removed):
123
+ rule = before_map[rule_id]
124
+ removed_branch.add(f"[red][strike]- {rule_id}: {rule.text[:60]}...[/strike][/red]")
125
+
126
+ # Build legend
127
+ legend = "[dim]Legend: [green]+ Added[/green] | [yellow]~ Modified[/yellow] | [red][strike]- Removed[/strike][/red][/dim]"
128
+
129
+ self.console.print()
130
+ self.console.print(
131
+ Panel(
132
+ tree,
133
+ title=f"[bold]📜 Logic Map ({len(after.rules)} rules)[/bold]",
134
+ subtitle=legend,
135
+ border_style="cyan",
136
+ )
137
+ )
138
+
139
+ def display_rule(self, rule_id: str, logic_map: "LogicMap") -> None:
140
+ """Display details of a specific rule."""
141
+ from rich.panel import Panel
142
+
143
+ rule = logic_map.get_rule(rule_id)
144
+ if not rule:
145
+ self.console.print(f"[red]Rule {rule_id} not found[/red]")
146
+ return
147
+
148
+ content = f"""[bold]ID:[/bold] {rule.rule_id}
149
+ [bold]Text:[/bold] {rule.text}
150
+ [bold]Category:[/bold] {rule.category}
151
+ [bold]Condition:[/bold] {rule.condition or 'N/A'}
152
+ [bold]Action:[/bold] {rule.action or 'N/A'}
153
+ [bold]Dependencies:[/bold] {', '.join(rule.dependencies) if rule.dependencies else 'None (root rule)'}"""
154
+
155
+ self.console.print(Panel(content, title=f"Rule {rule_id}", border_style="cyan"))
156
+
157
+ def show_error(self, message: str) -> None:
158
+ """Display an error message."""
159
+ self.console.print(f"[red]Error:[/red] {message}")
160
+
161
+ def show_success(self, message: str) -> None:
162
+ """Display a success message."""
163
+ self.console.print(f"[green]✓[/green] {message}")
164
+
165
+ def spinner(self, message: str = "Processing..."):
166
+ """Context manager that shows a loading spinner.
167
+
168
+ Usage:
169
+ with display.spinner("Applying changes..."):
170
+ await some_llm_call()
171
+ """
172
+ from rich.status import Status
173
+ self.console.print() # Add space above spinner
174
+ return Status(f"[cyan]{message}[/cyan]", spinner="dots", console=self.console)
175
+
176
+ def display_session_state(
177
+ self,
178
+ plan: "Plan",
179
+ logic_map: "LogicMap",
180
+ current_turns: int,
181
+ ) -> None:
182
+ """Display both conversation settings and logic map together."""
183
+ from rich.panel import Panel
184
+ from rich.table import Table
185
+
186
+ # Conversation settings panel
187
+ turns_table = Table(show_header=False, box=None)
188
+ turns_table.add_row(
189
+ "[dim]Complexity:[/dim]",
190
+ f"[cyan]{plan.complexity_level.title()}[/cyan]",
191
+ )
192
+ turns_table.add_row(
193
+ "[dim]Turns:[/dim]",
194
+ f"[green]{current_turns}[/green]",
195
+ )
196
+
197
+ self.console.print()
198
+ self.console.print(
199
+ Panel(
200
+ turns_table,
201
+ title="[cyan]Conversation Settings[/cyan]",
202
+ border_style="cyan",
203
+ )
204
+ )
205
+
206
+ # Then display logic map (existing method)
207
+ self.display_full(logic_map)
208
+
209
+ def display_scenarios(
210
+ self,
211
+ scenarios: list["GoldenScenario"],
212
+ distribution: dict[str, int] | None = None,
213
+ ) -> None:
214
+ """Display all scenarios with S1, S2... IDs grouped by type."""
215
+ from rich.panel import Panel
216
+ from rich.tree import Tree
217
+
218
+ # Scenario tree view grouped by type
219
+ tree = Tree("[bold cyan]Scenarios[/bold cyan]")
220
+
221
+ # Group by type
222
+ by_type: dict[str, list[tuple[int, "GoldenScenario"]]] = {}
223
+ for i, scenario in enumerate(scenarios, start=1):
224
+ type_key = scenario.scenario_type.value
225
+ if type_key not in by_type:
226
+ by_type[type_key] = []
227
+ by_type[type_key].append((i, scenario))
228
+
229
+ # Add each type as a branch
230
+ for type_name in ["positive", "negative", "edge_case", "irrelevant"]:
231
+ if type_name in by_type:
232
+ type_display = type_name.replace("_", " ").title()
233
+ branch = tree.add(f"[bold]{type_display}[/bold] ({len(by_type[type_name])})")
234
+ for idx, scenario in by_type[type_name]:
235
+ desc = scenario.description[:50] + "..." if len(scenario.description) > 50 else scenario.description
236
+ rules = ", ".join(scenario.target_rule_ids[:3]) if scenario.target_rule_ids else "None"
237
+ if len(scenario.target_rule_ids) > 3:
238
+ rules += "..."
239
+ branch.add(f"[cyan]S{idx}[/cyan]: {desc} [dim]→ {rules}[/dim]")
240
+
241
+ self.console.print()
242
+ self.console.print(
243
+ Panel(
244
+ tree,
245
+ title=f"[bold]📋 Scenarios ({len(scenarios)} total)[/bold]",
246
+ border_style="green",
247
+ )
248
+ )
249
+
250
+ def display_scenario(
251
+ self,
252
+ scenario_id: str,
253
+ scenarios: list["GoldenScenario"],
254
+ ) -> None:
255
+ """Display details of a specific scenario."""
256
+ from rich.panel import Panel
257
+
258
+ # Parse S1, S2, etc. to index
259
+ try:
260
+ idx = int(scenario_id.upper().replace("S", "")) - 1
261
+ if idx < 0 or idx >= len(scenarios):
262
+ self.console.print(f"[red]Scenario {scenario_id} not found (valid: S1-S{len(scenarios)})[/red]")
263
+ return
264
+ except ValueError:
265
+ self.console.print(f"[red]Invalid scenario ID: {scenario_id}[/red]")
266
+ return
267
+
268
+ scenario = scenarios[idx]
269
+ content = f"""[bold]ID:[/bold] S{idx + 1}
270
+ [bold]Type:[/bold] {scenario.scenario_type.value.replace('_', ' ').title()}
271
+ [bold]Description:[/bold] {scenario.description}
272
+ [bold]Context:[/bold] {scenario.context or 'N/A'}
273
+ [bold]Target Rules:[/bold] {', '.join(scenario.target_rule_ids) if scenario.target_rule_ids else 'None'}
274
+ [bold]Expected Outcome:[/bold] {scenario.expected_outcome}"""
275
+
276
+ self.console.print(Panel(content, title=f"Scenario S{idx + 1}", border_style="green"))
277
+
278
+ def display_scenario_diff(
279
+ self,
280
+ before: list["GoldenScenario"],
281
+ after: list["GoldenScenario"],
282
+ ) -> None:
283
+ """Display all scenarios with changes highlighted in different colors."""
284
+ from rich.panel import Panel
285
+ from rich.tree import Tree
286
+
287
+ # Create simple ID comparison
288
+ before_descs = {s.description for s in before}
289
+ after_descs = {s.description for s in after}
290
+
291
+ added_descs = after_descs - before_descs
292
+ removed_descs = before_descs - after_descs
293
+
294
+ if not added_descs and not removed_descs:
295
+ self.console.print("[dim]No changes detected[/dim]")
296
+ return
297
+
298
+ # Build tree view grouped by type
299
+ tree = Tree("[bold cyan]Scenarios[/bold cyan]")
300
+
301
+ # Group by type
302
+ by_type: dict[str, list[tuple[int, "GoldenScenario", bool]]] = {}
303
+ for i, scenario in enumerate(after, start=1):
304
+ type_key = scenario.scenario_type.value
305
+ if type_key not in by_type:
306
+ by_type[type_key] = []
307
+ is_added = scenario.description in added_descs
308
+ by_type[type_key].append((i, scenario, is_added))
309
+
310
+ # Add each type as a branch
311
+ for type_name in ["positive", "negative", "edge_case", "irrelevant"]:
312
+ if type_name in by_type:
313
+ type_display = type_name.replace("_", " ").title()
314
+ branch = tree.add(f"[bold]{type_display}[/bold] ({len(by_type[type_name])})")
315
+ for idx, scenario, is_added in by_type[type_name]:
316
+ desc = scenario.description[:50] + "..." if len(scenario.description) > 50 else scenario.description
317
+
318
+ if is_added:
319
+ prefix = "[green]+ "
320
+ style_close = "[/green]"
321
+ id_style = "[green]"
322
+ else:
323
+ prefix = ""
324
+ style_close = ""
325
+ id_style = "[cyan]"
326
+
327
+ rules = ", ".join(scenario.target_rule_ids[:3]) if scenario.target_rule_ids else "None"
328
+ branch.add(f"{prefix}{id_style}S{idx}[/]: {desc}{style_close} [dim]→ {rules}[/dim]")
329
+
330
+ # Add removed scenarios at bottom
331
+ if removed_descs:
332
+ removed_branch = tree.add("[red][bold]REMOVED[/bold][/red]")
333
+ for scenario in before:
334
+ if scenario.description in removed_descs:
335
+ desc = scenario.description[:50] + "..." if len(scenario.description) > 50 else scenario.description
336
+ removed_branch.add(f"[red][strike]- {desc}[/strike][/red]")
337
+
338
+ # Build legend
339
+ legend = "[dim]Legend: [green]+ Added[/green] | [red][strike]- Removed[/strike][/red][/dim]"
340
+
341
+ self.console.print()
342
+ self.console.print(
343
+ Panel(
344
+ tree,
345
+ title=f"[bold]📋 Scenarios ({len(after)} total)[/bold]",
346
+ subtitle=legend,
347
+ border_style="green",
348
+ )
349
+ )
350
+
351
+ def display_full_session_state(
352
+ self,
353
+ plan: "Plan",
354
+ logic_map: "LogicMap",
355
+ current_turns: int,
356
+ scenarios: list["GoldenScenario"] | None,
357
+ distribution: dict[str, int] | None,
358
+ ) -> None:
359
+ """Display logic map, scenarios, and session details (at bottom)."""
360
+ from rich.panel import Panel
361
+
362
+ # Display logic map first
363
+ self.display_full(logic_map)
364
+
365
+ # Display scenarios if available
366
+ if scenarios:
367
+ self.display_scenarios(scenarios)
368
+
369
+ # Session Details panel at bottom (instructions + settings)
370
+ session_details = f"""[bold]Commands:[/bold] [cyan]done[/cyan] | [cyan]undo[/cyan] | [cyan]reset[/cyan] | [cyan]show R001[/cyan] | [cyan]show S3[/cyan] | [cyan]help[/cyan]
371
+
372
+ [bold]Feedback:[/bold] [yellow]"shorter"[/yellow] [yellow]"5 turns"[/yellow] [yellow]"remove R005"[/yellow] [yellow]"add scenario for..."[/yellow] [yellow]"delete S3"[/yellow]
373
+
374
+ [dim]Complexity:[/dim] [cyan]{plan.complexity_level.title()}[/cyan] [dim]Turns:[/dim] [green]{current_turns}[/green]"""
375
+
376
+ self.console.print()
377
+ self.console.print(
378
+ Panel(
379
+ session_details,
380
+ title="[bold cyan]Session Details[/bold cyan]",
381
+ border_style="cyan",
382
+ )
383
+ )
384
+
385
+
386
+ class InteractivePrompt:
387
+ """Handles user input for HITL sessions."""
388
+
389
+ def __init__(self) -> None:
390
+ from rich.console import Console
391
+
392
+ self.console = Console()
393
+
394
+ def show_instructions(self) -> None:
395
+ """Display instructions for the HITL session."""
396
+ from rich.panel import Panel
397
+
398
+ instructions = """[bold]Commands:[/bold]
399
+ • Type feedback to modify the Logic Map (e.g., "add a rule for...", "remove R005")
400
+ • [cyan]done[/cyan] - Continue with current Logic Map
401
+ • [cyan]undo[/cyan] - Revert last change
402
+ • [cyan]reset[/cyan] - Restore original Logic Map
403
+ • [cyan]show R001[/cyan] - Show details of a specific rule
404
+ • [cyan]help[/cyan] - Show this message"""
405
+
406
+ self.console.print()
407
+ self.console.print(
408
+ Panel(
409
+ instructions,
410
+ title="[bold cyan]Interactive Logic Map Editor[/bold cyan]",
411
+ border_style="cyan",
412
+ )
413
+ )
414
+
415
+ def show_unified_instructions(self) -> None:
416
+ """Display instructions for the unified HITL session (turns + rules + scenarios)."""
417
+ from rich.panel import Panel
418
+
419
+ instructions = """[bold]Commands:[/bold]
420
+ • [cyan]done[/cyan] - Continue with current settings
421
+ • [cyan]undo[/cyan] - Revert last change
422
+ • [cyan]reset[/cyan] - Restore original state
423
+ • [cyan]show R001[/cyan] - Show details of a specific rule
424
+ • [cyan]show S3[/cyan] - Show details of a specific scenario
425
+ • [cyan]help[/cyan] - Show this message
426
+
427
+ [bold]Feedback examples:[/bold]
428
+ [dim]Turns:[/dim]
429
+ • [yellow]"shorter conversations"[/yellow] - Adjust conversation turns
430
+ • [yellow]"I want 5 turns"[/yellow] - Set specific turn count
431
+ [dim]Rules:[/dim]
432
+ • [yellow]"remove R005"[/yellow] - Delete a rule
433
+ • [yellow]"add rule for late submissions"[/yellow] - Add a new rule
434
+ [dim]Scenarios:[/dim]
435
+ • [yellow]"add scenario for expenses at $50 limit"[/yellow] - Add edge case
436
+ • [yellow]"delete S3"[/yellow] - Remove a scenario
437
+ • [yellow]"more negative scenarios"[/yellow] - Adjust distribution"""
438
+
439
+ self.console.print()
440
+ self.console.print(
441
+ Panel(
442
+ instructions,
443
+ title="[bold cyan]Interactive Session[/bold cyan]",
444
+ border_style="cyan",
445
+ )
446
+ )
447
+
448
+ def get_feedback(self) -> str:
449
+ """Prompt user for feedback on the Logic Map."""
450
+ from rich.prompt import Prompt
451
+
452
+ self.console.print()
453
+ return Prompt.ask("[cyan]Enter feedback[/cyan]")
454
+
455
+ def confirm_continue(self) -> bool:
456
+ """Ask user if they want to continue with current Logic Map."""
457
+ from rich.prompt import Confirm
458
+
459
+ return Confirm.ask("Continue with this Logic Map?", default=True)
@@ -0,0 +1,198 @@
1
+ """Scenario Editor - LLM-powered interactive refinement of golden scenarios."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import TYPE_CHECKING
6
+
7
+ from synkro.llm.client import LLM
8
+ from synkro.models import Model, OpenAI
9
+ from synkro.schemas import RefinedScenariosOutput
10
+ from synkro.types.logic_map import LogicMap, GoldenScenario, ScenarioType
11
+ from synkro.prompts.interactive_templates import SCENARIO_REFINEMENT_PROMPT
12
+
13
+ if TYPE_CHECKING:
14
+ pass
15
+
16
+
17
+ class ScenarioEditor:
18
+ """
19
+ LLM-powered scenario editor that interprets natural language feedback.
20
+
21
+ The editor takes user feedback in natural language (e.g., "add a scenario for...",
22
+ "delete S3", "more edge cases") and uses an LLM to interpret and apply
23
+ the changes to the scenario list.
24
+
25
+ Examples:
26
+ >>> editor = ScenarioEditor(llm=LLM(model=OpenAI.GPT_4O))
27
+ >>> new_scenarios, distribution, summary = await editor.refine(
28
+ ... scenarios=current_scenarios,
29
+ ... distribution=current_distribution,
30
+ ... user_feedback="Add a scenario for expenses at exactly $50",
31
+ ... policy_text=policy.text,
32
+ ... logic_map=logic_map,
33
+ ... )
34
+ """
35
+
36
+ def __init__(
37
+ self,
38
+ llm: LLM | None = None,
39
+ model: Model = OpenAI.GPT_4O,
40
+ ):
41
+ """
42
+ Initialize the Scenario Editor.
43
+
44
+ Args:
45
+ llm: LLM client to use (creates one if not provided)
46
+ model: Model to use if creating LLM (default: GPT-4O for accuracy)
47
+ """
48
+ self.llm = llm or LLM(model=model, temperature=0.3)
49
+
50
+ async def refine(
51
+ self,
52
+ scenarios: list[GoldenScenario],
53
+ distribution: dict[str, int],
54
+ user_feedback: str,
55
+ policy_text: str,
56
+ logic_map: LogicMap,
57
+ conversation_history: str = "No previous feedback in this session.",
58
+ ) -> tuple[list[GoldenScenario], dict[str, int], str]:
59
+ """
60
+ Refine scenarios based on natural language feedback.
61
+
62
+ Args:
63
+ scenarios: Current list of golden scenarios
64
+ distribution: Current type distribution (e.g., {"positive": 5, "negative": 3})
65
+ user_feedback: Natural language instruction from user
66
+ policy_text: Original policy text for context
67
+ logic_map: Logic Map for rule references
68
+ conversation_history: Formatted history of previous feedback in this session
69
+
70
+ Returns:
71
+ Tuple of (refined scenarios, updated distribution, changes summary)
72
+ """
73
+ # Format current scenarios for prompt
74
+ scenarios_str = self._format_scenarios_for_prompt(scenarios)
75
+ distribution_str = self._format_distribution(distribution)
76
+ logic_map_str = self._format_logic_map_for_prompt(logic_map)
77
+
78
+ # Format the prompt
79
+ prompt = SCENARIO_REFINEMENT_PROMPT.format(
80
+ logic_map=logic_map_str,
81
+ scenarios_formatted=scenarios_str,
82
+ distribution=distribution_str,
83
+ policy_text=policy_text,
84
+ user_feedback=user_feedback,
85
+ conversation_history=conversation_history,
86
+ )
87
+
88
+ # Generate structured output
89
+ result = await self.llm.generate_structured(prompt, RefinedScenariosOutput)
90
+
91
+ # Convert to domain model
92
+ refined_scenarios = self._convert_to_scenarios(result.scenarios)
93
+
94
+ # Calculate new distribution
95
+ new_distribution = self._calculate_distribution(refined_scenarios)
96
+
97
+ return refined_scenarios, new_distribution, result.changes_summary
98
+
99
+ def _format_scenarios_for_prompt(self, scenarios: list[GoldenScenario]) -> str:
100
+ """Format scenarios with S1, S2... IDs for the LLM prompt."""
101
+ lines = []
102
+ for i, scenario in enumerate(scenarios, start=1):
103
+ lines.append(f"S{i}:")
104
+ lines.append(f" Description: {scenario.description}")
105
+ lines.append(f" Type: {scenario.scenario_type.value}")
106
+ lines.append(f" Target Rules: {', '.join(scenario.target_rule_ids) or 'None'}")
107
+ lines.append(f" Expected Outcome: {scenario.expected_outcome}")
108
+ if scenario.context:
109
+ lines.append(f" Context: {scenario.context}")
110
+ lines.append("")
111
+ return "\n".join(lines)
112
+
113
+ def _format_distribution(self, distribution: dict[str, int]) -> str:
114
+ """Format distribution as a string."""
115
+ lines = []
116
+ for type_name, count in sorted(distribution.items()):
117
+ lines.append(f" {type_name}: {count}")
118
+ return "\n".join(lines)
119
+
120
+ def _format_logic_map_for_prompt(self, logic_map: LogicMap) -> str:
121
+ """Format Logic Map rules for context."""
122
+ lines = []
123
+ lines.append(f"Total Rules: {len(logic_map.rules)}")
124
+ lines.append("Rules:")
125
+ for rule in logic_map.rules:
126
+ lines.append(f" {rule.rule_id}: {rule.text}")
127
+ return "\n".join(lines)
128
+
129
+ def _convert_to_scenarios(self, schema_scenarios: list) -> list[GoldenScenario]:
130
+ """Convert schema output to domain model."""
131
+ scenarios = []
132
+ for s in schema_scenarios:
133
+ # Handle scenario_type as string or enum
134
+ if isinstance(s.scenario_type, str):
135
+ scenario_type = ScenarioType(s.scenario_type)
136
+ else:
137
+ scenario_type = s.scenario_type
138
+
139
+ scenario = GoldenScenario(
140
+ description=s.description,
141
+ context=getattr(s, 'context', ''),
142
+ category=getattr(s, 'category', ''),
143
+ scenario_type=scenario_type,
144
+ target_rule_ids=s.target_rule_ids,
145
+ expected_outcome=s.expected_outcome,
146
+ )
147
+ scenarios.append(scenario)
148
+ return scenarios
149
+
150
+ def _calculate_distribution(self, scenarios: list[GoldenScenario]) -> dict[str, int]:
151
+ """Calculate type distribution from scenarios."""
152
+ distribution: dict[str, int] = {
153
+ "positive": 0,
154
+ "negative": 0,
155
+ "edge_case": 0,
156
+ "irrelevant": 0,
157
+ }
158
+ for scenario in scenarios:
159
+ type_key = scenario.scenario_type.value
160
+ distribution[type_key] = distribution.get(type_key, 0) + 1
161
+ return distribution
162
+
163
+ def validate_scenarios(
164
+ self,
165
+ scenarios: list[GoldenScenario],
166
+ logic_map: LogicMap,
167
+ ) -> tuple[bool, list[str]]:
168
+ """
169
+ Validate scenarios against the Logic Map.
170
+
171
+ Args:
172
+ scenarios: List of scenarios to validate
173
+ logic_map: Logic Map for rule reference validation
174
+
175
+ Returns:
176
+ Tuple of (is_valid, list of issue descriptions)
177
+ """
178
+ issues = []
179
+ rule_ids = {r.rule_id for r in logic_map.rules}
180
+
181
+ for i, scenario in enumerate(scenarios, start=1):
182
+ # Check target_rule_ids reference existing rules
183
+ for rule_id in scenario.target_rule_ids:
184
+ if rule_id not in rule_ids:
185
+ issues.append(f"S{i} references non-existent rule {rule_id}")
186
+
187
+ # Check scenario_type is valid
188
+ if scenario.scenario_type.value not in ["positive", "negative", "edge_case", "irrelevant"]:
189
+ issues.append(f"S{i} has invalid scenario_type: {scenario.scenario_type}")
190
+
191
+ # Check expected_outcome is not empty
192
+ if not scenario.expected_outcome.strip():
193
+ issues.append(f"S{i} has empty expected_outcome")
194
+
195
+ return len(issues) == 0, issues
196
+
197
+
198
+ __all__ = ["ScenarioEditor"]
synkro/llm/__init__.py ADDED
@@ -0,0 +1,7 @@
1
+ """LLM client wrapper for multiple providers via LiteLLM."""
2
+
3
+ from synkro.llm.client import LLM
4
+ from synkro.llm.rate_limits import auto_workers, get_provider
5
+
6
+ __all__ = ["LLM", "auto_workers", "get_provider"]
7
+