synkro 0.4.5__py3-none-any.whl

synkro/quality/tool_refiner.py

@@ -0,0 +1,128 @@
+"""Specialized refinement for tool call traces that preserves format."""
+
+from typing import TYPE_CHECKING
+
+from synkro.quality.refiner import Refiner
+from synkro.llm.client import LLM
+from synkro.models import Model, OpenAI
+from synkro.types.core import Trace, GradeResult, Scenario
+
+if TYPE_CHECKING:
+    from synkro.types.tool import ToolDefinition
+    from synkro.generation.tool_simulator import ToolSimulator
+
+
+class ToolCallRefiner(Refiner):
+    """
+    Specialized refiner for tool call traces.
+    
+    Unlike the base Refiner which generates plain text responses, this refiner
+    uses the ToolCallResponseGenerator to regenerate traces, ensuring the
+    tool_calls format is preserved in the output.
+    
+    The grading feedback is incorporated into the scenario context so the
+    LLM knows what to fix during regeneration.
+    
+    Examples:
+        >>> refiner = ToolCallRefiner(
+        ...     tools=[web_search, db_lookup],
+        ...     simulator=tool_simulator,
+        ... )
+        >>> improved = await refiner.refine(failed_trace, grade, policy_text)
+        >>> # improved trace has proper tool_calls format
+    """
+    
+    def __init__(
+        self,
+        tools: list["ToolDefinition"],
+        simulator: "ToolSimulator",
+        llm: LLM | None = None,
+        model: Model = OpenAI.GPT_4O_MINI,
+    ):
+        """
+        Initialize the tool call refiner.
+        
+        Args:
+            tools: List of available tool definitions
+            simulator: Tool simulator for generating tool responses
+            llm: LLM client to use (creates one if not provided)
+            model: Model to use if creating LLM
+        """
+        super().__init__(llm=llm, model=model)
+        self.tools = tools
+        self.simulator = simulator
+        self._response_generator = None
+    
+    def _get_response_generator(self):
+        """Lazily create the ToolCallResponseGenerator."""
+        if self._response_generator is None:
+            from synkro.generation.tool_responses import ToolCallResponseGenerator
+            self._response_generator = ToolCallResponseGenerator(
+                tools=self.tools,
+                llm=self.llm,
+                simulator=self.simulator,
+            )
+        return self._response_generator
+    
+    def _build_enhanced_scenario(
+        self, trace: Trace, grade: GradeResult
+    ) -> Scenario:
+        """
+        Build an enhanced scenario that includes grading feedback.
+        
+        The feedback helps the LLM understand what went wrong and how to fix it.
+        """
+        # Build feedback context
+        feedback_parts = []
+        if grade.issues:
+            feedback_parts.append("PREVIOUS ISSUES TO FIX:")
+            for issue in grade.issues:
+                feedback_parts.append(f"  - {issue}")
+        if grade.feedback:
+            feedback_parts.append(f"\nGRADER FEEDBACK: {grade.feedback}")
+        
+        feedback_context = "\n".join(feedback_parts) if feedback_parts else ""
+        
+        # Enhance the context with feedback
+        enhanced_context = trace.scenario.context
+        if feedback_context:
+            enhanced_context = f"{trace.scenario.context}\n\n--- REFINEMENT GUIDANCE ---\n{feedback_context}"
+        
+        return Scenario(
+            description=trace.scenario.description,
+            context=enhanced_context,
+            category=trace.scenario.category,
+        )
+    
+    async def refine(
+        self, trace: Trace, grade: GradeResult, policy_text: str
+    ) -> Trace:
+        """
+        Refine a failed tool call trace by regenerating with feedback.
+        
+        Uses the ToolCallResponseGenerator to ensure the regenerated trace
+        maintains proper tool_calls format.
+        
+        Args:
+            trace: The trace that failed grading
+            grade: The grade result with feedback
+            policy_text: The policy/guidelines text
+            
+        Returns:
+            New trace with improved response and preserved tool_calls format
+        """
+        # Create enhanced scenario with grading feedback
+        enhanced_scenario = self._build_enhanced_scenario(trace, grade)
+        
+        # Regenerate using ToolCallResponseGenerator (preserves format)
+        generator = self._get_response_generator()
+        refined_trace = await generator.generate_single(policy_text, enhanced_scenario)
+        
+        # Preserve the original scenario reference (without the feedback context)
+        refined_trace.scenario = trace.scenario
+        
+        return refined_trace
+
+
+__all__ = ["ToolCallRefiner"]
+

synkro/reporting.py

@@ -0,0 +1,213 @@
+"""Progress reporting abstraction for generation pipeline.
+
+This module provides a Protocol for progress reporting and implementations
+for different use cases (rich console, silent for testing, etc.).
+"""
+
+from typing import Protocol
+
+from synkro.types.core import Plan, Scenario, Trace, GradeResult
+
+
+class ProgressReporter(Protocol):
+    """
+    Protocol for reporting generation progress.
+    
+    Implement this to customize how progress is displayed or logged.
+    
+    Examples:
+        >>> # Use silent reporter for testing
+        >>> generator = Generator(reporter=SilentReporter())
+        
+        >>> # Use rich reporter for CLI (default)
+        >>> generator = Generator(reporter=RichReporter())
+    """
+    
+    def on_start(self, traces: int, model: str, dataset_type: str) -> None:
+        """Called when generation starts."""
+        ...
+    
+    def on_plan_complete(self, plan: Plan) -> None:
+        """Called when planning phase completes."""
+        ...
+    
+    def on_scenario_progress(self, completed: int, total: int) -> None:
+        """Called during scenario generation."""
+        ...
+    
+    def on_scenarios_complete(self, scenarios: list[Scenario]) -> None:
+        """Called when all scenarios are generated."""
+        ...
+    
+    def on_response_progress(self, completed: int, total: int) -> None:
+        """Called during response generation."""
+        ...
+    
+    def on_responses_complete(self, traces: list[Trace]) -> None:
+        """Called when all responses are generated."""
+        ...
+    
+    def on_grading_progress(self, completed: int, total: int) -> None:
+        """Called during grading."""
+        ...
+    
+    def on_grading_complete(self, traces: list[Trace], pass_rate: float) -> None:
+        """Called when grading completes."""
+        ...
+    
+    def on_refinement_start(self, iteration: int, failed_count: int) -> None:
+        """Called when a refinement iteration starts."""
+        ...
+    
+    def on_grading_skipped(self) -> None:
+        """Called when grading is skipped."""
+        ...
+    
+    def on_complete(self, dataset_size: int, elapsed_seconds: float, pass_rate: float | None) -> None:
+        """Called when generation is complete."""
+        ...
+
+
+class SilentReporter:
+    """
+    No-op reporter for testing and embedding.
+    
+    Use this when you don't want any console output.
+    
+    Examples:
+        >>> generator = Generator(reporter=SilentReporter())
+        >>> dataset = generator.generate(policy)  # No console output
+    """
+    
+    def on_start(self, traces: int, model: str, dataset_type: str) -> None:
+        pass
+    
+    def on_plan_complete(self, plan: Plan) -> None:
+        pass
+    
+    def on_scenario_progress(self, completed: int, total: int) -> None:
+        pass
+    
+    def on_scenarios_complete(self, scenarios: list[Scenario]) -> None:
+        pass
+    
+    def on_response_progress(self, completed: int, total: int) -> None:
+        pass
+    
+    def on_responses_complete(self, traces: list[Trace]) -> None:
+        pass
+    
+    def on_grading_progress(self, completed: int, total: int) -> None:
+        pass
+    
+    def on_grading_complete(self, traces: list[Trace], pass_rate: float) -> None:
+        pass
+    
+    def on_refinement_start(self, iteration: int, failed_count: int) -> None:
+        pass
+    
+    def on_grading_skipped(self) -> None:
+        pass
+    
+    def on_complete(self, dataset_size: int, elapsed_seconds: float, pass_rate: float | None) -> None:
+        pass
+
+
+class RichReporter:
+    """
+    Rich console reporter with progress bars and formatted output.
+    
+    This is the default reporter that provides the familiar synkro CLI experience.
+    """
+    
+    def __init__(self):
+        from rich.console import Console
+        self.console = Console()
+        self._progress = None
+        self._current_task = None
+    
+    def on_start(self, traces: int, model: str, dataset_type: str) -> None:
+        from rich.panel import Panel
+        
+        self.console.print()
+        self.console.print(Panel.fit(
+            f"[bold]Generating {traces} traces[/bold]\n"
+            f"[dim]Type: {dataset_type.upper()} | Model: {model}[/dim]",
+            title="[cyan]synkro[/cyan]",
+            border_style="cyan"
+        ))
+        self.console.print()
+    
+    def on_plan_complete(self, plan: Plan) -> None:
+        from rich.table import Table
+        
+        self.console.print(f"[green]📋 Planning[/green] [dim]{len(plan.categories)} categories[/dim]")
+        
+        cat_table = Table(title="Categories", show_header=True, header_style="bold cyan")
+        cat_table.add_column("Name")
+        cat_table.add_column("Description")
+        cat_table.add_column("Count", justify="right")
+        for cat in plan.categories:
+            cat_table.add_row(cat.name, cat.description, str(cat.count))
+        self.console.print(cat_table)
+        self.console.print()
+    
+    def on_scenario_progress(self, completed: int, total: int) -> None:
+        pass  # Progress shown in on_scenarios_complete
+    
+    def on_scenarios_complete(self, scenarios: list[Scenario]) -> None:
+        self.console.print(f"[green]💡 Scenarios[/green] [dim]{len(scenarios)} created[/dim]")
+        for idx, s in enumerate(scenarios, 1):
+            desc = s.description[:80] + "..." if len(s.description) > 80 else s.description
+            self.console.print(f"  [dim]#{idx}[/dim] [yellow]{desc}[/yellow]")
+    
+    def on_response_progress(self, completed: int, total: int) -> None:
+        pass  # Progress shown in on_responses_complete
+    
+    def on_responses_complete(self, traces: list[Trace]) -> None:
+        self.console.print(f"[green]✍️  Responses[/green] [dim]{len(traces)} generated[/dim]")
+        for idx, trace in enumerate(traces, 1):
+            user_preview = trace.user_message[:60] + "..." if len(trace.user_message) > 60 else trace.user_message
+            asst_preview = trace.assistant_message[:60] + "..." if len(trace.assistant_message) > 60 else trace.assistant_message
+            self.console.print(f"  [dim]#{idx}[/dim] [blue]User:[/blue] {user_preview}")
+            self.console.print(f"       [green]Assistant:[/green] {asst_preview}")
+    
+    def on_grading_progress(self, completed: int, total: int) -> None:
+        pass  # Progress shown in on_grading_complete
+    
+    def on_grading_complete(self, traces: list[Trace], pass_rate: float) -> None:
+        self.console.print(f"[green]⚖️  Grading[/green] [dim]{pass_rate:.0f}% passed[/dim]")
+        for idx, trace in enumerate(traces, 1):
+            scenario_preview = trace.scenario.description[:40] + "..." if len(trace.scenario.description) > 40 else trace.scenario.description
+            if trace.grade and trace.grade.passed:
+                self.console.print(f"  [dim]#{idx}[/dim] [cyan]{scenario_preview}[/cyan] [green]✓ Passed[/green]")
+            else:
+                issues = ", ".join(trace.grade.issues[:2]) if trace.grade and trace.grade.issues else "No specific issues"
+                issues_preview = issues[:40] + "..." if len(issues) > 40 else issues
+                self.console.print(f"  [dim]#{idx}[/dim] [cyan]{scenario_preview}[/cyan] [red]✗ Failed[/red] [dim]{issues_preview}[/dim]")
+    
+    def on_refinement_start(self, iteration: int, failed_count: int) -> None:
+        self.console.print(f"  [yellow]↻ Refining {failed_count} failed traces (iteration {iteration})...[/yellow]")
+    
+    def on_grading_skipped(self) -> None:
+        self.console.print(f"  [dim]⚖️  Grading skipped[/dim]")
+    
+    def on_complete(self, dataset_size: int, elapsed_seconds: float, pass_rate: float | None) -> None:
+        from rich.panel import Panel
+        from rich.table import Table
+        
+        elapsed_str = f"{int(elapsed_seconds) // 60}m {int(elapsed_seconds) % 60}s" if elapsed_seconds >= 60 else f"{int(elapsed_seconds)}s"
+        
+        self.console.print()
+        summary = Table.grid(padding=(0, 2))
+        summary.add_column(style="green")
+        summary.add_column()
+        summary.add_row("✅ Done!", f"Generated {dataset_size} traces in {elapsed_str}")
+        if pass_rate is not None:
+            summary.add_row("📊 Quality:", f"{pass_rate:.0f}% passed grading")
+        self.console.print(Panel(summary, border_style="green", title="[green]Complete[/green]"))
+        self.console.print()
+
+
+__all__ = ["ProgressReporter", "SilentReporter", "RichReporter"]
+

synkro/schemas.py

@@ -0,0 +1,325 @@
+"""Pydantic schemas for structured LLM outputs and validation."""
+
+from typing import Literal
+from pydantic import BaseModel, Field
+
+
+# =============================================================================
+# SCENARIO SCHEMAS
+# =============================================================================
+
+
+class ScenarioOutput(BaseModel):
+    """Output schema for scenario generation."""
+
+    scenario: str = Field(description="Detailed scenario description")
+    context: str = Field(description="Relevant background information")
+
+
+class ScenariosArray(BaseModel):
+    """Array of generated scenarios."""
+
+    scenarios: list[ScenarioOutput]
+
+
+# =============================================================================
+# POLICY ANALYSIS SCHEMAS
+# =============================================================================
+
+
+class PolicyComplexity(BaseModel):
+    """Policy complexity analysis for auto-detecting optimal turns."""
+
+    variable_count: int = Field(
+        description="Number of variables/conditions in the policy (rules, exceptions, conditions)"
+    )
+    complexity_level: Literal["simple", "conditional", "complex"] = Field(
+        description="Overall complexity: simple (1 var), conditional (2-3 vars), complex (4+ vars)"
+    )
+    recommended_turns: int = Field(
+        ge=1, le=6, description="Recommended conversation turns based on complexity"
+    )
+    reasoning: str = Field(description="Brief explanation of the complexity assessment")
+
+
+class PlanCategory(BaseModel):
+    """A category in the generation plan."""
+
+    name: str = Field(description='Short category name (e.g., "Consent Violations", "Edge Cases")')
+    description: str = Field(description="What this category tests")
+    traces: int = Field(ge=1, description="Number of traces to generate for this category")
+
+
+class PolicyPlan(BaseModel):
+    """LLM-generated plan for dataset generation."""
+
+    categories: list[PlanCategory] = Field(
+        min_length=2, max_length=10, description="Scenario categories with trace allocations"
+    )
+    reasoning: str = Field(
+        description="Explanation of why these categories were chosen based on policy content"
+    )
+
+
+# =============================================================================
+# CHAT MESSAGE SCHEMAS
+# =============================================================================
+
+
+class ChatMessage(BaseModel):
+    """A single chat message in OpenAI format."""
+
+    role: Literal["system", "user", "assistant"] = Field(description="Message role")
+    content: str = Field(description="Message content")
+
+
+class ConversationOutput(BaseModel):
+    """Output from response generation - a complete conversation."""
+
+    index: int = Field(description="Scenario index (0-based)")
+    messages: list[ChatMessage] = Field(
+        description="Full conversation with system, user, and assistant messages"
+    )
+
+
+class BatchedConversations(BaseModel):
+    """Batch of generated conversations."""
+
+    conversations: list[ConversationOutput]
+
+
+# =============================================================================
+# GRADING SCHEMAS
+# =============================================================================
+
+
+class GradeOutput(BaseModel):
+    """Grading result for a single response."""
+
+    index: int = Field(description="Scenario index (0-based)")
+    passed: bool = Field(
+        alias="pass", description="Is the response FULLY correct, policy-compliant, and format-valid?"
+    )
+    policy_violations: list[str] = Field(
+        default_factory=list,
+        description="Specific policy rules that were violated or misinterpreted",
+    )
+    missing_citations: list[str] = Field(
+        default_factory=list,
+        description="Policy sections that should have been cited but were not",
+    )
+    incomplete_reasoning: list[str] = Field(
+        default_factory=list, description="Logical gaps or missing steps in the chain of thought"
+    )
+    vague_recommendations: list[str] = Field(
+        default_factory=list,
+        description="Recommendations that need to be more specific or actionable",
+    )
+    feedback: str = Field(description="Summary of how to fix the issues")
+
+    class Config:
+        populate_by_name = True
+
+
+class BatchedGrades(BaseModel):
+    """Batch of grading results."""
+
+    grades: list[GradeOutput]
+
+
+# =============================================================================
+# SINGLE-ITEM SCHEMAS (for parallel generation)
+# =============================================================================
+
+
+class SingleResponse(BaseModel):
+    """Single response output for parallel generation."""
+
+    messages: list[ChatMessage] = Field(
+        min_length=3, max_length=3, description="Exactly 3 messages: system, user, assistant"
+    )
+
+
+class SingleGrade(BaseModel):
+    """Single grade output for parallel generation."""
+
+    passed: bool = Field(
+        alias="pass", description="Is the response FULLY correct, policy-compliant, and format-valid?"
+    )
+    policy_violations: list[str] = Field(
+        default_factory=list, description="Specific policy rules that were violated"
+    )
+    missing_citations: list[str] = Field(
+        default_factory=list, description="Policy sections that should have been cited"
+    )
+    incomplete_reasoning: list[str] = Field(
+        default_factory=list, description="Logical gaps or missing reasoning steps"
+    )
+    vague_recommendations: list[str] = Field(
+        default_factory=list, description="Recommendations that need to be more specific"
+    )
+    feedback: str = Field(description='Summary of issues or "Correct" if passing')
+
+    class Config:
+        populate_by_name = True
+
+
+# =============================================================================
+# MULTI-TURN SCHEMAS
+# =============================================================================
+
+
+class FollowUpQuestion(BaseModel):
+    """A follow-up question for multi-turn conversations."""
+
+    index: int = Field(description="Scenario index")
+    question: str = Field(description="Follow-up question from the user")
+    question_type: Literal["clarification", "edge_case", "what_if", "specificity", "challenge"] = (
+        Field(description="Type of follow-up")
+    )
+
+
+class TurnGrade(BaseModel):
+    """Grade for a single turn in a multi-turn conversation."""
+
+    turn_index: int = Field(description="Which turn (0-based, only assistant turns)")
+    passed: bool = Field(alias="pass", description="Does this turn pass all criteria?")
+    policy_violations: list[str] = Field(
+        default_factory=list, description="Policy violations in this turn"
+    )
+    missing_citations: list[str] = Field(
+        default_factory=list, description="Missing citations in this turn"
+    )
+    incomplete_reasoning: list[str] = Field(
+        default_factory=list, description="Reasoning gaps in this turn"
+    )
+    vague_recommendations: list[str] = Field(
+        default_factory=list, description="Vague recommendations in this turn"
+    )
+    feedback: str = Field(description="Specific feedback for this turn")
+
+    class Config:
+        populate_by_name = True
+
+
+class ConversationGrade(BaseModel):
+    """Full grading for a multi-turn conversation."""
+
+    index: int = Field(description="Scenario index")
+    overall_pass: bool = Field(description="Does the ENTIRE conversation pass?")
+    turn_grades: list[TurnGrade] = Field(description="Grade for each assistant turn")
+    coherence_pass: bool = Field(
+        description="Is the conversation coherent with no contradictions?"
+    )
+    coherence_issues: list[str] = Field(
+        default_factory=list, description="Any contradictions or incoherence across turns"
+    )
+    progressive_depth: bool = Field(
+        description="Does each turn build on previous context appropriately?"
+    )
+    overall_feedback: str = Field(
+        description="Summary of what needs to be fixed across the conversation"
+    )
+
+
+# =============================================================================
+# AGENTIC SCHEMAS
+# =============================================================================
+
+
+class ToolCall(BaseModel):
+    """A tool call in an agentic trace."""
+
+    tool_name: str = Field(description="Name of the tool to call")
+    arguments: dict[str, str] = Field(description="Arguments to pass to the tool")
+
+
+class AgenticStep(BaseModel):
+    """A single step in an agentic trace."""
+
+    reasoning: str = Field(description="Reasoning before tool call")
+    tool_name: str = Field(description="Tool to call")
+    tool_args: dict = Field(description="Tool arguments")
+
+
+class AgenticTrace(BaseModel):
+    """Complete agentic trace with tool usage."""
+
+    index: int = Field(description="Scenario index")
+    steps: list[AgenticStep] = Field(description="Steps of tool usage")
+    final_answer: str = Field(description="Final comprehensive answer")
+
+
+# =============================================================================
+# TOOL CALL GRADING SCHEMAS
+# =============================================================================
+
+
+class ToolCallGrade(BaseModel):
+    """Grading result for a tool call trace.
+    
+    Evaluates tool usage on four criteria:
+    - Tool Selection: Did they use the right tool?
+    - Parameter Accuracy: Were the parameters correct?
+    - Response Synthesis: Did they use tool results correctly?
+    - Timing: Did they call tools at the right time?
+    """
+    
+    passed: bool = Field(
+        alias="pass",
+        description="Does the trace pass ALL criteria?"
+    )
+    
+    # Criterion 1: Tool Selection
+    tool_selection_correct: bool = Field(
+        description="Did the assistant choose the appropriate tool for the task?"
+    )
+    tool_selection_issues: list[str] = Field(
+        default_factory=list,
+        description="Specific issues with tool selection (wrong tool, missing tool, unnecessary tool)"
+    )
+    
+    # Criterion 2: Parameter Accuracy
+    parameters_valid: bool = Field(
+        description="Were the tool parameters correct (types, values, required fields)?"
+    )
+    parameter_issues: list[str] = Field(
+        default_factory=list,
+        description="Specific issues with parameters (wrong type, invalid value, missing required)"
+    )
+    
+    # Criterion 3: Response Synthesis
+    synthesis_accurate: bool = Field(
+        description="Did the assistant correctly use tool results without hallucination?"
+    )
+    synthesis_issues: list[str] = Field(
+        default_factory=list,
+        description="Specific issues with synthesis (hallucinated data, ignored results, misinterpreted)"
+    )
+    
+    # Criterion 4: Timing
+    timing_appropriate: bool = Field(
+        description="Did the assistant call tools at the right moment?"
+    )
+    timing_issues: list[str] = Field(
+        default_factory=list,
+        description="Specific issues with timing (premature call, delayed call, should have called earlier)"
+    )
+    
+    # Overall feedback
+    feedback: str = Field(
+        description="Summary of issues or 'Correct' if passing"
+    )
+    
+    class Config:
+        populate_by_name = True
+    
+    def get_all_issues(self) -> list[str]:
+        """Get all issues combined."""
+        return (
+            self.tool_selection_issues
+            + self.parameter_issues
+            + self.synthesis_issues
+            + self.timing_issues
+        )
+