synkro 0.4.36__py3-none-any.whl

synkro/quality/multiturn_grader.py

@@ -0,0 +1,150 @@
+"""Multi-turn conversation grading with per-turn and overall evaluation."""
+
+from synkro.llm.client import LLM
+from synkro.models import Model, OpenAI
+from synkro.types.core import Trace, Message, GradeResult
+from synkro.prompts.multiturn_templates import MULTI_TURN_GRADE_PROMPT
+from synkro.schemas import ConversationGrade, TurnGrade
+
+
+class MultiTurnGrader:
+    """
+    Grades multi-turn conversations using per-turn and overall criteria.
+
+    Uses existing schemas:
+    - TurnGrade: Per-turn policy violations, citations, reasoning
+    - ConversationGrade: Overall pass, coherence, progressive depth
+
+    Examples:
+        >>> grader = MultiTurnGrader()
+        >>> result = await grader.grade(trace, policy_text)
+        >>> print(result.passed, result.feedback)
+    """
+
+    def __init__(self, llm: LLM | None = None, model: Model = OpenAI.GPT_4O):
+        """
+        Initialize the multi-turn grader.
+
+        Args:
+            llm: LLM client to use (creates one if not provided)
+            model: Model to use if creating LLM (recommend stronger model)
+        """
+        self.llm = llm or LLM(model=model)
+
+    def _count_assistant_turns(self, trace: Trace) -> int:
+        """Count the number of assistant messages (turns) in a trace."""
+        return sum(1 for m in trace.messages if m.role == "assistant")
+
+    def _format_conversation(self, messages: list[Message]) -> str:
+        """Format conversation messages for prompt inclusion."""
+        formatted = []
+        for msg in messages:
+            role = msg.role.upper()
+            content = msg.content or "[No content]"
+            formatted.append(f"{role}: {content}")
+        return "\n\n".join(formatted)
+
+    def _extract_all_issues(self, conversation_grade: ConversationGrade) -> list[str]:
+        """Extract all issues from conversation grade into flat list."""
+        issues = []
+
+        # Add coherence issues
+        issues.extend(conversation_grade.coherence_issues)
+
+        # Add per-turn issues
+        for turn_grade in conversation_grade.turn_grades:
+            issues.extend(turn_grade.policy_violations)
+            issues.extend(turn_grade.missing_citations)
+            issues.extend(turn_grade.incomplete_reasoning)
+            issues.extend(turn_grade.vague_recommendations)
+
+        return issues
+
+    async def _grade_conversation(
+        self,
+        trace: Trace,
+        policy_text: str,
+    ) -> ConversationGrade:
+        """
+        Grade the full conversation using ConversationGrade schema.
+
+        Args:
+            trace: The trace to grade
+            policy_text: The policy for evaluation
+
+        Returns:
+            ConversationGrade with per-turn and overall assessment
+        """
+        conversation = self._format_conversation(trace.messages)
+
+        prompt = f"""{MULTI_TURN_GRADE_PROMPT.format(
+            conversation=conversation,
+            policy=policy_text,
+        )}"""
+
+        try:
+            return await self.llm.generate_structured(prompt, ConversationGrade)
+        except Exception:
+            # Fallback - create a failing grade
+            num_turns = self._count_assistant_turns(trace)
+            turn_grades = [
+                TurnGrade(
+                    turn_index=i,
+                    passed=False,
+                    policy_violations=[],
+                    missing_citations=[],
+                    incomplete_reasoning=[],
+                    vague_recommendations=[],
+                    feedback="Unable to grade - parsing error",
+                )
+                for i in range(num_turns)
+            ]
+            return ConversationGrade(
+                index=0,
+                overall_pass=False,
+                turn_grades=turn_grades,
+                coherence_pass=False,
+                coherence_issues=["Unable to evaluate - grading error"],
+                progressive_depth=False,
+                overall_feedback="Grading failed - please retry",
+            )
+
+    async def grade(self, trace: Trace, policy_text: str) -> GradeResult:
+        """
+        Grade a multi-turn conversation.
+
+        Args:
+            trace: The trace to grade
+            policy_text: The policy for evaluation
+
+        Returns:
+            GradeResult with pass/fail, issues, and feedback
+        """
+        # Get full conversation grade
+        conversation_grade = await self._grade_conversation(trace, policy_text)
+
+        # Convert to standard GradeResult
+        return GradeResult(
+            passed=conversation_grade.overall_pass,
+            issues=self._extract_all_issues(conversation_grade),
+            feedback=conversation_grade.overall_feedback,
+        )
+
+    async def grade_detailed(
+        self,
+        trace: Trace,
+        policy_text: str,
+    ) -> ConversationGrade:
+        """
+        Get detailed per-turn grading for a conversation.
+
+        Use this when you need access to individual turn grades.
+
+        Args:
+            trace: The trace to grade
+            policy_text: The policy for evaluation
+
+        Returns:
+            ConversationGrade with full per-turn breakdown
+        """
+        return await self._grade_conversation(trace, policy_text)

synkro/quality/refiner.py

@@ -0,0 +1,137 @@
+"""Refinement of failed traces based on grader feedback."""
+
+from synkro.llm.client import LLM
+from synkro.models import Model, OpenAI
+from synkro.types.core import Trace, GradeResult, Message
+from synkro.prompts.templates import BATCHED_REFINER_PROMPT, SYSTEM_PROMPT
+from synkro.parsers import parse_single_response, extract_content
+
+
+class Refiner:
+    """
+    Refines traces that failed grading.
+
+    Takes failed traces and their grader feedback and generates
+    improved versions that address the issues.
+
+    Examples:
+        >>> refiner = Refiner()
+        >>> improved = await refiner.refine(failed_trace, grade_result, policy.text)
+    """
+
+    def __init__(self, llm: LLM | None = None, model: Model = OpenAI.GPT_4O_MINI):
+        """
+        Initialize the refiner.
+
+        Args:
+            llm: LLM client to use (creates one if not provided)
+            model: Model to use if creating LLM
+        """
+        self.llm = llm or LLM(model=model)
+        self.prompt_template = BATCHED_REFINER_PROMPT
+
+    async def refine(
+        self, trace: Trace, grade: GradeResult, policy_text: str
+    ) -> Trace:
+        """
+        Refine a failed trace based on grader feedback.
+
+        Args:
+            trace: The trace that failed grading
+            grade: The grade result with feedback
+            policy_text: The policy text
+
+        Returns:
+            New trace with improved response
+        """
+        prompt = self._build_prompt(trace, grade, policy_text)
+
+        response = await self.llm.generate(prompt)
+        parsed = parse_single_response(response)
+
+        if parsed and len(parsed.messages) >= 3:
+            messages = [
+                Message(role=m.role, content=m.content) for m in parsed.messages
+            ]
+        else:
+            # Fallback: construct from response
+            content = extract_content(response)
+            messages = [
+                Message(role="system", content=SYSTEM_PROMPT),
+                Message(
+                    role="user",
+                    content=f"Scenario: {trace.scenario.description}\n\nContext: {trace.scenario.context}",
+                ),
+                Message(role="assistant", content=content),
+            ]
+
+        return Trace(messages=messages, scenario=trace.scenario)
+
+    def _build_prompt(
+        self, trace: Trace, grade: GradeResult, policy_text: str
+    ) -> str:
+        """Build the refinement prompt."""
+        return f"""You are improving a response that failed quality checks.
+
+SCENARIO:
+{trace.scenario.description}
+
+CONTEXT:
+{trace.scenario.context}
+
+ORIGINAL RESPONSE:
+{trace.assistant_message}
+
+GRADER FEEDBACK:
+Issues: {', '.join(grade.issues) if grade.issues else 'None listed'}
+Summary: {grade.feedback}
+
+POLICY:
+{policy_text}
+
+Generate an IMPROVED response that fixes all the issues. Output a JSON object:
+{{
+  "messages": [
+    {{"role": "system", "content": "<system prompt>"}},
+    {{"role": "user", "content": "<the scenario>"}},
+    {{"role": "assistant", "content": "<your IMPROVED response>"}}
+  ]
+}}
+
+The improved response must:
+- Fix all policy violations
+- Add missing citations
+- Complete reasoning with no gaps
+- Make recommendations specific and actionable
+- Keep what was correct from the original
+
+Respond with ONLY the JSON object."""
+
+    async def refine_batch(
+        self,
+        traces: list[Trace],
+        grades: list[GradeResult],
+        policy_text: str,
+    ) -> list[Trace]:
+        """
+        Refine multiple failed traces.
+
+        Args:
+            traces: List of traces that failed grading
+            grades: Corresponding grade results
+            policy_text: The policy text
+
+        Returns:
+            List of refined traces
+        """
+        refined = []
+
+        for trace, grade in zip(traces, grades):
+            if not grade.passed:
+                improved = await self.refine(trace, grade, policy_text)
+                refined.append(improved)
+            else:
+                refined.append(trace)
+
+        return refined
+

synkro/quality/tool_grader.py

@@ -0,0 +1,126 @@
+"""Specialized grading for tool call traces."""
+
+import json
+from typing import TYPE_CHECKING
+
+from synkro.quality.grader import Grader
+from synkro.llm.client import LLM
+from synkro.models import Model, OpenAI
+from synkro.types.core import Trace, GradeResult
+from synkro.schemas import ToolCallGrade
+from synkro.prompts.tool_templates import TOOL_GRADE_PROMPT
+
+if TYPE_CHECKING:
+    from synkro.types.tool import ToolDefinition
+
+
+class ToolCallGrader(Grader):
+    """
+    Specialized grader for tool call traces.
+    
+    Evaluates tool usage on four criteria:
+    - Tool Selection: Did they use the right tool?
+    - Parameter Accuracy: Were the parameters correct?
+    - Response Synthesis: Did they use tool results correctly?
+    - Timing: Did they call tools at the right time?
+    
+    Examples:
+        >>> grader = ToolCallGrader(tools=[web_search, db_lookup])
+        >>> result = await grader.grade(trace, policy_text)
+        >>> if not result.passed:
+        ...     print(f"Issues: {result.issues}")
+    """
+    
+    def __init__(
+        self,
+        tools: list["ToolDefinition"],
+        llm: LLM | None = None,
+        model: Model = OpenAI.GPT_52,
+    ):
+        """
+        Initialize the tool call grader.
+        
+        Args:
+            tools: List of available tool definitions (for context)
+            llm: LLM client to use (creates one if not provided)
+            model: Model to use if creating LLM (recommend stronger model)
+        """
+        super().__init__(llm=llm, model=model)
+        self.tools = tools
+    
+    def _get_tools_description(self) -> str:
+        """Get formatted description of all tools for grading context."""
+        descriptions = []
+        for tool in self.tools:
+            descriptions.append(tool.to_system_prompt())
+        return "\n\n".join(descriptions)
+    
+    def _format_conversation(self, trace: Trace) -> str:
+        """Format the trace messages for the grading prompt, including tool_calls."""
+        lines = []
+        for msg in trace.messages:
+            if msg.role == "system":
+                lines.append(f"[SYSTEM]\n{msg.content}")
+            elif msg.role == "user":
+                lines.append(f"[USER]\n{msg.content}")
+            elif msg.role == "assistant":
+                if msg.tool_calls:
+                    # Format assistant message with tool calls
+                    tool_calls_str = []
+                    for tc in msg.tool_calls:
+                        tool_calls_str.append(
+                            f"  - {tc.function.name}({tc.function.arguments})"
+                        )
+                    lines.append(
+                        f"[ASSISTANT - TOOL CALLS]\n" + "\n".join(tool_calls_str)
+                    )
+                else:
+                    lines.append(f"[ASSISTANT]\n{msg.content}")
+            elif msg.role == "tool":
+                lines.append(
+                    f"[TOOL RESULT - {msg.tool_call_id}]\n{msg.content}"
+                )
+        return "\n\n".join(lines)
+    
+    async def grade(self, trace: Trace, policy_text: str) -> GradeResult:
+        """
+        Grade a tool call trace using tool-specific criteria.
+        
+        Args:
+            trace: The trace to grade
+            policy_text: The policy/guidelines text
+            
+        Returns:
+            GradeResult with pass/fail and detailed feedback
+        """
+        tools_desc = self._get_tools_description()
+        conversation = self._format_conversation(trace)
+        
+        prompt = TOOL_GRADE_PROMPT.format(
+            TOOLS_DESCRIPTION=tools_desc,
+            GUIDELINES=policy_text,
+            SCENARIO=trace.scenario.description,
+            CONVERSATION=conversation,
+        )
+        
+        try:
+            # Use structured output for consistent grading
+            parsed = await self.llm.generate_structured(prompt, ToolCallGrade)
+            
+            # Convert to standard GradeResult format
+            return GradeResult(
+                passed=parsed.passed,
+                issues=parsed.get_all_issues(),
+                feedback=parsed.feedback,
+            )
+        except Exception:
+            # Fallback: assume fail if we can't parse
+            return GradeResult(
+                passed=False,
+                issues=["Unable to parse grade response"],
+                feedback="Grading failed - unable to parse response",
+            )
+
+
+__all__ = ["ToolCallGrader"]
+

synkro/quality/tool_refiner.py

@@ -0,0 +1,128 @@
+"""Specialized refinement for tool call traces that preserves format."""
+
+from typing import TYPE_CHECKING
+
+from synkro.quality.refiner import Refiner
+from synkro.llm.client import LLM
+from synkro.models import Model, OpenAI
+from synkro.types.core import Trace, GradeResult, Scenario
+
+if TYPE_CHECKING:
+    from synkro.types.tool import ToolDefinition
+    from synkro.generation.tool_simulator import ToolSimulator
+
+
+class ToolCallRefiner(Refiner):
+    """
+    Specialized refiner for tool call traces.
+    
+    Unlike the base Refiner which generates plain text responses, this refiner
+    uses the ToolCallResponseGenerator to regenerate traces, ensuring the
+    tool_calls format is preserved in the output.
+    
+    The grading feedback is incorporated into the scenario context so the
+    LLM knows what to fix during regeneration.
+    
+    Examples:
+        >>> refiner = ToolCallRefiner(
+        ...     tools=[web_search, db_lookup],
+        ...     simulator=tool_simulator,
+        ... )
+        >>> improved = await refiner.refine(failed_trace, grade, policy_text)
+        >>> # improved trace has proper tool_calls format
+    """
+    
+    def __init__(
+        self,
+        tools: list["ToolDefinition"],
+        simulator: "ToolSimulator",
+        llm: LLM | None = None,
+        model: Model = OpenAI.GPT_4O_MINI,
+    ):
+        """
+        Initialize the tool call refiner.
+        
+        Args:
+            tools: List of available tool definitions
+            simulator: Tool simulator for generating tool responses
+            llm: LLM client to use (creates one if not provided)
+            model: Model to use if creating LLM
+        """
+        super().__init__(llm=llm, model=model)
+        self.tools = tools
+        self.simulator = simulator
+        self._response_generator = None
+    
+    def _get_response_generator(self):
+        """Lazily create the ToolCallResponseGenerator."""
+        if self._response_generator is None:
+            from synkro.generation.tool_responses import ToolCallResponseGenerator
+            self._response_generator = ToolCallResponseGenerator(
+                tools=self.tools,
+                llm=self.llm,
+                simulator=self.simulator,
+            )
+        return self._response_generator
+    
+    def _build_enhanced_scenario(
+        self, trace: Trace, grade: GradeResult
+    ) -> Scenario:
+        """
+        Build an enhanced scenario that includes grading feedback.
+        
+        The feedback helps the LLM understand what went wrong and how to fix it.
+        """
+        # Build feedback context
+        feedback_parts = []
+        if grade.issues:
+            feedback_parts.append("PREVIOUS ISSUES TO FIX:")
+            for issue in grade.issues:
+                feedback_parts.append(f"  - {issue}")
+        if grade.feedback:
+            feedback_parts.append(f"\nGRADER FEEDBACK: {grade.feedback}")
+        
+        feedback_context = "\n".join(feedback_parts) if feedback_parts else ""
+        
+        # Enhance the context with feedback
+        enhanced_context = trace.scenario.context
+        if feedback_context:
+            enhanced_context = f"{trace.scenario.context}\n\n--- REFINEMENT GUIDANCE ---\n{feedback_context}"
+        
+        return Scenario(
+            description=trace.scenario.description,
+            context=enhanced_context,
+            category=trace.scenario.category,
+        )
+    
+    async def refine(
+        self, trace: Trace, grade: GradeResult, policy_text: str
+    ) -> Trace:
+        """
+        Refine a failed tool call trace by regenerating with feedback.
+        
+        Uses the ToolCallResponseGenerator to ensure the regenerated trace
+        maintains proper tool_calls format.
+        
+        Args:
+            trace: The trace that failed grading
+            grade: The grade result with feedback
+            policy_text: The policy/guidelines text
+            
+        Returns:
+            New trace with improved response and preserved tool_calls format
+        """
+        # Create enhanced scenario with grading feedback
+        enhanced_scenario = self._build_enhanced_scenario(trace, grade)
+        
+        # Regenerate using ToolCallResponseGenerator (preserves format)
+        generator = self._get_response_generator()
+        refined_trace = await generator.generate_single(policy_text, enhanced_scenario)
+        
+        # Preserve the original scenario reference (without the feedback context)
+        refined_trace.scenario = trace.scenario
+        
+        return refined_trace
+
+
+__all__ = ["ToolCallRefiner"]
+