synkro 0.4.5__py3-none-any.whl

synkro/prompts/tool_templates.py

@@ -0,0 +1,201 @@
+"""Prompt templates for tool call trace generation."""
+
+# =============================================================================
+# TOOL SCENARIO GENERATION
+# =============================================================================
+
+TOOL_SCENARIO_PROMPT = """You are an expert at creating realistic scenarios that require tool usage.
+
+Given a set of available tools and usage guidelines, generate diverse scenarios that test when and how to use these tools correctly.
+
+AVAILABLE TOOLS:
+{TOOLS_DESCRIPTION}
+
+USAGE GUIDELINES:
+{GUIDELINES}
+
+Generate scenarios that cover:
+
+1. **Clear Tool Use Cases** - Situations where a specific tool is clearly needed
+2. **Tool Selection** - Scenarios requiring choosing between multiple tools
+3. **No Tool Needed** - Cases where the assistant should respond directly without tools
+4. **Multi-Tool Workflows** - Complex tasks requiring multiple tool calls
+5. **Parameter Variations** - Different parameter combinations and edge cases
+6. **Error Handling** - What to do when tools return errors or unexpected results
+
+Each scenario should include:
+- A realistic user request
+- Context about what information is available vs what needs to be looked up
+- Expected tool usage pattern (or lack thereof)
+
+Focus on creating "golden traces" - perfect examples of correct tool usage."""
+
+TOOL_CATEGORY_SCENARIO_PROMPT = """You are an expert at creating realistic scenarios for tool usage.
+
+Generate scenarios specifically for the following CATEGORY:
+**Category Name**: {CATEGORY_NAME}
+**Category Description**: {CATEGORY_DESCRIPTION}
+
+AVAILABLE TOOLS:
+{TOOLS_DESCRIPTION}
+
+USAGE GUIDELINES:
+{GUIDELINES}
+
+Create scenarios that:
+- Are deeply relevant to this specific category
+- Test the nuances of tool usage in this context
+- Include realistic user requests with appropriate context
+- Cover both happy paths and edge cases within this category"""
+
+# =============================================================================
+# TOOL RESPONSE GENERATION
+# =============================================================================
+
+TOOL_RESPONSE_PROMPT = """You are generating a training example for teaching an AI assistant to use tools correctly.
+
+AVAILABLE TOOLS:
+{TOOLS_DESCRIPTION}
+
+USAGE GUIDELINES:
+{GUIDELINES}
+
+SCENARIO:
+{SCENARIO}
+
+USER REQUEST:
+{USER_REQUEST}
+
+Generate a complete conversation that demonstrates correct tool usage:
+
+1. If a tool should be called:
+   - The assistant's first response should include appropriate tool_calls
+   - Include the simulated tool response
+   - The assistant should then synthesize the tool results into a helpful response
+
+2. If no tool is needed:
+   - The assistant should respond directly with helpful information
+   - Explain why no tool lookup was necessary
+
+The assistant should:
+- Only call tools when necessary (don't call tools for information you already know)
+- Use correct parameters with proper types
+- Wait for tool results before providing final answers
+- Synthesize tool results naturally without exposing raw data
+- Handle missing or partial information gracefully
+
+Output as JSON with this structure:
+{{
+  "messages": [
+    {{"role": "system", "content": "..."}},
+    {{"role": "user", "content": "..."}},
+    {{"role": "assistant", "content": null, "tool_calls": [...]}},  // if tool needed
+    {{"role": "tool", "tool_call_id": "...", "content": "..."}},    // tool result
+    {{"role": "assistant", "content": "..."}}                        // final response
+  ]
+}}"""
+
+# =============================================================================
+# TOOL GRADING
+# =============================================================================
+
+TOOL_GRADE_PROMPT = """You are a strict evaluator of tool usage in AI assistant responses.
+
+AVAILABLE TOOLS:
+{TOOLS_DESCRIPTION}
+
+USAGE GUIDELINES:
+{GUIDELINES}
+
+SCENARIO:
+{SCENARIO}
+
+CONVERSATION TO GRADE:
+{CONVERSATION}
+
+Evaluate the assistant's tool usage on these criteria:
+
+1. **Tool Selection** (Did they use the right tool?)
+   - Chose appropriate tool for the task
+   - Didn't use tools when not needed
+   - Used all necessary tools
+
+2. **Parameter Accuracy** (Were the parameters correct?)
+   - Correct parameter types
+   - Sensible parameter values
+   - Required parameters included
+
+3. **Response Synthesis** (Did they use tool results correctly?)
+   - Accurately incorporated tool results
+   - Didn't hallucinate beyond tool data
+   - Provided helpful, complete response
+
+4. **Timing** (Did they call tools at the right time?)
+   - Called tools before making claims
+   - Didn't call tools for known information
+   - Efficient tool call ordering
+
+A response PASSES only if ALL criteria are met.
+
+Grade this response."""
+
+# =============================================================================
+# TOOL REFINEMENT
+# =============================================================================
+
+TOOL_REFINE_PROMPT = """You are improving a tool-calling conversation that failed quality checks.
+
+AVAILABLE TOOLS:
+{TOOLS_DESCRIPTION}
+
+USAGE GUIDELINES:
+{GUIDELINES}
+
+ORIGINAL SCENARIO:
+{SCENARIO}
+
+FAILED CONVERSATION:
+{CONVERSATION}
+
+ISSUES FOUND:
+{ISSUES}
+
+GRADER FEEDBACK:
+{FEEDBACK}
+
+Generate an IMPROVED conversation that fixes all the issues while maintaining the same user request.
+
+Focus on:
+- Correct tool selection
+- Accurate parameters
+- Proper synthesis of tool results
+- No hallucination beyond tool data
+
+Output the corrected conversation as JSON."""
+
+# =============================================================================
+# TOOL SIMULATION
+# =============================================================================
+
+TOOL_SIMULATION_PROMPT = """You are simulating a tool response for training data generation.
+
+TOOL BEING CALLED:
+Name: {TOOL_NAME}
+Description: {TOOL_DESCRIPTION}
+Parameters: {TOOL_PARAMETERS}
+
+CALL ARGUMENTS:
+{ARGUMENTS}
+
+EXAMPLE RESPONSES (for reference):
+{MOCK_RESPONSES}
+
+Generate a realistic, plausible response that this tool would return for the given arguments.
+
+The response should:
+- Be realistic and internally consistent
+- Match the type of data this tool would return
+- Include appropriate detail level
+- Handle edge cases gracefully (e.g., no results found)
+
+Return only the tool response content as a string."""

synkro/quality/__init__.py

@@ -0,0 +1,14 @@
+"""Quality control components for trace grading and refinement."""
+
+from synkro.quality.grader import Grader
+from synkro.quality.refiner import Refiner
+from synkro.quality.tool_grader import ToolCallGrader
+from synkro.quality.tool_refiner import ToolCallRefiner
+
+__all__ = [
+    "Grader",
+    "Refiner",
+    "ToolCallGrader",
+    "ToolCallRefiner",
+]
+

synkro/quality/grader.py

@@ -0,0 +1,130 @@
+"""Grading of generated traces for quality control."""
+
+from synkro.llm.client import LLM
+from synkro.models import Model, OpenAI
+from synkro.types.core import Trace, GradeResult
+from synkro.prompts.templates import BATCHED_GRADER_PROMPT
+from synkro.schemas import SingleGrade
+from synkro.parsers import parse_batched_grades
+
+
+class Grader:
+    """
+    Grades generated traces for quality and policy compliance.
+
+    Uses an LLM to evaluate each trace against strict criteria:
+    - Policy compliance
+    - Proper citations
+    - Complete reasoning
+    - Actionable recommendations
+
+    Examples:
+        >>> grader = Grader()
+        >>> result = await grader.grade(trace, policy.text)
+        >>> if result.passed:
+        ...     print("Trace passes quality checks!")
+    """
+
+    def __init__(self, llm: LLM | None = None, model: Model = OpenAI.GPT_4O):
+        """
+        Initialize the grader.
+
+        Args:
+            llm: LLM client to use (creates one if not provided)
+            model: Model to use if creating LLM (recommend stronger model for grading)
+        """
+        self.llm = llm or LLM(model=model)
+
+    async def grade(self, trace: Trace, policy_text: str) -> GradeResult:
+        """
+        Grade a single trace.
+
+        Args:
+            trace: The trace to grade
+            policy_text: The policy text to grade against
+
+        Returns:
+            GradeResult with pass/fail and feedback
+        """
+        prompt = f"""You are a strict evaluator. Grade this response.
+
+A response PASSES only if ALL are true:
+1. Policy Compliant - Every recommendation follows the policy exactly
+2. Fully Supported - Every claim backed by specific policy section
+3. Properly Cited - All relevant policy sections referenced
+4. Complete Reasoning - Chain of thought has no gaps
+5. Actionable & Specific - Recommendations are concrete, not vague
+
+SCENARIO:
+{trace.scenario.description}
+
+POLICY:
+{policy_text}
+
+RESPONSE TO GRADE:
+{trace.assistant_message}
+
+Grade this response."""
+
+        try:
+            # Use structured output for reliable grading
+            parsed = await self.llm.generate_structured(prompt, SingleGrade)
+            return GradeResult(
+                passed=parsed.passed,
+                issues=(
+                    parsed.policy_violations
+                    + parsed.missing_citations
+                    + parsed.incomplete_reasoning
+                    + parsed.vague_recommendations
+                ),
+                feedback=parsed.feedback,
+            )
+        except Exception:
+            # Fallback: assume fail if we can't parse
+            return GradeResult(
+                passed=False,
+                issues=["Unable to parse grade response"],
+                feedback="Grading failed - unable to parse response",
+            )
+
+    async def grade_batch(
+        self, traces: list[Trace], policy_text: str
+    ) -> list[GradeResult]:
+        """
+        Grade multiple traces.
+
+        Args:
+            traces: List of traces to grade
+            policy_text: The policy text to grade against
+
+        Returns:
+            List of GradeResults in same order as input
+        """
+        results = []
+
+        for trace in traces:
+            result = await self.grade(trace, policy_text)
+            results.append(result)
+
+        return results
+
+    async def grade_batch_parallel(
+        self, traces: list[Trace], policy_text: str
+    ) -> list[GradeResult]:
+        """
+        Grade multiple traces in parallel.
+
+        More efficient for large batches but uses more API calls concurrently.
+
+        Args:
+            traces: List of traces to grade
+            policy_text: The policy text to grade against
+
+        Returns:
+            List of GradeResults in same order as input
+        """
+        import asyncio
+
+        tasks = [self.grade(trace, policy_text) for trace in traces]
+        return await asyncio.gather(*tasks)
+

synkro/quality/refiner.py

@@ -0,0 +1,137 @@
+"""Refinement of failed traces based on grader feedback."""
+
+from synkro.llm.client import LLM
+from synkro.models import Model, OpenAI
+from synkro.types.core import Trace, GradeResult, Message
+from synkro.prompts.templates import BATCHED_REFINER_PROMPT, SYSTEM_PROMPT
+from synkro.parsers import parse_single_response, extract_content
+
+
+class Refiner:
+    """
+    Refines traces that failed grading.
+
+    Takes failed traces and their grader feedback and generates
+    improved versions that address the issues.
+
+    Examples:
+        >>> refiner = Refiner()
+        >>> improved = await refiner.refine(failed_trace, grade_result, policy.text)
+    """
+
+    def __init__(self, llm: LLM | None = None, model: Model = OpenAI.GPT_4O_MINI):
+        """
+        Initialize the refiner.
+
+        Args:
+            llm: LLM client to use (creates one if not provided)
+            model: Model to use if creating LLM
+        """
+        self.llm = llm or LLM(model=model)
+        self.prompt_template = BATCHED_REFINER_PROMPT
+
+    async def refine(
+        self, trace: Trace, grade: GradeResult, policy_text: str
+    ) -> Trace:
+        """
+        Refine a failed trace based on grader feedback.
+
+        Args:
+            trace: The trace that failed grading
+            grade: The grade result with feedback
+            policy_text: The policy text
+
+        Returns:
+            New trace with improved response
+        """
+        prompt = self._build_prompt(trace, grade, policy_text)
+
+        response = await self.llm.generate(prompt)
+        parsed = parse_single_response(response)
+
+        if parsed and len(parsed.messages) >= 3:
+            messages = [
+                Message(role=m.role, content=m.content) for m in parsed.messages
+            ]
+        else:
+            # Fallback: construct from response
+            content = extract_content(response)
+            messages = [
+                Message(role="system", content=SYSTEM_PROMPT),
+                Message(
+                    role="user",
+                    content=f"Scenario: {trace.scenario.description}\n\nContext: {trace.scenario.context}",
+                ),
+                Message(role="assistant", content=content),
+            ]
+
+        return Trace(messages=messages, scenario=trace.scenario)
+
+    def _build_prompt(
+        self, trace: Trace, grade: GradeResult, policy_text: str
+    ) -> str:
+        """Build the refinement prompt."""
+        return f"""You are improving a response that failed quality checks.
+
+SCENARIO:
+{trace.scenario.description}
+
+CONTEXT:
+{trace.scenario.context}
+
+ORIGINAL RESPONSE:
+{trace.assistant_message}
+
+GRADER FEEDBACK:
+Issues: {', '.join(grade.issues) if grade.issues else 'None listed'}
+Summary: {grade.feedback}
+
+POLICY:
+{policy_text}
+
+Generate an IMPROVED response that fixes all the issues. Output a JSON object:
+{{
+  "messages": [
+    {{"role": "system", "content": "<system prompt>"}},
+    {{"role": "user", "content": "<the scenario>"}},
+    {{"role": "assistant", "content": "<your IMPROVED response>"}}
+  ]
+}}
+
+The improved response must:
+- Fix all policy violations
+- Add missing citations
+- Complete reasoning with no gaps
+- Make recommendations specific and actionable
+- Keep what was correct from the original
+
+Respond with ONLY the JSON object."""
+
+    async def refine_batch(
+        self,
+        traces: list[Trace],
+        grades: list[GradeResult],
+        policy_text: str,
+    ) -> list[Trace]:
+        """
+        Refine multiple failed traces.
+
+        Args:
+            traces: List of traces that failed grading
+            grades: Corresponding grade results
+            policy_text: The policy text
+
+        Returns:
+            List of refined traces
+        """
+        refined = []
+
+        for trace, grade in zip(traces, grades):
+            if not grade.passed:
+                improved = await self.refine(trace, grade, policy_text)
+                refined.append(improved)
+            else:
+                refined.append(trace)
+
+        return refined
+

synkro/quality/tool_grader.py

@@ -0,0 +1,126 @@
+"""Specialized grading for tool call traces."""
+
+import json
+from typing import TYPE_CHECKING
+
+from synkro.quality.grader import Grader
+from synkro.llm.client import LLM
+from synkro.models import Model, OpenAI
+from synkro.types.core import Trace, GradeResult
+from synkro.schemas import ToolCallGrade
+from synkro.prompts.tool_templates import TOOL_GRADE_PROMPT
+
+if TYPE_CHECKING:
+    from synkro.types.tool import ToolDefinition
+
+
+class ToolCallGrader(Grader):
+    """
+    Specialized grader for tool call traces.
+    
+    Evaluates tool usage on four criteria:
+    - Tool Selection: Did they use the right tool?
+    - Parameter Accuracy: Were the parameters correct?
+    - Response Synthesis: Did they use tool results correctly?
+    - Timing: Did they call tools at the right time?
+    
+    Examples:
+        >>> grader = ToolCallGrader(tools=[web_search, db_lookup])
+        >>> result = await grader.grade(trace, policy_text)
+        >>> if not result.passed:
+        ...     print(f"Issues: {result.issues}")
+    """
+    
+    def __init__(
+        self,
+        tools: list["ToolDefinition"],
+        llm: LLM | None = None,
+        model: Model = OpenAI.GPT_52,
+    ):
+        """
+        Initialize the tool call grader.
+        
+        Args:
+            tools: List of available tool definitions (for context)
+            llm: LLM client to use (creates one if not provided)
+            model: Model to use if creating LLM (recommend stronger model)
+        """
+        super().__init__(llm=llm, model=model)
+        self.tools = tools
+    
+    def _get_tools_description(self) -> str:
+        """Get formatted description of all tools for grading context."""
+        descriptions = []
+        for tool in self.tools:
+            descriptions.append(tool.to_system_prompt())
+        return "\n\n".join(descriptions)
+    
+    def _format_conversation(self, trace: Trace) -> str:
+        """Format the trace messages for the grading prompt, including tool_calls."""
+        lines = []
+        for msg in trace.messages:
+            if msg.role == "system":
+                lines.append(f"[SYSTEM]\n{msg.content}")
+            elif msg.role == "user":
+                lines.append(f"[USER]\n{msg.content}")
+            elif msg.role == "assistant":
+                if msg.tool_calls:
+                    # Format assistant message with tool calls
+                    tool_calls_str = []
+                    for tc in msg.tool_calls:
+                        tool_calls_str.append(
+                            f"  - {tc.function.name}({tc.function.arguments})"
+                        )
+                    lines.append(
+                        f"[ASSISTANT - TOOL CALLS]\n" + "\n".join(tool_calls_str)
+                    )
+                else:
+                    lines.append(f"[ASSISTANT]\n{msg.content}")
+            elif msg.role == "tool":
+                lines.append(
+                    f"[TOOL RESULT - {msg.tool_call_id}]\n{msg.content}"
+                )
+        return "\n\n".join(lines)
+    
+    async def grade(self, trace: Trace, policy_text: str) -> GradeResult:
+        """
+        Grade a tool call trace using tool-specific criteria.
+        
+        Args:
+            trace: The trace to grade
+            policy_text: The policy/guidelines text
+            
+        Returns:
+            GradeResult with pass/fail and detailed feedback
+        """
+        tools_desc = self._get_tools_description()
+        conversation = self._format_conversation(trace)
+        
+        prompt = TOOL_GRADE_PROMPT.format(
+            TOOLS_DESCRIPTION=tools_desc,
+            GUIDELINES=policy_text,
+            SCENARIO=trace.scenario.description,
+            CONVERSATION=conversation,
+        )
+        
+        try:
+            # Use structured output for consistent grading
+            parsed = await self.llm.generate_structured(prompt, ToolCallGrade)
+            
+            # Convert to standard GradeResult format
+            return GradeResult(
+                passed=parsed.passed,
+                issues=parsed.get_all_issues(),
+                feedback=parsed.feedback,
+            )
+        except Exception:
+            # Fallback: assume fail if we can't parse
+            return GradeResult(
+                passed=False,
+                issues=["Unable to parse grade response"],
+                feedback="Grading failed - unable to parse response",
+            )
+
+
+__all__ = ["ToolCallGrader"]
+