synkro 0.4.36__py3-none-any.whl

synkro/prompts/tool_templates.py

@@ -0,0 +1,318 @@
+"""Prompt templates for tool call trace generation."""
+
+# =============================================================================
+# TOOL SCENARIO GENERATION
+# =============================================================================
+
+TOOL_SCENARIO_PROMPT = """You are an expert at creating realistic scenarios that require tool usage.
+
+Given a set of available tools and usage guidelines, generate diverse scenarios that test when and how to use these tools correctly.
+
+AVAILABLE TOOLS:
+{TOOLS_DESCRIPTION}
+
+USAGE GUIDELINES:
+{GUIDELINES}
+
+Generate scenarios that cover:
+
+1. **Clear Tool Use Cases** - Situations where a specific tool is clearly needed
+2. **Tool Selection** - Scenarios requiring choosing between multiple tools
+3. **No Tool Needed** - Cases where the assistant should respond directly without tools
+4. **Multi-Tool Workflows** - Complex tasks requiring multiple tool calls
+5. **Parameter Variations** - Different parameter combinations and edge cases
+6. **Error Handling** - What to do when tools return errors or unexpected results
+
+Each scenario should include:
+- A realistic user request
+- Context about what information is available vs what needs to be looked up
+- Expected tool usage pattern (or lack thereof)
+
+Focus on creating "golden traces" - perfect examples of correct tool usage."""
+
+TOOL_CATEGORY_SCENARIO_PROMPT = """You are an expert at creating realistic scenarios for tool usage.
+
+Generate scenarios specifically for the following CATEGORY:
+**Category Name**: {CATEGORY_NAME}
+**Category Description**: {CATEGORY_DESCRIPTION}
+
+AVAILABLE TOOLS:
+{TOOLS_DESCRIPTION}
+
+USAGE GUIDELINES:
+{GUIDELINES}
+
+Create scenarios that:
+- Are deeply relevant to this specific category
+- Test the nuances of tool usage in this context
+- Include realistic user requests with appropriate context
+- Cover both happy paths and edge cases within this category"""
+
+# =============================================================================
+# TOOL RESPONSE GENERATION
+# =============================================================================
+
+TOOL_RESPONSE_PROMPT = """You are generating a training example for teaching an AI assistant to use tools correctly.
+
+AVAILABLE TOOLS:
+{TOOLS_DESCRIPTION}
+
+USAGE GUIDELINES:
+{GUIDELINES}
+
+SCENARIO:
+{SCENARIO}
+
+USER REQUEST:
+{USER_REQUEST}
+
+Generate a complete conversation that demonstrates correct tool usage:
+
+1. If a tool should be called:
+   - The assistant's first response should include appropriate tool_calls
+   - Include the simulated tool response
+   - The assistant should then synthesize the tool results into a helpful response
+
+2. If no tool is needed:
+   - The assistant should respond directly with helpful information
+   - Explain why no tool lookup was necessary
+
+The assistant should:
+- Only call tools when necessary (don't call tools for information you already know)
+- Use correct parameters with proper types
+- Wait for tool results before providing final answers
+- Synthesize tool results naturally without exposing raw data
+- Handle missing or partial information gracefully
+
+Output as JSON with this structure:
+{{
+  "messages": [
+    {{"role": "system", "content": "..."}},
+    {{"role": "user", "content": "..."}},
+    {{"role": "assistant", "content": null, "tool_calls": [...]}},  // if tool needed
+    {{"role": "tool", "tool_call_id": "...", "content": "..."}},    // tool result
+    {{"role": "assistant", "content": "..."}}                        // final response
+  ]
+}}"""
+
+# =============================================================================
+# TOOL GRADING
+# =============================================================================
+
+TOOL_GRADE_PROMPT = """You are a strict evaluator of tool usage in AI assistant responses.
+
+AVAILABLE TOOLS:
+{TOOLS_DESCRIPTION}
+
+USAGE GUIDELINES:
+{GUIDELINES}
+
+SCENARIO:
+{SCENARIO}
+
+CONVERSATION TO GRADE:
+{CONVERSATION}
+
+Evaluate the assistant's tool usage on these criteria:
+
+1. **Tool Selection** (Did they use the right tool?)
+   - Chose appropriate tool for the task
+   - Didn't use tools when not needed
+   - Used all necessary tools
+
+2. **Parameter Accuracy** (Were the parameters correct?)
+   - Correct parameter types
+   - Sensible parameter values
+   - Required parameters included
+
+3. **Response Synthesis** (Did they use tool results correctly?)
+   - Accurately incorporated tool results
+   - Didn't hallucinate beyond tool data
+   - Provided helpful, complete response
+
+4. **Timing** (Did they call tools at the right time?)
+   - Called tools before making claims
+   - Didn't call tools for known information
+   - Efficient tool call ordering
+
+A response PASSES only if ALL criteria are met.
+
+Grade this response."""
+
+# =============================================================================
+# TOOL REFINEMENT
+# =============================================================================
+
+TOOL_REFINE_PROMPT = """You are improving a tool-calling conversation that failed quality checks.
+
+AVAILABLE TOOLS:
+{TOOLS_DESCRIPTION}
+
+USAGE GUIDELINES:
+{GUIDELINES}
+
+ORIGINAL SCENARIO:
+{SCENARIO}
+
+FAILED CONVERSATION:
+{CONVERSATION}
+
+ISSUES FOUND:
+{ISSUES}
+
+GRADER FEEDBACK:
+{FEEDBACK}
+
+Generate an IMPROVED conversation that fixes all the issues while maintaining the same user request.
+
+Focus on:
+- Correct tool selection
+- Accurate parameters
+- Proper synthesis of tool results
+- No hallucination beyond tool data
+
+Output the corrected conversation as JSON."""
+
+# =============================================================================
+# TOOL SIMULATION
+# =============================================================================
+
+TOOL_SIMULATION_PROMPT = """You are simulating a tool response for training data generation.
+
+TOOL BEING CALLED:
+Name: {TOOL_NAME}
+Description: {TOOL_DESCRIPTION}
+Parameters: {TOOL_PARAMETERS}
+
+CALL ARGUMENTS:
+{ARGUMENTS}
+
+EXAMPLE RESPONSES (for reference):
+{MOCK_RESPONSES}
+
+Generate a realistic, plausible response that this tool would return for the given arguments.
+
+The response should:
+- Be realistic and internally consistent
+- Match the type of data this tool would return
+- Include appropriate detail level
+- Handle edge cases gracefully (e.g., no results found)
+
+Return only the tool response content as a string."""
+
+# =============================================================================
+# MULTI-TURN TOOL CALLING
+# =============================================================================
+
+MULTI_TURN_TOOL_DECISION_PROMPT = """You are a customer support agent deciding whether to use tools for a follow-up question.
+
+AVAILABLE TOOLS:
+{tools_desc}
+
+TOOL USAGE GUIDELINES:
+{policy_text}
+
+CONVERSATION HISTORY (including previous tool calls and results):
+{conversation_history}
+
+NEW FOLLOW-UP QUESTION:
+{follow_up_question}
+
+Analyze this follow-up question and decide:
+1. Can this be answered using information from previous tool results?
+2. Does this require NEW tool calls?
+3. If new tools are needed, which ones and with what arguments?
+
+Important rules:
+- If previous tool results contain the needed information, DON'T call tools again
+- If the follow-up asks about something different, you MAY need new tools
+- Use correct tool names and parameter types
+- Provide clear reasoning for your decision"""
+
+MULTI_TURN_TOOL_SYNTHESIS_PROMPT = """Based on the conversation and tool results, respond to the follow-up question.
+
+CONVERSATION HISTORY:
+{conversation_history}
+
+LATEST FOLLOW-UP QUESTION:
+{follow_up_question}
+
+NEW TOOL RESULTS (if any):
+{new_tool_results}
+
+GUIDELINES:
+{policy_text}
+
+Synthesize a response that:
+- Directly addresses the follow-up question
+- Incorporates relevant information from ALL tool results (previous and new)
+- Maintains consistency with previous responses
+- Is conversational and helpful
+- Does not expose raw JSON or technical details"""
+
+# =============================================================================
+# GOLDEN MULTI-TURN TOOL CALLING
+# =============================================================================
+
+GOLDEN_MULTI_TURN_TOOL_DECISION_PROMPT = """You are a customer support agent deciding whether to use tools for a follow-up.
+Your decisions must be GROUNDED in the Logic Map rules.
+
+AVAILABLE TOOLS:
+{tools_desc}
+
+LOGIC MAP (Rules to Apply):
+{logic_map_str}
+
+POLICY GUIDELINES:
+{policy_text}
+
+CONVERSATION HISTORY (including previous tool calls and results):
+{conversation_history}
+
+RULES ALREADY APPLIED: {cumulative_rules_applied}
+
+NEW FOLLOW-UP QUESTION:
+{follow_up_question}
+
+YOUR TASK:
+1. Identify which NEW rules from the Logic Map apply to this follow-up
+2. Determine if any rule requires information that a tool can provide
+3. Consider if previous tool results satisfy the rule requirements
+4. If new tools needed, specify which rule requires each tool call
+
+TOOL CALLING RULES:
+- Only call a tool if a SPECIFIC RULE requires information not yet available
+- Cite the Rule ID that necessitates each tool call
+- If previous tool results satisfy the rule, don't call tools again
+- Explain your reasoning in terms of rule evaluation"""
+
+GOLDEN_MULTI_TURN_TOOL_SYNTHESIS_PROMPT = """Based on the conversation and tool results, respond to the follow-up question.
+Your response must be GROUNDED in the Logic Map rules.
+
+LOGIC MAP (Rules to Apply):
+{logic_map_str}
+
+CONVERSATION HISTORY:
+{conversation_history}
+
+LATEST FOLLOW-UP QUESTION:
+{follow_up_question}
+
+NEW TOOL RESULTS (if any):
+{new_tool_results}
+
+RULES ALREADY APPLIED: {cumulative_rules_applied}
+
+POLICY GUIDELINES:
+{policy_text}
+
+Synthesize a response that:
+- Directly addresses the follow-up question
+- Cites the Rule IDs that apply to this response
+- Incorporates relevant information from ALL tool results (previous and new)
+- Maintains consistency with previous responses
+- Does not expose raw JSON or technical details
+
+Also identify:
+- Which rules were applied in THIS turn's response
+- Which rules were explicitly excluded and why"""

synkro/quality/__init__.py

@@ -0,0 +1,14 @@
+"""Quality control components for trace grading and refinement."""
+
+from synkro.quality.grader import Grader
+from synkro.quality.refiner import Refiner
+from synkro.quality.tool_grader import ToolCallGrader
+from synkro.quality.tool_refiner import ToolCallRefiner
+
+__all__ = [
+    "Grader",
+    "Refiner",
+    "ToolCallGrader",
+    "ToolCallRefiner",
+]
+

synkro/quality/golden_refiner.py

@@ -0,0 +1,163 @@
+"""Golden Refiner - Refines traces that failed verification.
+
+Refines traces with Logic Map context to fix:
+- Skipped rules
+- Hallucinated rules
+- Contradictions
+- DAG violations
+"""
+
+from synkro.llm.client import LLM
+from synkro.models import Model, OpenAI
+from synkro.schemas import GoldenTraceOutput
+from synkro.types.core import Trace, Message
+from synkro.types.logic_map import LogicMap, GoldenScenario, VerificationResult
+from synkro.prompts.golden_templates import GOLDEN_REFINE_PROMPT
+
+
+class GoldenRefiner:
+    """
+    Refiner that uses Logic Map context to fix verification failures.
+
+    Addresses specific issues:
+    1. Skipped Rules: Adds evaluation of missed rules
+    2. Hallucinated Rules: Removes references to non-existent rules
+    3. Contradictions: Resolves logical inconsistencies
+    4. DAG Violations: Reorders reasoning to follow dependencies
+
+    Examples:
+        >>> refiner = GoldenRefiner(llm=LLM(model=OpenAI.GPT_4O_MINI))
+        >>> refined = await refiner.refine(trace, logic_map, scenario, verification)
+    """
+
+    def __init__(
+        self,
+        llm: LLM | None = None,
+        model: Model = OpenAI.GPT_4O_MINI,
+    ):
+        """
+        Initialize the Golden Refiner.
+
+        Args:
+            llm: LLM client to use (creates one if not provided)
+            model: Model to use if creating LLM
+        """
+        self.llm = llm or LLM(model=model, temperature=0.5)
+
+    async def refine(
+        self,
+        trace: Trace,
+        logic_map: LogicMap,
+        scenario: GoldenScenario,
+        verification: VerificationResult,
+    ) -> Trace:
+        """
+        Refine a trace that failed verification.
+
+        Args:
+            trace: The original trace that failed
+            logic_map: The Logic Map (ground truth)
+            scenario: The golden scenario
+            verification: The verification result with issues
+
+        Returns:
+            Refined trace with issues addressed
+        """
+        # Format inputs for prompt
+        logic_map_str = self._format_logic_map(logic_map)
+        original_trace_str = self._format_trace(trace)
+        verification_str = self._format_verification(verification)
+
+        # Build prompt
+        prompt = GOLDEN_REFINE_PROMPT.format(
+            original_trace=original_trace_str,
+            verification_result=verification_str,
+            logic_map=logic_map_str,
+            scenario_description=scenario.description,
+            skipped_rules=", ".join(verification.skipped_rules) if verification.skipped_rules else "None",
+            hallucinated_rules=", ".join(verification.hallucinated_rules) if verification.hallucinated_rules else "None",
+            contradictions="; ".join(verification.contradictions) if verification.contradictions else "None",
+        )
+
+        # Generate refined trace
+        result = await self.llm.generate_structured(prompt, GoldenTraceOutput)
+
+        # Convert to Trace
+        messages = [
+            Message(role=m.role, content=m.content)
+            for m in result.messages
+        ]
+
+        # Preserve scenario from original trace
+        return Trace(
+            messages=messages,
+            scenario=trace.scenario,
+        )
+
+    def _format_logic_map(self, logic_map: LogicMap) -> str:
+        """Format Logic Map for refinement prompt."""
+        lines = []
+        lines.append("RULES:")
+        for rule in logic_map.rules:
+            deps = f" [depends on: {', '.join(rule.dependencies)}]" if rule.dependencies else ""
+            lines.append(
+                f"  {rule.rule_id} ({rule.category.value}): {rule.text}{deps}"
+            )
+            lines.append(f"    IF: {rule.condition}")
+            lines.append(f"    THEN: {rule.action}")
+
+        lines.append("\nDEPENDENCY ORDER:")
+        for root_id in logic_map.root_rules:
+            chain = logic_map.get_chain(root_id)
+            if chain:
+                chain_str = " -> ".join(r.rule_id for r in chain)
+                lines.append(f"  {chain_str}")
+
+        return "\n".join(lines)
+
+    def _format_trace(self, trace: Trace) -> str:
+        """Format trace for refinement prompt."""
+        lines = []
+        for msg in trace.messages:
+            role = msg.role.upper()
+            content = msg.content or "(no content)"
+
+            # Handle tool calls
+            if msg.tool_calls:
+                tool_info = []
+                for tc in msg.tool_calls:
+                    if hasattr(tc, 'function'):
+                        tool_info.append(f"  - {tc.function.name}({tc.function.arguments})")
+                    elif isinstance(tc, dict):
+                        func = tc.get('function', {})
+                        tool_info.append(f"  - {func.get('name', 'unknown')}({func.get('arguments', '{}')})")
+                content = "Tool calls:\n" + "\n".join(tool_info)
+
+            lines.append(f"[{role}]: {content}")
+
+        return "\n\n".join(lines)
+
+    def _format_verification(self, verification: VerificationResult) -> str:
+        """Format verification result for refinement prompt."""
+        lines = []
+        lines.append(f"Passed: {verification.passed}")
+
+        if verification.issues:
+            lines.append(f"Issues: {'; '.join(verification.issues)}")
+
+        if verification.skipped_rules:
+            lines.append(f"Skipped Rules: {', '.join(verification.skipped_rules)}")
+
+        if verification.hallucinated_rules:
+            lines.append(f"Hallucinated Rules: {', '.join(verification.hallucinated_rules)}")
+
+        if verification.contradictions:
+            lines.append(f"Contradictions: {'; '.join(verification.contradictions)}")
+
+        if verification.rules_verified:
+            lines.append(f"Rules Verified: {', '.join(verification.rules_verified)}")
+
+        return "\n".join(lines)
+
+
+__all__ = ["GoldenRefiner"]

synkro/quality/grader.py

@@ -0,0 +1,153 @@
+"""Grading of generated traces for quality control."""
+
+from synkro.llm.client import LLM
+from synkro.models import Model, OpenAI
+from synkro.types.core import Trace, GradeResult
+from synkro.prompts.templates import BATCHED_GRADER_PROMPT
+from synkro.schemas import SingleGrade
+from synkro.parsers import parse_batched_grades
+from synkro.quality.multiturn_grader import MultiTurnGrader
+
+
+class Grader:
+    """
+    Grades generated traces for quality and policy compliance.
+
+    Uses an LLM to evaluate each trace against strict criteria:
+    - Policy compliance
+    - Proper citations
+    - Complete reasoning
+    - Actionable recommendations
+
+    Automatically detects multi-turn traces and delegates to MultiTurnGrader.
+
+    Examples:
+        >>> grader = Grader()
+        >>> result = await grader.grade(trace, policy.text)
+        >>> if result.passed:
+        ...     print("Trace passes quality checks!")
+    """
+
+    def __init__(self, llm: LLM | None = None, model: Model = OpenAI.GPT_4O):
+        """
+        Initialize the grader.
+
+        Args:
+            llm: LLM client to use (creates one if not provided)
+            model: Model to use if creating LLM (recommend stronger model for grading)
+        """
+        self.llm = llm or LLM(model=model)
+        self._multi_turn_grader: MultiTurnGrader | None = None
+
+    @property
+    def multi_turn_grader(self) -> MultiTurnGrader:
+        """Lazy initialization of multi-turn grader."""
+        if self._multi_turn_grader is None:
+            self._multi_turn_grader = MultiTurnGrader(llm=self.llm)
+        return self._multi_turn_grader
+
+    def _count_assistant_turns(self, trace: Trace) -> int:
+        """Count the number of assistant messages (turns) in a trace."""
+        return sum(1 for m in trace.messages if m.role == "assistant")
+
+    async def grade(self, trace: Trace, policy_text: str) -> GradeResult:
+        """
+        Grade a single trace.
+
+        Automatically detects multi-turn traces and delegates to MultiTurnGrader.
+
+        Args:
+            trace: The trace to grade
+            policy_text: The policy text to grade against
+
+        Returns:
+            GradeResult with pass/fail and feedback
+        """
+        # Detect multi-turn and delegate
+        assistant_count = self._count_assistant_turns(trace)
+        if assistant_count > 1:
+            return await self.multi_turn_grader.grade(trace, policy_text)
+
+        # Single-turn grading
+        prompt = f"""You are a strict evaluator. Grade this response.
+
+A response PASSES only if ALL are true:
+1. Policy Compliant - Every recommendation follows the policy exactly
+2. Fully Supported - Every claim backed by specific policy section
+3. Properly Cited - All relevant policy sections referenced
+4. Complete Reasoning - Chain of thought has no gaps
+5. Actionable & Specific - Recommendations are concrete, not vague
+
+SCENARIO:
+{trace.scenario.description}
+
+POLICY:
+{policy_text}
+
+RESPONSE TO GRADE:
+{trace.assistant_message}
+
+Grade this response."""
+
+        try:
+            # Use structured output for reliable grading
+            parsed = await self.llm.generate_structured(prompt, SingleGrade)
+            return GradeResult(
+                passed=parsed.passed,
+                issues=(
+                    parsed.policy_violations
+                    + parsed.missing_citations
+                    + parsed.incomplete_reasoning
+                    + parsed.vague_recommendations
+                ),
+                feedback=parsed.feedback,
+            )
+        except Exception:
+            # Fallback: assume fail if we can't parse
+            return GradeResult(
+                passed=False,
+                issues=["Unable to parse grade response"],
+                feedback="Grading failed - unable to parse response",
+            )
+
+    async def grade_batch(
+        self, traces: list[Trace], policy_text: str
+    ) -> list[GradeResult]:
+        """
+        Grade multiple traces.
+
+        Args:
+            traces: List of traces to grade
+            policy_text: The policy text to grade against
+
+        Returns:
+            List of GradeResults in same order as input
+        """
+        results = []
+
+        for trace in traces:
+            result = await self.grade(trace, policy_text)
+            results.append(result)
+
+        return results
+
+    async def grade_batch_parallel(
+        self, traces: list[Trace], policy_text: str
+    ) -> list[GradeResult]:
+        """
+        Grade multiple traces in parallel.
+
+        More efficient for large batches but uses more API calls concurrently.
+
+        Args:
+            traces: List of traces to grade
+            policy_text: The policy text to grade against
+
+        Returns:
+            List of GradeResults in same order as input
+        """
+        import asyncio
+
+        tasks = [self.grade(trace, policy_text) for trace in traces]
+        return await asyncio.gather(*tasks)
+