synkro 0.4.12__py3-none-any.whl

synkro/prompts/tool_templates.py

@@ -0,0 +1,201 @@
+"""Prompt templates for tool call trace generation."""
+
+# =============================================================================
+# TOOL SCENARIO GENERATION
+# =============================================================================
+
+TOOL_SCENARIO_PROMPT = """You are an expert at creating realistic scenarios that require tool usage.
+
+Given a set of available tools and usage guidelines, generate diverse scenarios that test when and how to use these tools correctly.
+
+AVAILABLE TOOLS:
+{TOOLS_DESCRIPTION}
+
+USAGE GUIDELINES:
+{GUIDELINES}
+
+Generate scenarios that cover:
+
+1. **Clear Tool Use Cases** - Situations where a specific tool is clearly needed
+2. **Tool Selection** - Scenarios requiring choosing between multiple tools
+3. **No Tool Needed** - Cases where the assistant should respond directly without tools
+4. **Multi-Tool Workflows** - Complex tasks requiring multiple tool calls
+5. **Parameter Variations** - Different parameter combinations and edge cases
+6. **Error Handling** - What to do when tools return errors or unexpected results
+
+Each scenario should include:
+- A realistic user request
+- Context about what information is available vs what needs to be looked up
+- Expected tool usage pattern (or lack thereof)
+
+Focus on creating "golden traces" - perfect examples of correct tool usage."""
+
+TOOL_CATEGORY_SCENARIO_PROMPT = """You are an expert at creating realistic scenarios for tool usage.
+
+Generate scenarios specifically for the following CATEGORY:
+**Category Name**: {CATEGORY_NAME}
+**Category Description**: {CATEGORY_DESCRIPTION}
+
+AVAILABLE TOOLS:
+{TOOLS_DESCRIPTION}
+
+USAGE GUIDELINES:
+{GUIDELINES}
+
+Create scenarios that:
+- Are deeply relevant to this specific category
+- Test the nuances of tool usage in this context
+- Include realistic user requests with appropriate context
+- Cover both happy paths and edge cases within this category"""
+
+# =============================================================================
+# TOOL RESPONSE GENERATION
+# =============================================================================
+
+TOOL_RESPONSE_PROMPT = """You are generating a training example for teaching an AI assistant to use tools correctly.
+
+AVAILABLE TOOLS:
+{TOOLS_DESCRIPTION}
+
+USAGE GUIDELINES:
+{GUIDELINES}
+
+SCENARIO:
+{SCENARIO}
+
+USER REQUEST:
+{USER_REQUEST}
+
+Generate a complete conversation that demonstrates correct tool usage:
+
+1. If a tool should be called:
+   - The assistant's first response should include appropriate tool_calls
+   - Include the simulated tool response
+   - The assistant should then synthesize the tool results into a helpful response
+
+2. If no tool is needed:
+   - The assistant should respond directly with helpful information
+   - Explain why no tool lookup was necessary
+
+The assistant should:
+- Only call tools when necessary (don't call tools for information you already know)
+- Use correct parameters with proper types
+- Wait for tool results before providing final answers
+- Synthesize tool results naturally without exposing raw data
+- Handle missing or partial information gracefully
+
+Output as JSON with this structure:
+{{
+  "messages": [
+    {{"role": "system", "content": "..."}},
+    {{"role": "user", "content": "..."}},
+    {{"role": "assistant", "content": null, "tool_calls": [...]}},  // if tool needed
+    {{"role": "tool", "tool_call_id": "...", "content": "..."}},    // tool result
+    {{"role": "assistant", "content": "..."}}                        // final response
+  ]
+}}"""
+
+# =============================================================================
+# TOOL GRADING
+# =============================================================================
+
+TOOL_GRADE_PROMPT = """You are a strict evaluator of tool usage in AI assistant responses.
+
+AVAILABLE TOOLS:
+{TOOLS_DESCRIPTION}
+
+USAGE GUIDELINES:
+{GUIDELINES}
+
+SCENARIO:
+{SCENARIO}
+
+CONVERSATION TO GRADE:
+{CONVERSATION}
+
+Evaluate the assistant's tool usage on these criteria:
+
+1. **Tool Selection** (Did they use the right tool?)
+   - Chose appropriate tool for the task
+   - Didn't use tools when not needed
+   - Used all necessary tools
+
+2. **Parameter Accuracy** (Were the parameters correct?)
+   - Correct parameter types
+   - Sensible parameter values
+   - Required parameters included
+
+3. **Response Synthesis** (Did they use tool results correctly?)
+   - Accurately incorporated tool results
+   - Didn't hallucinate beyond tool data
+   - Provided helpful, complete response
+
+4. **Timing** (Did they call tools at the right time?)
+   - Called tools before making claims
+   - Didn't call tools for known information
+   - Efficient tool call ordering
+
+A response PASSES only if ALL criteria are met.
+
+Grade this response."""
+
+# =============================================================================
+# TOOL REFINEMENT
+# =============================================================================
+
+TOOL_REFINE_PROMPT = """You are improving a tool-calling conversation that failed quality checks.
+
+AVAILABLE TOOLS:
+{TOOLS_DESCRIPTION}
+
+USAGE GUIDELINES:
+{GUIDELINES}
+
+ORIGINAL SCENARIO:
+{SCENARIO}
+
+FAILED CONVERSATION:
+{CONVERSATION}
+
+ISSUES FOUND:
+{ISSUES}
+
+GRADER FEEDBACK:
+{FEEDBACK}
+
+Generate an IMPROVED conversation that fixes all the issues while maintaining the same user request.
+
+Focus on:
+- Correct tool selection
+- Accurate parameters
+- Proper synthesis of tool results
+- No hallucination beyond tool data
+
+Output the corrected conversation as JSON."""
+
+# =============================================================================
+# TOOL SIMULATION
+# =============================================================================
+
+TOOL_SIMULATION_PROMPT = """You are simulating a tool response for training data generation.
+
+TOOL BEING CALLED:
+Name: {TOOL_NAME}
+Description: {TOOL_DESCRIPTION}
+Parameters: {TOOL_PARAMETERS}
+
+CALL ARGUMENTS:
+{ARGUMENTS}
+
+EXAMPLE RESPONSES (for reference):
+{MOCK_RESPONSES}
+
+Generate a realistic, plausible response that this tool would return for the given arguments.
+
+The response should:
+- Be realistic and internally consistent
+- Match the type of data this tool would return
+- Include appropriate detail level
+- Handle edge cases gracefully (e.g., no results found)
+
+Return only the tool response content as a string."""

synkro/quality/__init__.py

@@ -0,0 +1,14 @@
+"""Quality control components for trace grading and refinement."""
+
+from synkro.quality.grader import Grader
+from synkro.quality.refiner import Refiner
+from synkro.quality.tool_grader import ToolCallGrader
+from synkro.quality.tool_refiner import ToolCallRefiner
+
+__all__ = [
+    "Grader",
+    "Refiner",
+    "ToolCallGrader",
+    "ToolCallRefiner",
+]
+

synkro/quality/golden_refiner.py

@@ -0,0 +1,163 @@
+"""Golden Refiner - Refines traces that failed verification.
+
+Refines traces with Logic Map context to fix:
+- Skipped rules
+- Hallucinated rules
+- Contradictions
+- DAG violations
+"""
+
+from synkro.llm.client import LLM
+from synkro.models import Model, OpenAI
+from synkro.schemas import GoldenTraceOutput
+from synkro.types.core import Trace, Message
+from synkro.types.logic_map import LogicMap, GoldenScenario, VerificationResult
+from synkro.prompts.golden_templates import GOLDEN_REFINE_PROMPT
+
+
+class GoldenRefiner:
+    """
+    Refiner that uses Logic Map context to fix verification failures.
+
+    Addresses specific issues:
+    1. Skipped Rules: Adds evaluation of missed rules
+    2. Hallucinated Rules: Removes references to non-existent rules
+    3. Contradictions: Resolves logical inconsistencies
+    4. DAG Violations: Reorders reasoning to follow dependencies
+
+    Examples:
+        >>> refiner = GoldenRefiner(llm=LLM(model=OpenAI.GPT_4O_MINI))
+        >>> refined = await refiner.refine(trace, logic_map, scenario, verification)
+    """
+
+    def __init__(
+        self,
+        llm: LLM | None = None,
+        model: Model = OpenAI.GPT_4O_MINI,
+    ):
+        """
+        Initialize the Golden Refiner.
+
+        Args:
+            llm: LLM client to use (creates one if not provided)
+            model: Model to use if creating LLM
+        """
+        self.llm = llm or LLM(model=model, temperature=0.5)
+
+    async def refine(
+        self,
+        trace: Trace,
+        logic_map: LogicMap,
+        scenario: GoldenScenario,
+        verification: VerificationResult,
+    ) -> Trace:
+        """
+        Refine a trace that failed verification.
+
+        Args:
+            trace: The original trace that failed
+            logic_map: The Logic Map (ground truth)
+            scenario: The golden scenario
+            verification: The verification result with issues
+
+        Returns:
+            Refined trace with issues addressed
+        """
+        # Format inputs for prompt
+        logic_map_str = self._format_logic_map(logic_map)
+        original_trace_str = self._format_trace(trace)
+        verification_str = self._format_verification(verification)
+
+        # Build prompt
+        prompt = GOLDEN_REFINE_PROMPT.format(
+            original_trace=original_trace_str,
+            verification_result=verification_str,
+            logic_map=logic_map_str,
+            scenario_description=scenario.description,
+            skipped_rules=", ".join(verification.skipped_rules) if verification.skipped_rules else "None",
+            hallucinated_rules=", ".join(verification.hallucinated_rules) if verification.hallucinated_rules else "None",
+            contradictions="; ".join(verification.contradictions) if verification.contradictions else "None",
+        )
+
+        # Generate refined trace
+        result = await self.llm.generate_structured(prompt, GoldenTraceOutput)
+
+        # Convert to Trace
+        messages = [
+            Message(role=m.role, content=m.content)
+            for m in result.messages
+        ]
+
+        # Preserve scenario from original trace
+        return Trace(
+            messages=messages,
+            scenario=trace.scenario,
+        )
+
+    def _format_logic_map(self, logic_map: LogicMap) -> str:
+        """Format Logic Map for refinement prompt."""
+        lines = []
+        lines.append("RULES:")
+        for rule in logic_map.rules:
+            deps = f" [depends on: {', '.join(rule.dependencies)}]" if rule.dependencies else ""
+            lines.append(
+                f"  {rule.rule_id} ({rule.category.value}): {rule.text}{deps}"
+            )
+            lines.append(f"    IF: {rule.condition}")
+            lines.append(f"    THEN: {rule.action}")
+
+        lines.append("\nDEPENDENCY ORDER:")
+        for root_id in logic_map.root_rules:
+            chain = logic_map.get_chain(root_id)
+            if chain:
+                chain_str = " -> ".join(r.rule_id for r in chain)
+                lines.append(f"  {chain_str}")
+
+        return "\n".join(lines)
+
+    def _format_trace(self, trace: Trace) -> str:
+        """Format trace for refinement prompt."""
+        lines = []
+        for msg in trace.messages:
+            role = msg.role.upper()
+            content = msg.content or "(no content)"
+
+            # Handle tool calls
+            if msg.tool_calls:
+                tool_info = []
+                for tc in msg.tool_calls:
+                    if hasattr(tc, 'function'):
+                        tool_info.append(f"  - {tc.function.name}({tc.function.arguments})")
+                    elif isinstance(tc, dict):
+                        func = tc.get('function', {})
+                        tool_info.append(f"  - {func.get('name', 'unknown')}({func.get('arguments', '{}')})")
+                content = "Tool calls:\n" + "\n".join(tool_info)
+
+            lines.append(f"[{role}]: {content}")
+
+        return "\n\n".join(lines)
+
+    def _format_verification(self, verification: VerificationResult) -> str:
+        """Format verification result for refinement prompt."""
+        lines = []
+        lines.append(f"Passed: {verification.passed}")
+
+        if verification.issues:
+            lines.append(f"Issues: {'; '.join(verification.issues)}")
+
+        if verification.skipped_rules:
+            lines.append(f"Skipped Rules: {', '.join(verification.skipped_rules)}")
+
+        if verification.hallucinated_rules:
+            lines.append(f"Hallucinated Rules: {', '.join(verification.hallucinated_rules)}")
+
+        if verification.contradictions:
+            lines.append(f"Contradictions: {'; '.join(verification.contradictions)}")
+
+        if verification.rules_verified:
+            lines.append(f"Rules Verified: {', '.join(verification.rules_verified)}")
+
+        return "\n".join(lines)
+
+
+__all__ = ["GoldenRefiner"]

synkro/quality/grader.py

@@ -0,0 +1,153 @@
+"""Grading of generated traces for quality control."""
+
+from synkro.llm.client import LLM
+from synkro.models import Model, OpenAI
+from synkro.types.core import Trace, GradeResult
+from synkro.prompts.templates import BATCHED_GRADER_PROMPT
+from synkro.schemas import SingleGrade
+from synkro.parsers import parse_batched_grades
+from synkro.quality.multiturn_grader import MultiTurnGrader
+
+
+class Grader:
+    """
+    Grades generated traces for quality and policy compliance.
+
+    Uses an LLM to evaluate each trace against strict criteria:
+    - Policy compliance
+    - Proper citations
+    - Complete reasoning
+    - Actionable recommendations
+
+    Automatically detects multi-turn traces and delegates to MultiTurnGrader.
+
+    Examples:
+        >>> grader = Grader()
+        >>> result = await grader.grade(trace, policy.text)
+        >>> if result.passed:
+        ...     print("Trace passes quality checks!")
+    """
+
+    def __init__(self, llm: LLM | None = None, model: Model = OpenAI.GPT_4O):
+        """
+        Initialize the grader.
+
+        Args:
+            llm: LLM client to use (creates one if not provided)
+            model: Model to use if creating LLM (recommend stronger model for grading)
+        """
+        self.llm = llm or LLM(model=model)
+        self._multi_turn_grader: MultiTurnGrader | None = None
+
+    @property
+    def multi_turn_grader(self) -> MultiTurnGrader:
+        """Lazy initialization of multi-turn grader."""
+        if self._multi_turn_grader is None:
+            self._multi_turn_grader = MultiTurnGrader(llm=self.llm)
+        return self._multi_turn_grader
+
+    def _count_assistant_turns(self, trace: Trace) -> int:
+        """Count the number of assistant messages (turns) in a trace."""
+        return sum(1 for m in trace.messages if m.role == "assistant")
+
+    async def grade(self, trace: Trace, policy_text: str) -> GradeResult:
+        """
+        Grade a single trace.
+
+        Automatically detects multi-turn traces and delegates to MultiTurnGrader.
+
+        Args:
+            trace: The trace to grade
+            policy_text: The policy text to grade against
+
+        Returns:
+            GradeResult with pass/fail and feedback
+        """
+        # Detect multi-turn and delegate
+        assistant_count = self._count_assistant_turns(trace)
+        if assistant_count > 1:
+            return await self.multi_turn_grader.grade(trace, policy_text)
+
+        # Single-turn grading
+        prompt = f"""You are a strict evaluator. Grade this response.
+
+A response PASSES only if ALL are true:
+1. Policy Compliant - Every recommendation follows the policy exactly
+2. Fully Supported - Every claim backed by specific policy section
+3. Properly Cited - All relevant policy sections referenced
+4. Complete Reasoning - Chain of thought has no gaps
+5. Actionable & Specific - Recommendations are concrete, not vague
+
+SCENARIO:
+{trace.scenario.description}
+
+POLICY:
+{policy_text}
+
+RESPONSE TO GRADE:
+{trace.assistant_message}
+
+Grade this response."""
+
+        try:
+            # Use structured output for reliable grading
+            parsed = await self.llm.generate_structured(prompt, SingleGrade)
+            return GradeResult(
+                passed=parsed.passed,
+                issues=(
+                    parsed.policy_violations
+                    + parsed.missing_citations
+                    + parsed.incomplete_reasoning
+                    + parsed.vague_recommendations
+                ),
+                feedback=parsed.feedback,
+            )
+        except Exception:
+            # Fallback: assume fail if we can't parse
+            return GradeResult(
+                passed=False,
+                issues=["Unable to parse grade response"],
+                feedback="Grading failed - unable to parse response",
+            )
+
+    async def grade_batch(
+        self, traces: list[Trace], policy_text: str
+    ) -> list[GradeResult]:
+        """
+        Grade multiple traces.
+
+        Args:
+            traces: List of traces to grade
+            policy_text: The policy text to grade against
+
+        Returns:
+            List of GradeResults in same order as input
+        """
+        results = []
+
+        for trace in traces:
+            result = await self.grade(trace, policy_text)
+            results.append(result)
+
+        return results
+
+    async def grade_batch_parallel(
+        self, traces: list[Trace], policy_text: str
+    ) -> list[GradeResult]:
+        """
+        Grade multiple traces in parallel.
+
+        More efficient for large batches but uses more API calls concurrently.
+
+        Args:
+            traces: List of traces to grade
+            policy_text: The policy text to grade against
+
+        Returns:
+            List of GradeResults in same order as input
+        """
+        import asyncio
+
+        tasks = [self.grade(trace, policy_text) for trace in traces]
+        return await asyncio.gather(*tasks)
+

synkro/quality/multiturn_grader.py

@@ -0,0 +1,150 @@
+"""Multi-turn conversation grading with per-turn and overall evaluation."""
+
+from synkro.llm.client import LLM
+from synkro.models import Model, OpenAI
+from synkro.types.core import Trace, Message, GradeResult
+from synkro.prompts.multiturn_templates import MULTI_TURN_GRADE_PROMPT
+from synkro.schemas import ConversationGrade, TurnGrade
+
+
+class MultiTurnGrader:
+    """
+    Grades multi-turn conversations using per-turn and overall criteria.
+
+    Uses existing schemas:
+    - TurnGrade: Per-turn policy violations, citations, reasoning
+    - ConversationGrade: Overall pass, coherence, progressive depth
+
+    Examples:
+        >>> grader = MultiTurnGrader()
+        >>> result = await grader.grade(trace, policy_text)
+        >>> print(result.passed, result.feedback)
+    """
+
+    def __init__(self, llm: LLM | None = None, model: Model = OpenAI.GPT_4O):
+        """
+        Initialize the multi-turn grader.
+
+        Args:
+            llm: LLM client to use (creates one if not provided)
+            model: Model to use if creating LLM (recommend stronger model)
+        """
+        self.llm = llm or LLM(model=model)
+
+    def _count_assistant_turns(self, trace: Trace) -> int:
+        """Count the number of assistant messages (turns) in a trace."""
+        return sum(1 for m in trace.messages if m.role == "assistant")
+
+    def _format_conversation(self, messages: list[Message]) -> str:
+        """Format conversation messages for prompt inclusion."""
+        formatted = []
+        for msg in messages:
+            role = msg.role.upper()
+            content = msg.content or "[No content]"
+            formatted.append(f"{role}: {content}")
+        return "\n\n".join(formatted)
+
+    def _extract_all_issues(self, conversation_grade: ConversationGrade) -> list[str]:
+        """Extract all issues from conversation grade into flat list."""
+        issues = []
+
+        # Add coherence issues
+        issues.extend(conversation_grade.coherence_issues)
+
+        # Add per-turn issues
+        for turn_grade in conversation_grade.turn_grades:
+            issues.extend(turn_grade.policy_violations)
+            issues.extend(turn_grade.missing_citations)
+            issues.extend(turn_grade.incomplete_reasoning)
+            issues.extend(turn_grade.vague_recommendations)
+
+        return issues
+
+    async def _grade_conversation(
+        self,
+        trace: Trace,
+        policy_text: str,
+    ) -> ConversationGrade:
+        """
+        Grade the full conversation using ConversationGrade schema.
+
+        Args:
+            trace: The trace to grade
+            policy_text: The policy for evaluation
+
+        Returns:
+            ConversationGrade with per-turn and overall assessment
+        """
+        conversation = self._format_conversation(trace.messages)
+
+        prompt = f"""{MULTI_TURN_GRADE_PROMPT.format(
+            conversation=conversation,
+            policy=policy_text,
+        )}"""
+
+        try:
+            return await self.llm.generate_structured(prompt, ConversationGrade)
+        except Exception:
+            # Fallback - create a failing grade
+            num_turns = self._count_assistant_turns(trace)
+            turn_grades = [
+                TurnGrade(
+                    turn_index=i,
+                    passed=False,
+                    policy_violations=[],
+                    missing_citations=[],
+                    incomplete_reasoning=[],
+                    vague_recommendations=[],
+                    feedback="Unable to grade - parsing error",
+                )
+                for i in range(num_turns)
+            ]
+            return ConversationGrade(
+                index=0,
+                overall_pass=False,
+                turn_grades=turn_grades,
+                coherence_pass=False,
+                coherence_issues=["Unable to evaluate - grading error"],
+                progressive_depth=False,
+                overall_feedback="Grading failed - please retry",
+            )
+
+    async def grade(self, trace: Trace, policy_text: str) -> GradeResult:
+        """
+        Grade a multi-turn conversation.
+
+        Args:
+            trace: The trace to grade
+            policy_text: The policy for evaluation
+
+        Returns:
+            GradeResult with pass/fail, issues, and feedback
+        """
+        # Get full conversation grade
+        conversation_grade = await self._grade_conversation(trace, policy_text)
+
+        # Convert to standard GradeResult
+        return GradeResult(
+            passed=conversation_grade.overall_pass,
+            issues=self._extract_all_issues(conversation_grade),
+            feedback=conversation_grade.overall_feedback,
+        )
+
+    async def grade_detailed(
+        self,
+        trace: Trace,
+        policy_text: str,
+    ) -> ConversationGrade:
+        """
+        Get detailed per-turn grading for a conversation.
+
+        Use this when you need access to individual turn grades.
+
+        Args:
+            trace: The trace to grade
+            policy_text: The policy for evaluation
+
+        Returns:
+            ConversationGrade with full per-turn breakdown
+        """
+        return await self._grade_conversation(trace, policy_text)