synkro 0.4.12__py3-none-any.whl

synkro/quality/refiner.py

@@ -0,0 +1,137 @@
+"""Refinement of failed traces based on grader feedback."""
+
+from synkro.llm.client import LLM
+from synkro.models import Model, OpenAI
+from synkro.types.core import Trace, GradeResult, Message
+from synkro.prompts.templates import BATCHED_REFINER_PROMPT, SYSTEM_PROMPT
+from synkro.parsers import parse_single_response, extract_content
+
+
+class Refiner:
+    """
+    Refines traces that failed grading.
+
+    Takes failed traces and their grader feedback and generates
+    improved versions that address the issues.
+
+    Examples:
+        >>> refiner = Refiner()
+        >>> improved = await refiner.refine(failed_trace, grade_result, policy.text)
+    """
+
+    def __init__(self, llm: LLM | None = None, model: Model = OpenAI.GPT_4O_MINI):
+        """
+        Initialize the refiner.
+
+        Args:
+            llm: LLM client to use (creates one if not provided)
+            model: Model to use if creating LLM
+        """
+        self.llm = llm or LLM(model=model)
+        self.prompt_template = BATCHED_REFINER_PROMPT
+
+    async def refine(
+        self, trace: Trace, grade: GradeResult, policy_text: str
+    ) -> Trace:
+        """
+        Refine a failed trace based on grader feedback.
+
+        Args:
+            trace: The trace that failed grading
+            grade: The grade result with feedback
+            policy_text: The policy text
+
+        Returns:
+            New trace with improved response
+        """
+        prompt = self._build_prompt(trace, grade, policy_text)
+
+        response = await self.llm.generate(prompt)
+        parsed = parse_single_response(response)
+
+        if parsed and len(parsed.messages) >= 3:
+            messages = [
+                Message(role=m.role, content=m.content) for m in parsed.messages
+            ]
+        else:
+            # Fallback: construct from response
+            content = extract_content(response)
+            messages = [
+                Message(role="system", content=SYSTEM_PROMPT),
+                Message(
+                    role="user",
+                    content=f"Scenario: {trace.scenario.description}\n\nContext: {trace.scenario.context}",
+                ),
+                Message(role="assistant", content=content),
+            ]
+
+        return Trace(messages=messages, scenario=trace.scenario)
+
+    def _build_prompt(
+        self, trace: Trace, grade: GradeResult, policy_text: str
+    ) -> str:
+        """Build the refinement prompt."""
+        return f"""You are improving a response that failed quality checks.
+
+SCENARIO:
+{trace.scenario.description}
+
+CONTEXT:
+{trace.scenario.context}
+
+ORIGINAL RESPONSE:
+{trace.assistant_message}
+
+GRADER FEEDBACK:
+Issues: {', '.join(grade.issues) if grade.issues else 'None listed'}
+Summary: {grade.feedback}
+
+POLICY:
+{policy_text}
+
+Generate an IMPROVED response that fixes all the issues. Output a JSON object:
+{{
+  "messages": [
+    {{"role": "system", "content": "<system prompt>"}},
+    {{"role": "user", "content": "<the scenario>"}},
+    {{"role": "assistant", "content": "<your IMPROVED response>"}}
+  ]
+}}
+
+The improved response must:
+- Fix all policy violations
+- Add missing citations
+- Complete reasoning with no gaps
+- Make recommendations specific and actionable
+- Keep what was correct from the original
+
+Respond with ONLY the JSON object."""
+
+    async def refine_batch(
+        self,
+        traces: list[Trace],
+        grades: list[GradeResult],
+        policy_text: str,
+    ) -> list[Trace]:
+        """
+        Refine multiple failed traces.
+
+        Args:
+            traces: List of traces that failed grading
+            grades: Corresponding grade results
+            policy_text: The policy text
+
+        Returns:
+            List of refined traces
+        """
+        refined = []
+
+        for trace, grade in zip(traces, grades):
+            if not grade.passed:
+                improved = await self.refine(trace, grade, policy_text)
+                refined.append(improved)
+            else:
+                refined.append(trace)
+
+        return refined
+

synkro/quality/tool_grader.py

@@ -0,0 +1,126 @@
+"""Specialized grading for tool call traces."""
+
+import json
+from typing import TYPE_CHECKING
+
+from synkro.quality.grader import Grader
+from synkro.llm.client import LLM
+from synkro.models import Model, OpenAI
+from synkro.types.core import Trace, GradeResult
+from synkro.schemas import ToolCallGrade
+from synkro.prompts.tool_templates import TOOL_GRADE_PROMPT
+
+if TYPE_CHECKING:
+    from synkro.types.tool import ToolDefinition
+
+
+class ToolCallGrader(Grader):
+    """
+    Specialized grader for tool call traces.
+    
+    Evaluates tool usage on four criteria:
+    - Tool Selection: Did they use the right tool?
+    - Parameter Accuracy: Were the parameters correct?
+    - Response Synthesis: Did they use tool results correctly?
+    - Timing: Did they call tools at the right time?
+    
+    Examples:
+        >>> grader = ToolCallGrader(tools=[web_search, db_lookup])
+        >>> result = await grader.grade(trace, policy_text)
+        >>> if not result.passed:
+        ...     print(f"Issues: {result.issues}")
+    """
+    
+    def __init__(
+        self,
+        tools: list["ToolDefinition"],
+        llm: LLM | None = None,
+        model: Model = OpenAI.GPT_52,
+    ):
+        """
+        Initialize the tool call grader.
+        
+        Args:
+            tools: List of available tool definitions (for context)
+            llm: LLM client to use (creates one if not provided)
+            model: Model to use if creating LLM (recommend stronger model)
+        """
+        super().__init__(llm=llm, model=model)
+        self.tools = tools
+    
+    def _get_tools_description(self) -> str:
+        """Get formatted description of all tools for grading context."""
+        descriptions = []
+        for tool in self.tools:
+            descriptions.append(tool.to_system_prompt())
+        return "\n\n".join(descriptions)
+    
+    def _format_conversation(self, trace: Trace) -> str:
+        """Format the trace messages for the grading prompt, including tool_calls."""
+        lines = []
+        for msg in trace.messages:
+            if msg.role == "system":
+                lines.append(f"[SYSTEM]\n{msg.content}")
+            elif msg.role == "user":
+                lines.append(f"[USER]\n{msg.content}")
+            elif msg.role == "assistant":
+                if msg.tool_calls:
+                    # Format assistant message with tool calls
+                    tool_calls_str = []
+                    for tc in msg.tool_calls:
+                        tool_calls_str.append(
+                            f"  - {tc.function.name}({tc.function.arguments})"
+                        )
+                    lines.append(
+                        f"[ASSISTANT - TOOL CALLS]\n" + "\n".join(tool_calls_str)
+                    )
+                else:
+                    lines.append(f"[ASSISTANT]\n{msg.content}")
+            elif msg.role == "tool":
+                lines.append(
+                    f"[TOOL RESULT - {msg.tool_call_id}]\n{msg.content}"
+                )
+        return "\n\n".join(lines)
+    
+    async def grade(self, trace: Trace, policy_text: str) -> GradeResult:
+        """
+        Grade a tool call trace using tool-specific criteria.
+        
+        Args:
+            trace: The trace to grade
+            policy_text: The policy/guidelines text
+            
+        Returns:
+            GradeResult with pass/fail and detailed feedback
+        """
+        tools_desc = self._get_tools_description()
+        conversation = self._format_conversation(trace)
+        
+        prompt = TOOL_GRADE_PROMPT.format(
+            TOOLS_DESCRIPTION=tools_desc,
+            GUIDELINES=policy_text,
+            SCENARIO=trace.scenario.description,
+            CONVERSATION=conversation,
+        )
+        
+        try:
+            # Use structured output for consistent grading
+            parsed = await self.llm.generate_structured(prompt, ToolCallGrade)
+            
+            # Convert to standard GradeResult format
+            return GradeResult(
+                passed=parsed.passed,
+                issues=parsed.get_all_issues(),
+                feedback=parsed.feedback,
+            )
+        except Exception:
+            # Fallback: assume fail if we can't parse
+            return GradeResult(
+                passed=False,
+                issues=["Unable to parse grade response"],
+                feedback="Grading failed - unable to parse response",
+            )
+
+
+__all__ = ["ToolCallGrader"]
+

synkro/quality/tool_refiner.py

@@ -0,0 +1,128 @@
+"""Specialized refinement for tool call traces that preserves format."""
+
+from typing import TYPE_CHECKING
+
+from synkro.quality.refiner import Refiner
+from synkro.llm.client import LLM
+from synkro.models import Model, OpenAI
+from synkro.types.core import Trace, GradeResult, Scenario
+
+if TYPE_CHECKING:
+    from synkro.types.tool import ToolDefinition
+    from synkro.generation.tool_simulator import ToolSimulator
+
+
+class ToolCallRefiner(Refiner):
+    """
+    Specialized refiner for tool call traces.
+    
+    Unlike the base Refiner which generates plain text responses, this refiner
+    uses the ToolCallResponseGenerator to regenerate traces, ensuring the
+    tool_calls format is preserved in the output.
+    
+    The grading feedback is incorporated into the scenario context so the
+    LLM knows what to fix during regeneration.
+    
+    Examples:
+        >>> refiner = ToolCallRefiner(
+        ...     tools=[web_search, db_lookup],
+        ...     simulator=tool_simulator,
+        ... )
+        >>> improved = await refiner.refine(failed_trace, grade, policy_text)
+        >>> # improved trace has proper tool_calls format
+    """
+    
+    def __init__(
+        self,
+        tools: list["ToolDefinition"],
+        simulator: "ToolSimulator",
+        llm: LLM | None = None,
+        model: Model = OpenAI.GPT_4O_MINI,
+    ):
+        """
+        Initialize the tool call refiner.
+        
+        Args:
+            tools: List of available tool definitions
+            simulator: Tool simulator for generating tool responses
+            llm: LLM client to use (creates one if not provided)
+            model: Model to use if creating LLM
+        """
+        super().__init__(llm=llm, model=model)
+        self.tools = tools
+        self.simulator = simulator
+        self._response_generator = None
+    
+    def _get_response_generator(self):
+        """Lazily create the ToolCallResponseGenerator."""
+        if self._response_generator is None:
+            from synkro.generation.tool_responses import ToolCallResponseGenerator
+            self._response_generator = ToolCallResponseGenerator(
+                tools=self.tools,
+                llm=self.llm,
+                simulator=self.simulator,
+            )
+        return self._response_generator
+    
+    def _build_enhanced_scenario(
+        self, trace: Trace, grade: GradeResult
+    ) -> Scenario:
+        """
+        Build an enhanced scenario that includes grading feedback.
+        
+        The feedback helps the LLM understand what went wrong and how to fix it.
+        """
+        # Build feedback context
+        feedback_parts = []
+        if grade.issues:
+            feedback_parts.append("PREVIOUS ISSUES TO FIX:")
+            for issue in grade.issues:
+                feedback_parts.append(f"  - {issue}")
+        if grade.feedback:
+            feedback_parts.append(f"\nGRADER FEEDBACK: {grade.feedback}")
+        
+        feedback_context = "\n".join(feedback_parts) if feedback_parts else ""
+        
+        # Enhance the context with feedback
+        enhanced_context = trace.scenario.context
+        if feedback_context:
+            enhanced_context = f"{trace.scenario.context}\n\n--- REFINEMENT GUIDANCE ---\n{feedback_context}"
+        
+        return Scenario(
+            description=trace.scenario.description,
+            context=enhanced_context,
+            category=trace.scenario.category,
+        )
+    
+    async def refine(
+        self, trace: Trace, grade: GradeResult, policy_text: str
+    ) -> Trace:
+        """
+        Refine a failed tool call trace by regenerating with feedback.
+        
+        Uses the ToolCallResponseGenerator to ensure the regenerated trace
+        maintains proper tool_calls format.
+        
+        Args:
+            trace: The trace that failed grading
+            grade: The grade result with feedback
+            policy_text: The policy/guidelines text
+            
+        Returns:
+            New trace with improved response and preserved tool_calls format
+        """
+        # Create enhanced scenario with grading feedback
+        enhanced_scenario = self._build_enhanced_scenario(trace, grade)
+        
+        # Regenerate using ToolCallResponseGenerator (preserves format)
+        generator = self._get_response_generator()
+        refined_trace = await generator.generate_single(policy_text, enhanced_scenario)
+        
+        # Preserve the original scenario reference (without the feedback context)
+        refined_trace.scenario = trace.scenario
+        
+        return refined_trace
+
+
+__all__ = ["ToolCallRefiner"]
+

synkro/quality/verifier.py

@@ -0,0 +1,228 @@
+"""Trace Verifier - The Auditor.
+
+Verifies generated traces against the Logic Map to ensure:
+- No skipped rules
+- No hallucinated rules
+- No contradictions
+- DAG compliance
+
+This is Stage 4 of the Golden Trace pipeline.
+"""
+
+from synkro.llm.client import LLM
+from synkro.models import Model, OpenAI
+from synkro.schemas import VerificationOutput
+from synkro.types.core import Trace, GradeResult
+from synkro.types.logic_map import LogicMap, GoldenScenario, VerificationResult
+from synkro.prompts.golden_templates import VERIFICATION_PROMPT
+
+
+class TraceVerifier:
+    """
+    The Auditor - Verifies traces against the Logic Map.
+
+    Performs strict verification to ensure:
+    1. No Skipped Rules: All target rules were evaluated
+    2. No Hallucinated Rules: Only valid rules were cited
+    3. No Contradictions: Reasoning is internally consistent
+    4. DAG Compliance: Dependency order was followed
+    5. Outcome Alignment: Response matches expected outcome
+
+    Examples:
+        >>> verifier = TraceVerifier(llm=LLM(model=OpenAI.GPT_4O))
+        >>> result = await verifier.verify(trace, logic_map, scenario)
+        >>> if result.passed:
+        ...     print("Trace verified successfully")
+    """
+
+    def __init__(
+        self,
+        llm: LLM | None = None,
+        model: Model = OpenAI.GPT_4O,
+    ):
+        """
+        Initialize the Trace Verifier.
+
+        Args:
+            llm: LLM client to use (creates one if not provided)
+            model: Model to use if creating LLM (default: GPT-4O for accuracy)
+        """
+        self.llm = llm or LLM(model=model, temperature=0.1)
+
+    async def verify(
+        self,
+        trace: Trace,
+        logic_map: LogicMap,
+        scenario: GoldenScenario,
+        reasoning_chain: list | None = None,
+        rules_applied: list[str] | None = None,
+        rules_excluded: list[str] | None = None,
+    ) -> VerificationResult:
+        """
+        Verify a trace against the Logic Map.
+
+        Args:
+            trace: The trace to verify
+            logic_map: The Logic Map (ground truth)
+            scenario: The golden scenario
+            reasoning_chain: Optional reasoning chain from generation
+            rules_applied: Optional list of rules claimed applied
+            rules_excluded: Optional list of rules claimed excluded
+
+        Returns:
+            VerificationResult with pass/fail and detailed issues
+        """
+        # Format inputs for prompt
+        logic_map_str = self._format_logic_map(logic_map)
+        trace_messages_str = self._format_trace_messages(trace)
+        reasoning_str = self._format_reasoning_chain(reasoning_chain) if reasoning_chain else "Not provided"
+
+        # Build prompt
+        prompt = VERIFICATION_PROMPT.format(
+            logic_map=logic_map_str,
+            scenario_type=scenario.scenario_type.value.upper(),
+            scenario_description=scenario.description,
+            target_rule_ids=", ".join(scenario.target_rule_ids),
+            expected_outcome=scenario.expected_outcome,
+            trace_messages=trace_messages_str,
+            reasoning_chain=reasoning_str,
+            rules_applied=", ".join(rules_applied) if rules_applied else "Not specified",
+            rules_excluded=", ".join(rules_excluded) if rules_excluded else "Not specified",
+        )
+
+        # Generate structured output
+        result = await self.llm.generate_structured(prompt, VerificationOutput)
+
+        # Convert to domain model
+        return VerificationResult(
+            passed=result.passed,
+            issues=result.issues,
+            skipped_rules=result.skipped_rules,
+            hallucinated_rules=result.hallucinated_rules,
+            contradictions=result.contradictions,
+            rules_verified=result.rules_verified,
+        )
+
+    def _format_logic_map(self, logic_map: LogicMap) -> str:
+        """Format Logic Map for verification prompt."""
+        lines = []
+        lines.append("RULES:")
+        for rule in logic_map.rules:
+            deps = f" [depends on: {', '.join(rule.dependencies)}]" if rule.dependencies else ""
+            lines.append(
+                f"  {rule.rule_id} ({rule.category.value}): {rule.text}{deps}"
+            )
+            lines.append(f"    IF: {rule.condition}")
+            lines.append(f"    THEN: {rule.action}")
+
+        lines.append("\nROOT RULES (Entry Points):")
+        lines.append(f"  {', '.join(logic_map.root_rules)}")
+
+        return "\n".join(lines)
+
+    def _format_trace_messages(self, trace: Trace) -> str:
+        """Format trace messages for verification prompt."""
+        lines = []
+        for i, msg in enumerate(trace.messages):
+            role = msg.role.upper()
+            content = msg.content or "(no content)"
+
+            # Handle tool calls
+            if msg.tool_calls:
+                tool_info = []
+                for tc in msg.tool_calls:
+                    if hasattr(tc, 'function'):
+                        tool_info.append(f"  - {tc.function.name}({tc.function.arguments})")
+                    elif isinstance(tc, dict):
+                        func = tc.get('function', {})
+                        tool_info.append(f"  - {func.get('name', 'unknown')}({func.get('arguments', '{}')})")
+                content = "Tool calls:\n" + "\n".join(tool_info)
+
+            # Handle tool responses
+            if msg.tool_call_id:
+                role = f"TOOL (call_id: {msg.tool_call_id})"
+
+            lines.append(f"[{role}] {content}")
+
+        return "\n\n".join(lines)
+
+    def _format_reasoning_chain(self, reasoning_chain: list) -> str:
+        """Format reasoning chain for verification prompt."""
+        lines = []
+        for i, step in enumerate(reasoning_chain, 1):
+            if hasattr(step, 'rule_id'):
+                applies = "APPLIES" if step.applies else "DOES NOT APPLY"
+                lines.append(f"Step {i}: {step.rule_id} - {applies}")
+                lines.append(f"  Rule: {step.rule_text}")
+                lines.append(f"  Reasoning: {step.reasoning}")
+                if step.exclusions:
+                    lines.append(f"  Excludes: {', '.join(step.exclusions)}")
+            else:
+                # Handle dict format
+                applies = "APPLIES" if step.get('applies', False) else "DOES NOT APPLY"
+                lines.append(f"Step {i}: {step.get('rule_id', 'unknown')} - {applies}")
+                lines.append(f"  Reasoning: {step.get('reasoning', 'N/A')}")
+
+        return "\n".join(lines)
+
+    async def verify_and_grade(
+        self,
+        trace: Trace,
+        logic_map: LogicMap,
+        scenario: GoldenScenario,
+    ) -> tuple[VerificationResult, GradeResult]:
+        """
+        Verify a trace and convert to GradeResult for pipeline compatibility.
+
+        Args:
+            trace: The trace to verify
+            logic_map: The Logic Map
+            scenario: The golden scenario
+
+        Returns:
+            Tuple of (VerificationResult, GradeResult)
+        """
+        # Extract reasoning chain metadata from trace (if present)
+        reasoning_chain = getattr(trace, 'reasoning_chain', None)
+        rules_applied = getattr(trace, 'rules_applied', None)
+        rules_excluded = getattr(trace, 'rules_excluded', None)
+
+        verification = await self.verify(
+            trace, logic_map, scenario,
+            reasoning_chain=reasoning_chain,
+            rules_applied=rules_applied,
+            rules_excluded=rules_excluded,
+        )
+
+        # Convert to GradeResult for pipeline compatibility
+        grade = GradeResult(
+            passed=verification.passed,
+            issues=verification.issues,
+            feedback=self._create_feedback(verification),
+        )
+
+        return verification, grade
+
+    def _create_feedback(self, verification: VerificationResult) -> str:
+        """Create feedback string from verification result."""
+        if verification.passed:
+            return f"Verified. Rules correctly applied: {', '.join(verification.rules_verified)}"
+
+        feedback_parts = []
+
+        if verification.skipped_rules:
+            feedback_parts.append(f"Skipped rules: {', '.join(verification.skipped_rules)}")
+
+        if verification.hallucinated_rules:
+            feedback_parts.append(f"Hallucinated rules: {', '.join(verification.hallucinated_rules)}")
+
+        if verification.contradictions:
+            feedback_parts.append(f"Contradictions: {'; '.join(verification.contradictions)}")
+
+        if verification.issues:
+            feedback_parts.append(f"Other issues: {'; '.join(verification.issues)}")
+
+        return " | ".join(feedback_parts) if feedback_parts else "Verification failed"
+
+
+__all__ = ["TraceVerifier"]