synkro 0.4.36__py3-none-any.whl

synkro/formatters/qa.py

@@ -0,0 +1,112 @@
+"""QA (Question-Answer) formatter for evaluation datasets."""
+
+import json
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from synkro.types.core import Trace
+
+
+class QAFormatter:
+    """
+    Format traces for evaluation datasets (Q&A format with ground truth).
+
+    QA format includes:
+    - question: The user's question
+    - answer: The assistant's response
+    - expected_outcome: Ground truth expected behavior
+    - ground_truth_rules: Rule IDs that should be applied
+    - difficulty: Scenario type (positive, negative, edge_case, irrelevant)
+    - category: Policy category
+    - context: Additional context for the scenario
+    - passed: Whether the response was graded as correct
+
+    Example output:
+        {
+            "question": "Can I submit a $200 expense without a receipt?",
+            "answer": "No, all expenses require receipts...",
+            "expected_outcome": "Deny - missing receipt violates R003",
+            "ground_truth_rules": ["R003", "R005"],
+            "difficulty": "negative",
+            "category": "Receipt Requirements",
+            "context": "Expense: $200, No receipt, Within 30 days",
+            "passed": true
+        }
+    """
+
+    def __init__(self, include_reasoning: bool = False):
+        """
+        Initialize the QA formatter.
+
+        Args:
+            include_reasoning: If True, include reasoning chain in output
+        """
+        self.include_reasoning = include_reasoning
+
+    def format(self, traces: list["Trace"]) -> list[dict]:
+        """
+        Format traces as QA evaluation examples.
+
+        Args:
+            traces: List of traces to format
+
+        Returns:
+            List of QA examples with ground truth
+        """
+        examples = []
+
+        for trace in traces:
+            example = {
+                "question": trace.user_message,
+                "answer": trace.assistant_message,
+                "expected_outcome": trace.scenario.expected_outcome or "",
+                "ground_truth_rules": trace.scenario.target_rule_ids or [],
+                "difficulty": trace.scenario.scenario_type or "unknown",
+                "category": trace.scenario.category or "",
+                "context": trace.scenario.context or "",
+                "passed": trace.grade.passed if trace.grade else None,
+            }
+
+            # Optionally include reasoning
+            if self.include_reasoning:
+                example["reasoning_chain"] = trace.reasoning_chain
+                example["rules_applied"] = trace.rules_applied
+                example["rules_excluded"] = trace.rules_excluded
+
+            # Include grading feedback if available
+            if trace.grade:
+                example["grade_feedback"] = trace.grade.feedback
+                example["grade_issues"] = trace.grade.issues
+
+            examples.append(example)
+
+        return examples
+
+    def save(self, traces: list["Trace"], path: str | Path) -> None:
+        """
+        Save formatted traces to a JSONL file.
+
+        Args:
+            traces: List of traces to save
+            path: Output file path (should end in .jsonl)
+        """
+        path = Path(path)
+        examples = self.format(traces)
+
+        with open(path, "w") as f:
+            for example in examples:
+                f.write(json.dumps(example) + "\n")
+
+    def to_jsonl(self, traces: list["Trace"]) -> str:
+        """
+        Convert traces to JSONL string.
+
+        Args:
+            traces: List of traces to convert
+
+        Returns:
+            JSONL formatted string
+        """
+        examples = self.format(traces)
+        return "\n".join(json.dumps(e) for e in examples)

synkro/formatters/sft.py

@@ -0,0 +1,90 @@
+"""SFT (Supervised Fine-Tuning) formatter."""
+
+import json
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from synkro.types.core import Trace
+
+
+class SFTFormatter:
+    """
+    Format traces for Supervised Fine-Tuning (SFT).
+
+    SFT format is a simple array of conversations, each with messages.
+    This is the standard format used by OpenAI, HuggingFace, and most
+    fine-tuning platforms.
+
+    Example output:
+        {"messages": [{"role": "system", "content": "..."}, ...]}
+        {"messages": [{"role": "system", "content": "..."}, ...]}
+    """
+
+    def __init__(self, include_metadata: bool = False):
+        """
+        Initialize the SFT formatter.
+
+        Args:
+            include_metadata: If True, include trace metadata in output
+        """
+        self.include_metadata = include_metadata
+
+    def format(self, traces: list["Trace"]) -> list[dict]:
+        """
+        Format traces as SFT training examples.
+
+        Args:
+            traces: List of traces to format
+
+        Returns:
+            List of SFT examples (dicts with 'messages' key)
+        """
+        examples = []
+
+        for trace in traces:
+            example = {
+                "messages": [
+                    {"role": m.role, "content": m.content} for m in trace.messages
+                ]
+            }
+
+            if self.include_metadata:
+                example["metadata"] = {
+                    "scenario": trace.scenario.description,
+                    "category": trace.scenario.category,
+                    "grade": trace.grade.model_dump() if trace.grade else None,
+                }
+
+            examples.append(example)
+
+        return examples
+
+    def save(self, traces: list["Trace"], path: str | Path) -> None:
+        """
+        Save formatted traces to a JSONL file.
+
+        Args:
+            traces: List of traces to save
+            path: Output file path (should end in .jsonl)
+        """
+        path = Path(path)
+        examples = self.format(traces)
+
+        with open(path, "w") as f:
+            for example in examples:
+                f.write(json.dumps(example) + "\n")
+
+    def to_jsonl(self, traces: list["Trace"]) -> str:
+        """
+        Convert traces to JSONL string.
+
+        Args:
+            traces: List of traces to convert
+
+        Returns:
+            JSONL formatted string
+        """
+        examples = self.format(traces)
+        return "\n".join(json.dumps(e) for e in examples)
+

synkro/formatters/tool_call.py

@@ -0,0 +1,127 @@
+"""Tool Call formatter for training data."""
+
+import json
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from synkro.types.core import Trace
+
+
+class ToolCallFormatter:
+    """
+    Format traces with tool calls for fine-tuning.
+    
+    Outputs OpenAI function calling format compatible with most fine-tuning platforms.
+    
+    Example output:
+        {
+          "messages": [
+            {"role": "system", "content": "You have access to: web_search(query)"},
+            {"role": "user", "content": "What's the weather in NYC?"},
+            {"role": "assistant", "content": null, "tool_calls": [
+              {"id": "call_1", "type": "function", "function": {"name": "web_search", "arguments": "{\\"query\\": \\"weather NYC\\"}"}}
+            ]},
+            {"role": "tool", "tool_call_id": "call_1", "content": "NYC: 72°F, sunny"},
+            {"role": "assistant", "content": "The weather in NYC is currently 72°F and sunny."}
+          ]
+        }
+    """
+
+    def __init__(self, include_metadata: bool = False):
+        """
+        Initialize the ToolCallFormatter.
+        
+        Args:
+            include_metadata: If True, include trace metadata in output
+        """
+        self.include_metadata = include_metadata
+
+    def format(self, traces: list["Trace"]) -> list[dict]:
+        """
+        Format traces as tool-calling training examples.
+        
+        Args:
+            traces: List of traces to format
+            
+        Returns:
+            List of formatted examples with tool calls
+        """
+        examples = []
+        
+        for trace in traces:
+            messages = []
+            
+            for m in trace.messages:
+                msg = {"role": m.role}
+                
+                # Handle content (can be None for tool-calling assistant messages)
+                if m.content is not None:
+                    msg["content"] = m.content
+                elif m.role == "assistant" and m.tool_calls:
+                    msg["content"] = None
+                else:
+                    msg["content"] = ""
+                
+                # Handle tool calls
+                if m.tool_calls:
+                    msg["tool_calls"] = [
+                        {
+                            "id": tc.id,
+                            "type": tc.type,
+                            "function": {
+                                "name": tc.function.name,
+                                "arguments": tc.function.arguments,
+                            }
+                        }
+                        for tc in m.tool_calls
+                    ]
+                
+                # Handle tool response
+                if m.tool_call_id:
+                    msg["tool_call_id"] = m.tool_call_id
+                
+                messages.append(msg)
+            
+            example = {"messages": messages}
+            
+            if self.include_metadata:
+                example["metadata"] = {
+                    "scenario": trace.scenario.description,
+                    "category": trace.scenario.category,
+                    "grade": trace.grade.model_dump() if trace.grade else None,
+                    "has_tool_calls": trace.has_tool_calls,
+                }
+            
+            examples.append(example)
+        
+        return examples
+
+    def save(self, traces: list["Trace"], path: str | Path) -> None:
+        """
+        Save formatted traces to a JSONL file.
+        
+        Args:
+            traces: List of traces to save
+            path: Output file path (should end in .jsonl)
+        """
+        path = Path(path)
+        examples = self.format(traces)
+        
+        with open(path, "w") as f:
+            for example in examples:
+                f.write(json.dumps(example) + "\n")
+
+    def to_jsonl(self, traces: list["Trace"]) -> str:
+        """
+        Convert traces to JSONL string.
+        
+        Args:
+            traces: List of traces to convert
+            
+        Returns:
+            JSONL formatted string
+        """
+        examples = self.format(traces)
+        return "\n".join(json.dumps(e) for e in examples)
+

synkro/generation/__init__.py

@@ -0,0 +1,9 @@
+"""Generation components for creating training data."""
+
+from synkro.generation.generator import Generator
+from synkro.generation.scenarios import ScenarioGenerator
+from synkro.generation.responses import ResponseGenerator
+from synkro.generation.planner import Planner
+
+__all__ = ["Generator", "ScenarioGenerator", "ResponseGenerator", "Planner"]
+

synkro/generation/follow_ups.py

@@ -0,0 +1,134 @@
+"""Follow-up question generation for multi-turn conversations."""
+
+from typing import Literal
+
+from synkro.llm.client import LLM
+from synkro.models import Model, OpenAI
+from synkro.types.core import Message
+from synkro.prompts.multiturn_templates import FOLLOW_UP_GENERATION_PROMPT
+from synkro.schemas import FollowUpQuestion
+
+
+QuestionType = Literal["clarification", "edge_case", "what_if", "specificity", "challenge"]
+
+# Question type progression for multi-turn conversations
+# Earlier turns focus on clarification, later turns probe deeper
+QUESTION_TYPE_BY_TURN = {
+    1: "clarification",
+    2: "specificity",
+    3: "edge_case",
+    4: "what_if",
+    5: "challenge",
+}
+
+
+class FollowUpGenerator:
+    """
+    Generates follow-up questions for multi-turn conversations.
+
+    Uses different question types based on turn index:
+    - Turn 1: clarification - Ask for more details
+    - Turn 2: specificity - Drill into specifics
+    - Turn 3: edge_case - Probe boundary conditions
+    - Turn 4: what_if - Explore hypotheticals
+    - Turn 5+: challenge - Question reasoning
+
+    Examples:
+        >>> gen = FollowUpGenerator()
+        >>> follow_up = await gen.generate(policy_text, messages, turn_index=2)
+        >>> print(follow_up.question)
+    """
+
+    def __init__(self, llm: LLM | None = None, model: Model = OpenAI.GPT_4O_MINI):
+        """
+        Initialize the follow-up generator.
+
+        Args:
+            llm: LLM client to use (creates one if not provided)
+            model: Model to use if creating LLM
+        """
+        self.llm = llm or LLM(model=model)
+
+    def _select_question_type(self, turn_index: int) -> QuestionType:
+        """
+        Select question type based on turn index.
+
+        Args:
+            turn_index: Which turn this is (1-based, counting user-assistant exchanges)
+
+        Returns:
+            Appropriate question type for this turn
+        """
+        if turn_index in QUESTION_TYPE_BY_TURN:
+            return QUESTION_TYPE_BY_TURN[turn_index]
+        # For turns beyond 5, cycle through challenging questions
+        return "challenge"
+
+    def _format_conversation(self, messages: list[Message]) -> str:
+        """Format conversation messages for prompt inclusion."""
+        formatted = []
+        for msg in messages:
+            role = msg.role.upper()
+            content = msg.content or "[No content]"
+            formatted.append(f"{role}: {content}")
+        return "\n\n".join(formatted)
+
+    async def generate(
+        self,
+        policy_text: str,
+        messages: list[Message],
+        turn_index: int,
+        question_type: QuestionType | None = None,
+        scenario_index: int = 0,
+    ) -> FollowUpQuestion:
+        """
+        Generate a follow-up question for the conversation.
+
+        Args:
+            policy_text: The policy text for context
+            messages: Conversation messages so far
+            turn_index: Which turn this is (1-based)
+            question_type: Override auto-selected question type
+            scenario_index: Index for the scenario (default 0)
+
+        Returns:
+            FollowUpQuestion with the generated question
+        """
+        # Select question type if not specified
+        if question_type is None:
+            question_type = self._select_question_type(turn_index)
+
+        # Format conversation for prompt
+        conversation = self._format_conversation(messages)
+
+        # Build prompt
+        prompt = FOLLOW_UP_GENERATION_PROMPT.format(
+            question_type=question_type,
+            conversation=conversation,
+            policy=policy_text,
+        )
+
+        try:
+            # Generate the follow-up question
+            response = await self.llm.generate(prompt)
+            question_text = response.strip()
+
+            return FollowUpQuestion(
+                index=scenario_index,
+                question=question_text,
+                question_type=question_type,
+            )
+        except Exception:
+            # Fallback generic follow-up
+            fallback_questions = {
+                "clarification": "Can you clarify that further?",
+                "edge_case": "What about edge cases?",
+                "what_if": "What if the situation changes?",
+                "specificity": "Can you be more specific?",
+                "challenge": "Why is that the best approach?",
+            }
+            return FollowUpQuestion(
+                index=scenario_index,
+                question=fallback_questions.get(question_type, "Can you elaborate?"),
+                question_type=question_type,
+            )