synkro 0.4.36__py3-none-any.whl

synkro/factory.py

@@ -0,0 +1,291 @@
+"""Component factory for dependency injection.
+
+This module provides a factory for creating pipeline components,
+enabling testability and flexible configuration.
+
+Supports both legacy components and Golden Trace components:
+- Logic Extractor (The Cartographer)
+- Golden Scenario Generator (The Adversary)
+- Golden Response Generator (The Thinker)
+- Trace Verifier (The Auditor)
+- Golden Refiner
+"""
+
+from typing import TYPE_CHECKING
+
+from synkro.llm.client import LLM
+from synkro.modes.config import ModeConfig
+from synkro.generation.planner import Planner
+from synkro.generation.scenarios import ScenarioGenerator
+from synkro.generation.responses import ResponseGenerator
+from synkro.generation.follow_ups import FollowUpGenerator
+from synkro.generation.multiturn_responses import MultiTurnResponseGenerator
+from synkro.quality.grader import Grader
+from synkro.quality.refiner import Refiner
+from synkro.quality.multiturn_grader import MultiTurnGrader
+
+if TYPE_CHECKING:
+    from synkro.types.tool import ToolDefinition
+    from synkro.generation.tool_simulator import ToolSimulator
+    from synkro.generation.tool_responses import ToolCallResponseGenerator
+    from synkro.quality.tool_grader import ToolCallGrader
+    from synkro.quality.tool_refiner import ToolCallRefiner
+    from synkro.generation.logic_extractor import LogicExtractor
+    from synkro.generation.golden_scenarios import GoldenScenarioGenerator
+    from synkro.generation.golden_responses import GoldenResponseGenerator
+    from synkro.generation.golden_tool_responses import GoldenToolCallResponseGenerator
+    from synkro.quality.verifier import TraceVerifier
+    from synkro.quality.golden_refiner import GoldenRefiner
+    from synkro.interactive.logic_map_editor import LogicMapEditor
+    from synkro.interactive.scenario_editor import ScenarioEditor
+
+
+class ComponentFactory:
+    """
+    Factory for creating pipeline components with shared LLM clients.
+    
+    This centralizes component creation and ensures consistent configuration
+    across the pipeline.
+    
+    Examples:
+        >>> factory = ComponentFactory(gen_llm, grade_llm, mode_config)
+        >>> planner = factory.create_planner()
+        >>> grader = factory.create_grader()
+        
+        >>> # With tools for tool_call dataset type
+        >>> factory = ComponentFactory(gen_llm, grade_llm, mode_config, tools=[...])
+        >>> simulator = factory.create_tool_simulator()
+    """
+    
+    def __init__(
+        self,
+        generation_llm: LLM,
+        grading_llm: LLM,
+        mode_config: ModeConfig,
+        tools: list["ToolDefinition"] | None = None,
+        thinking: bool = False,
+    ):
+        """
+        Initialize the factory.
+
+        Args:
+            generation_llm: LLM client for generation tasks (scenarios, responses, refinement)
+            grading_llm: LLM client for grading and planning (typically stronger model)
+            mode_config: Configuration for the dataset type (prompts, etc.)
+            tools: Optional list of tool definitions for tool_call dataset type
+            thinking: Enable thinking mode with <think> tags in responses
+        """
+        self.generation_llm = generation_llm
+        self.grading_llm = grading_llm
+        self.mode_config = mode_config
+        self.tools = tools or []
+        self.thinking = thinking
+    
+    def create_planner(self) -> Planner:
+        """Create a Planner instance."""
+        return Planner(llm=self.grading_llm)
+    
+    def create_scenario_generator(self) -> ScenarioGenerator:
+        """Create a ScenarioGenerator with mode-specific prompts."""
+        gen = ScenarioGenerator(llm=self.generation_llm)
+        gen.prompt_template = self.mode_config.scenario_prompt
+        return gen
+    
+    def create_response_generator(self) -> ResponseGenerator:
+        """Create a ResponseGenerator with mode-specific prompts."""
+        gen = ResponseGenerator(llm=self.generation_llm)
+        gen.prompt_template = self.mode_config.response_prompt
+        return gen
+    
+    def create_grader(self) -> "Grader | ToolCallGrader":
+        """
+        Create a Grader with mode-specific prompts.
+        
+        Auto-selects ToolCallGrader when tools are configured.
+        """
+        if self.has_tools:
+            from synkro.quality.tool_grader import ToolCallGrader
+            return ToolCallGrader(llm=self.grading_llm, tools=self.tools)
+        
+        grader = Grader(llm=self.grading_llm)
+        grader.prompt_template = self.mode_config.grade_prompt
+        return grader
+    
+    def create_refiner(self) -> "Refiner | ToolCallRefiner":
+        """
+        Create a Refiner with mode-specific prompts.
+        
+        Auto-selects ToolCallRefiner when tools are configured.
+        This ensures tool_calls format is preserved during refinement.
+        """
+        if self.has_tools:
+            from synkro.quality.tool_refiner import ToolCallRefiner
+            simulator = self.create_tool_simulator()
+            return ToolCallRefiner(
+                llm=self.generation_llm,
+                tools=self.tools,
+                simulator=simulator,
+            )
+        
+        refiner = Refiner(llm=self.generation_llm)
+        refiner.prompt_template = self.mode_config.refine_prompt
+        return refiner
+    
+    def create_tool_simulator(self) -> "ToolSimulator":
+        """Create a ToolSimulator instance for tool_call dataset type."""
+        from synkro.generation.tool_simulator import ToolSimulator
+        
+        if not self.tools:
+            raise ValueError("Cannot create ToolSimulator without tools")
+        
+        return ToolSimulator(tools=self.tools, llm=self.generation_llm)
+    
+    def create_tool_call_response_generator(self) -> "ToolCallResponseGenerator":
+        """
+        Create a ToolCallResponseGenerator for generating proper tool call traces.
+        
+        This generator uses JSON mode to produce structured tool calls in
+        OpenAI function calling format.
+        """
+        from synkro.generation.tool_responses import ToolCallResponseGenerator
+        
+        if not self.tools:
+            raise ValueError("Cannot create ToolCallResponseGenerator without tools")
+        
+        # Create simulator for generating tool responses
+        simulator = self.create_tool_simulator()
+        
+        return ToolCallResponseGenerator(
+            tools=self.tools,
+            llm=self.generation_llm,
+            simulator=simulator,
+        )
+    
+    def get_tools_description(self) -> str:
+        """Get formatted description of all available tools."""
+        if not self.tools:
+            return "No tools available"
+        
+        descriptions = []
+        for tool in self.tools:
+            descriptions.append(tool.to_system_prompt())
+        return "\n\n".join(descriptions)
+    
+    @property
+    def has_tools(self) -> bool:
+        """Check if tools are configured."""
+        return bool(self.tools)
+
+    def create_follow_up_generator(self) -> FollowUpGenerator:
+        """Create a FollowUpGenerator for multi-turn conversations."""
+        return FollowUpGenerator(llm=self.generation_llm)
+
+    def create_multi_turn_response_generator(self) -> MultiTurnResponseGenerator:
+        """Create a MultiTurnResponseGenerator for multi-turn trace generation."""
+        return MultiTurnResponseGenerator(llm=self.generation_llm)
+
+    def create_multi_turn_grader(self) -> MultiTurnGrader:
+        """Create a MultiTurnGrader for per-turn and overall conversation grading."""
+        return MultiTurnGrader(llm=self.grading_llm)
+
+    # =========================================================================
+    # GOLDEN TRACE COMPONENTS
+    # =========================================================================
+
+    def create_logic_extractor(self) -> "LogicExtractor":
+        """
+        Create a LogicExtractor (The Cartographer).
+
+        Uses the grading LLM (stronger model) for accurate rule extraction.
+        """
+        from synkro.generation.logic_extractor import LogicExtractor
+        return LogicExtractor(llm=self.grading_llm)
+
+    def create_golden_scenario_generator(self) -> "GoldenScenarioGenerator":
+        """
+        Create a GoldenScenarioGenerator (The Adversary).
+
+        Generates typed scenarios (positive, negative, edge_case, irrelevant)
+        with rule targeting.
+        """
+        from synkro.generation.golden_scenarios import GoldenScenarioGenerator
+        return GoldenScenarioGenerator(llm=self.generation_llm)
+
+    def create_golden_response_generator(self) -> "GoldenResponseGenerator":
+        """
+        Create a GoldenResponseGenerator (The Thinker).
+
+        Generates traces with grounded Chain-of-Thought reasoning
+        and rule citations.
+        """
+        from synkro.generation.golden_responses import GoldenResponseGenerator
+        return GoldenResponseGenerator(llm=self.generation_llm, thinking=self.thinking)
+
+    def create_golden_tool_call_generator(self) -> "GoldenToolCallResponseGenerator":
+        """
+        Create a GoldenToolCallResponseGenerator (The Thinker for Tools).
+
+        Generates tool call traces with rule citations for tool selection
+        decisions. Requires tools to be configured.
+        """
+        from synkro.generation.golden_tool_responses import GoldenToolCallResponseGenerator
+
+        if not self.tools:
+            raise ValueError("Cannot create GoldenToolCallResponseGenerator without tools")
+
+        simulator = self.create_tool_simulator()
+        return GoldenToolCallResponseGenerator(
+            tools=self.tools,
+            llm=self.generation_llm,
+            simulator=simulator,
+            thinking=self.thinking,
+        )
+
+    def create_verifier(self) -> "TraceVerifier":
+        """
+        Create a TraceVerifier (The Auditor).
+
+        Verifies traces against the Logic Map to ensure:
+        - No skipped rules
+        - No hallucinated rules
+        - No contradictions
+        - DAG compliance
+
+        Uses the grading LLM (stronger model) for accurate verification.
+        """
+        from synkro.quality.verifier import TraceVerifier
+        return TraceVerifier(llm=self.grading_llm)
+
+    def create_golden_refiner(self) -> "GoldenRefiner":
+        """
+        Create a GoldenRefiner.
+
+        Refines traces that fail verification, using Logic Map context
+        to fix skipped rules, hallucinations, and contradictions.
+        """
+        from synkro.quality.golden_refiner import GoldenRefiner
+        return GoldenRefiner(llm=self.generation_llm)
+
+    def create_logic_map_editor(self) -> "LogicMapEditor":
+        """
+        Create a LogicMapEditor for Human-in-the-Loop sessions.
+
+        The editor uses the grading LLM (stronger model) to interpret
+        natural language feedback and refine Logic Maps.
+        """
+        from synkro.interactive.logic_map_editor import LogicMapEditor
+        return LogicMapEditor(llm=self.grading_llm)
+
+    def create_scenario_editor(self) -> "ScenarioEditor":
+        """
+        Create a ScenarioEditor for Human-in-the-Loop scenario editing.
+
+        The editor uses the grading LLM (stronger model) to interpret
+        natural language feedback and refine scenarios.
+        """
+        from synkro.interactive.scenario_editor import ScenarioEditor
+        return ScenarioEditor(llm=self.grading_llm)
+
+
+__all__ = ["ComponentFactory"]
+

synkro/formatters/__init__.py

@@ -0,0 +1,18 @@
+"""Output formatters for different training data formats."""
+
+from synkro.formatters.sft import SFTFormatter
+from synkro.formatters.tool_call import ToolCallFormatter
+from synkro.formatters.chatml import ChatMLFormatter
+from synkro.formatters.qa import QAFormatter
+from synkro.formatters.langsmith import LangSmithFormatter
+from synkro.formatters.langfuse import LangfuseFormatter
+
+__all__ = [
+    "SFTFormatter",
+    "ToolCallFormatter",
+    "ChatMLFormatter",
+    "QAFormatter",
+    "LangSmithFormatter",
+    "LangfuseFormatter",
+]
+

synkro/formatters/chatml.py

@@ -0,0 +1,121 @@
+"""ChatML formatter with XML tags for tool calling."""
+
+import json
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from synkro.types.core import Trace
+
+
+class ChatMLFormatter:
+    """
+    Format traces as ChatML with XML tags for tool calls.
+
+    Uses <tool_call> and <tool_response> XML tags for tool interactions,
+    compatible with Hermes/Mistral style fine-tuning.
+
+    Example output:
+        {
+          "messages": [
+            {"role": "system", "content": "You have access to tools."},
+            {"role": "user", "content": "What's the weather in NYC?"},
+            {"role": "assistant", "content": "<tool_call>\\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}\\n</tool_call>"},
+            {"role": "tool", "content": "<tool_response>\\n{\"temp\": \"72F\"}\\n</tool_response>"},
+            {"role": "assistant", "content": "The weather in NYC is 72°F."}
+          ]
+        }
+    """
+
+    def __init__(self, include_metadata: bool = False):
+        """
+        Initialize the ChatMLFormatter.
+
+        Args:
+            include_metadata: If True, include trace metadata in output
+        """
+        self.include_metadata = include_metadata
+
+    def format(self, traces: list["Trace"]) -> list[dict]:
+        """
+        Format traces as ChatML with XML tags.
+
+        Args:
+            traces: List of traces to format
+
+        Returns:
+            List of formatted examples
+        """
+        examples = []
+
+        for trace in traces:
+            messages = []
+
+            for m in trace.messages:
+                # Handle assistant messages with tool calls
+                if m.role == "assistant" and m.tool_calls:
+                    # Convert tool calls to XML format
+                    tool_call_xmls = []
+                    for tc in m.tool_calls:
+                        tool_call_json = json.dumps({
+                            "name": tc.function.name,
+                            "arguments": json.loads(tc.function.arguments)
+                        })
+                        tool_call_xmls.append(f"<tool_call>\n{tool_call_json}\n</tool_call>")
+
+                    content = "\n".join(tool_call_xmls)
+                    messages.append({"role": "assistant", "content": content})
+
+                # Handle tool responses
+                elif m.role == "tool":
+                    content = f"<tool_response>\n{m.content}\n</tool_response>"
+                    messages.append({"role": "tool", "content": content})
+
+                # Regular messages (system, user, assistant without tools)
+                else:
+                    messages.append({
+                        "role": m.role,
+                        "content": m.content or ""
+                    })
+
+            example = {"messages": messages}
+
+            if self.include_metadata:
+                example["metadata"] = {
+                    "scenario": trace.scenario.description,
+                    "category": trace.scenario.category,
+                    "grade": trace.grade.model_dump() if trace.grade else None,
+                    "has_tool_calls": trace.has_tool_calls,
+                }
+
+            examples.append(example)
+
+        return examples
+
+    def save(self, traces: list["Trace"], path: str | Path) -> None:
+        """
+        Save formatted traces to a JSONL file.
+
+        Args:
+            traces: List of traces to save
+            path: Output file path
+        """
+        path = Path(path)
+        examples = self.format(traces)
+
+        with open(path, "w") as f:
+            for example in examples:
+                f.write(json.dumps(example) + "\n")
+
+    def to_jsonl(self, traces: list["Trace"]) -> str:
+        """
+        Convert traces to JSONL string.
+
+        Args:
+            traces: List of traces to convert
+
+        Returns:
+            JSONL formatted string
+        """
+        examples = self.format(traces)
+        return "\n".join(json.dumps(e) for e in examples)

synkro/formatters/langfuse.py

@@ -0,0 +1,98 @@
+"""Langfuse formatter for evaluation datasets."""
+
+import json
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from synkro.types.core import Trace
+
+
+class LangfuseFormatter:
+    """
+    Format traces for Langfuse datasets.
+
+    Langfuse format uses input/expectedOutput structure:
+    - input: any JSON object with input data
+    - expectedOutput: any JSON object with expected output
+    - metadata: optional key-value pairs
+
+    Example output:
+        {
+            "input": {
+                "question": "Can I submit a $200 expense without a receipt?",
+                "context": "Expense: $200, No receipt"
+            },
+            "expectedOutput": {
+                "answer": "All expenses require receipts...",
+                "expected_outcome": "Deny - missing receipt"
+            },
+            "metadata": {
+                "ground_truth_rules": ["R003"],
+                "difficulty": "negative",
+                "category": "Receipt Requirements"
+            }
+        }
+    """
+
+    def format(self, traces: list["Trace"]) -> list[dict]:
+        """
+        Format traces as Langfuse dataset items.
+
+        Args:
+            traces: List of traces to format
+
+        Returns:
+            List of Langfuse-compatible dataset items
+        """
+        examples = []
+
+        for trace in traces:
+            example = {
+                "input": {
+                    "question": trace.user_message,
+                    "context": trace.scenario.context or "",
+                },
+                "expectedOutput": {
+                    "answer": trace.assistant_message,
+                    "expected_outcome": trace.scenario.expected_outcome or "",
+                },
+                "metadata": {
+                    "ground_truth_rules": trace.scenario.target_rule_ids or [],
+                    "difficulty": trace.scenario.scenario_type or "unknown",
+                    "category": trace.scenario.category or "",
+                    "passed": trace.grade.passed if trace.grade else None,
+                },
+            }
+
+            examples.append(example)
+
+        return examples
+
+    def save(self, traces: list["Trace"], path: str | Path) -> None:
+        """
+        Save formatted traces to a JSONL file.
+
+        Args:
+            traces: List of traces to save
+            path: Output file path
+        """
+        path = Path(path)
+        examples = self.format(traces)
+
+        with open(path, "w") as f:
+            for example in examples:
+                f.write(json.dumps(example) + "\n")
+
+    def to_jsonl(self, traces: list["Trace"]) -> str:
+        """
+        Convert traces to JSONL string.
+
+        Args:
+            traces: List of traces to convert
+
+        Returns:
+            JSONL formatted string
+        """
+        examples = self.format(traces)
+        return "\n".join(json.dumps(e) for e in examples)

synkro/formatters/langsmith.py

@@ -0,0 +1,98 @@
+"""LangSmith formatter for evaluation datasets."""
+
+import json
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from synkro.types.core import Trace
+
+
+class LangSmithFormatter:
+    """
+    Format traces for LangSmith datasets.
+
+    LangSmith format uses nested inputs/outputs structure:
+    - inputs: dict of input fields
+    - outputs: dict of expected output fields
+    - metadata: optional additional info
+
+    Example output:
+        {
+            "inputs": {
+                "question": "Can I submit a $200 expense without a receipt?",
+                "context": "Expense: $200, No receipt"
+            },
+            "outputs": {
+                "answer": "All expenses require receipts..."
+            },
+            "metadata": {
+                "expected_outcome": "Deny - missing receipt",
+                "ground_truth_rules": ["R003"],
+                "difficulty": "negative",
+                "category": "Receipt Requirements"
+            }
+        }
+    """
+
+    def format(self, traces: list["Trace"]) -> list[dict]:
+        """
+        Format traces as LangSmith dataset examples.
+
+        Args:
+            traces: List of traces to format
+
+        Returns:
+            List of LangSmith-compatible examples
+        """
+        examples = []
+
+        for trace in traces:
+            example = {
+                "inputs": {
+                    "question": trace.user_message,
+                    "context": trace.scenario.context or "",
+                },
+                "outputs": {
+                    "answer": trace.assistant_message,
+                },
+                "metadata": {
+                    "expected_outcome": trace.scenario.expected_outcome or "",
+                    "ground_truth_rules": trace.scenario.target_rule_ids or [],
+                    "difficulty": trace.scenario.scenario_type or "unknown",
+                    "category": trace.scenario.category or "",
+                    "passed": trace.grade.passed if trace.grade else None,
+                },
+            }
+
+            examples.append(example)
+
+        return examples
+
+    def save(self, traces: list["Trace"], path: str | Path) -> None:
+        """
+        Save formatted traces to a JSONL file.
+
+        Args:
+            traces: List of traces to save
+            path: Output file path
+        """
+        path = Path(path)
+        examples = self.format(traces)
+
+        with open(path, "w") as f:
+            for example in examples:
+                f.write(json.dumps(example) + "\n")
+
+    def to_jsonl(self, traces: list["Trace"]) -> str:
+        """
+        Convert traces to JSONL string.
+
+        Args:
+            traces: List of traces to convert
+
+        Returns:
+            JSONL formatted string
+        """
+        examples = self.format(traces)
+        return "\n".join(json.dumps(e) for e in examples)