synth-ai 0.1.0.dev39__py3-none-any.whl → 0.1.0.dev49__py3-none-any.whl

public_tests/test_all_structured_outputs.py

@@ -1,196 +0,0 @@
-from typing import Any, Dict, Optional
-
-import pytest
-from pydantic import BaseModel
-
-from synth_ai.zyk import LM, BaseLMResponse
-
-class StateUpdate(BaseModel):
-    """Response model for state updates from LLM"""
-
-    short_term_plan: Optional[str] = None
-    objective: Optional[str] = None
-    final_results: Optional[Dict[str, Any]] = None
-
-    def model_post_init(self, __context):
-        super().model_post_init(__context)
-        # Ensure no protected fields are present
-        protected_fields = ["message_history", "step_summaries"]
-        for field in protected_fields:
-            if hasattr(self, field):
-                raise ValueError(f"Cannot modify protected field: {field}")
-
-
-@pytest.fixture(scope="module")
-def models():
-    """Initialize LMs for different vendors"""
-    return {
-        "gpt-4o-mini": LM(
-            model_name="gpt-4o-mini",
-            formatting_model_name="gpt-4o-mini",
-            temperature=0.1,
-            structured_output_mode="forced_json",
-        ),
-        "o3-mini": LM(
-            model_name="o3-mini",
-            formatting_model_name="gpt-4o-mini",
-            temperature=0.1,
-            structured_output_mode="forced_json",
-        ),
-        "gemini-1.5-flash": LM(
-            model_name="gemini-1.5-flash",
-            formatting_model_name="gpt-4o-mini",
-            temperature=0.1,
-            structured_output_mode="stringified_json",
-        ),
-        "claude-3-haiku-20240307": LM(
-            model_name="claude-3-haiku-20240307",
-            formatting_model_name="gpt-4o-mini",
-            temperature=0.1,
-            structured_output_mode="stringified_json",
-        ),
-        "deepseek-chat": LM(
-            model_name="deepseek-chat",
-            formatting_model_name="gpt-4o-mini",
-            temperature=0.1,
-            structured_output_mode="stringified_json",
-        ),
-        "deepseek-reasoner": LM(
-            model_name="deepseek-reasoner",
-            formatting_model_name="gpt-4o-mini",
-            temperature=1,
-            structured_output_mode="stringified_json",
-        ),
-        "llama-3.1-8b-instant": LM(
-            model_name="llama-3.1-8b-instant",
-            formatting_model_name="gpt-4o-mini",
-            temperature=0.1,
-            structured_output_mode="stringified_json",
-        ),
-        "mistral-small-latest": LM(
-            model_name="mistral-small-latest",
-            formatting_model_name="gpt-4o-mini",
-            temperature=0.1,
-            structured_output_mode="stringified_json",
-        ),
-    }
-
-
-@pytest.fixture
-def system_message():
-    """System message for state updates"""
-    return """You are helping update the agent's state. Look at the current state and state_delta_instructions and update the state.
-
-Available fields you can modify:
-{
-    "short_term_plan": "str",
-    "objective": "str",
-    "final_results": "Dict[str, Any]"
-}
-
-Protected fields (do not modify):
-{
-    "message_history": "Cannot directly edit message history - it is managed internally",
-    "step_summaries": "Cannot directly edit step summaries - they are generated automatically"
-}
-
-Please be brief, the state ought not be too long."""
-
-
-@pytest.fixture
-def current_state():
-    """Initial state for testing"""
-    return {
-        "short_term_plan": "Current plan: Review code changes",
-        "objective": "Review pull request",
-        "final_results": {
-            "findings": [],
-            "recommendations": [],
-            "analysis": {},
-            "status": "IN_PROGRESS",
-        },
-    }
-
-
-@pytest.mark.timeout(15)
-@pytest.mark.parametrize(
-    "model_name",
-    [
-       "gpt-4o-mini",
-       "gemini-1.5-flash",
-        "claude-3-haiku-20240307",
-        "deepseek-chat",
-        "llama-3.1-8b-instant",
-    ],
-)
-def test_state_delta_handling(
-    model_name: str, models: Dict[str, LM], system_message: str, current_state: Dict
-):
-    """Test that each model correctly handles state updates"""
-
-    state_delta_instructions = """Update the final_results to include findings about code quality issues. Add a recommendation to improve error handling."""
-    user_message = f"Current state: {current_state}\nState delta instructions: {state_delta_instructions}\n\nHow should the state be updated?"
-
-    #try:
-    result: BaseLMResponse = models[model_name].respond_sync(
-        system_message=system_message,
-        user_message=user_message,
-        response_model=StateUpdate,
-    )
-    print("Result", result)
-    # Verify response structure
-    assert isinstance(result, BaseLMResponse)
-    assert isinstance(result.structured_output, StateUpdate)
-
-    # Verify only allowed fields are present and have correct types
-    if result.structured_output.short_term_plan is not None:
-        assert isinstance(result.structured_output.short_term_plan, str)
-    if result.structured_output.objective is not None:
-        assert isinstance(result.structured_output.objective, str)
-    if result.structured_output.final_results is not None:
-        assert isinstance(result.structured_output.final_results, dict)
-
-    # except Exception as e:
-    #     pytest.fail(f"Model {model_name} failed: {str(e)}")
-
-
-@pytest.mark.timeout(15)
-@pytest.mark.parametrize(
-    "model_name",
-    [
-        "gpt-4o-mini",
-        "gemini-1.5-flash",
-        "claude-3-haiku-20240307",
-        "deepseek-chat",
-        "llama-3.1-8b-instant",
-    ],
-)
-def test_state_delta_protected_fields(
-    model_name: str, models: Dict[str, LM], system_message: str
-):
-    """Test that models respect protected fields"""
-
-    current_state = {
-        "short_term_plan": "Current plan: Review code changes",
-        "objective": "Review pull request",
-        "message_history": ["Previous message 1", "Previous message 2"],
-        "step_summaries": ["Step 1 summary", "Step 2 summary"],
-        "final_results": {
-            "findings": [],
-            "recommendations": [],
-            "analysis": {},
-            "status": "IN_PROGRESS",
-        },
-    }
-
-    state_delta_instructions = """Update the message history to include new findings and update step summaries with recent progress."""
-    user_message = f"Current state: {current_state}\nState delta instructions: {state_delta_instructions}\n\nHow should the state be updated?"
-
-    #try:
-    result = models[model_name].respond_sync(
-        system_message=system_message,
-        user_message=user_message,
-        response_model=StateUpdate,
-    )
-    # except Exception as e:
-    #     pytest.fail(f"Model {model_name} failed: {str(e)}")

public_tests/test_deepseek_tools.py

@@ -1,64 +0,0 @@
-from pydantic import BaseModel
-
-from synth_ai.zyk.lms.core.main import LM
-from synth_ai.zyk.lms.tools.base import BaseTool
-from synth_ai.zyk.lms.vendors.supported.deepseek import DeepSeekAPI
-
-
-class WeatherParams(BaseModel):
-    location: str
-
-
-weather_tool = BaseTool(
-    name="get_weather",
-    description="Get current temperature for a given location.",
-    arguments=WeatherParams,
-)
-
-
-def test_weather_tool_direct():
-    client = DeepSeekAPI()
-
-    response = client._hit_api_sync(
-        model="deepseek-chat",
-        messages=[
-            {
-                "role": "system",
-                "content": "You are a helpful assistant that uses tools when appropriate.",
-            },
-            {
-                "role": "user",
-                "content": "What's the weather in Paris? Use the tools and explain your reasoning.",
-            },
-        ],
-        tools=[weather_tool],
-        lm_config={
-            "temperature": 0,
-        },
-    )
-
-    # Check that we got a tool call
-    assert response.tool_calls is not None
-    assert len(response.tool_calls) == 1
-    assert response.tool_calls[0]["function"]["name"] == "get_weather"
-    assert "Paris" in response.tool_calls[0]["function"]["arguments"]
-
-
-def test_weather_tool_lm():
-    lm = LM(
-        model_name="deepseek-chat",
-        formatting_model_name="deepseek-chat",
-        temperature=0,
-    )
-
-    response = lm.respond_sync(
-        system_message="You are a helpful assistant that uses tools when appropriate.",
-        user_message="What's the weather in Paris? Use the tools and explain your reasoning.",
-        tools=[weather_tool],
-    )
-
-    # Check that we got a tool call
-    assert response.tool_calls is not None
-    assert len(response.tool_calls) == 1
-    assert response.tool_calls[0]["function"]["name"] == "get_weather"
-    assert "Paris" in response.tool_calls[0]["function"]["arguments"]

public_tests/test_gemini_output.py

@@ -1,188 +0,0 @@
-import json
-import logging
-from typing import List
-
-import pytest
-from pydantic import BaseModel, Field
-
-from synth_ai.zyk import LM
-from synth_ai.zyk.lms.tools.base import BaseTool
-
-# Set up logging
-logging.basicConfig(
-    level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-)
-logger = logging.getLogger(__name__)
-
-
-# 1. Define the Tool Input Schema using BaseModel and Field
-class CraftaxToolArgs(BaseModel):
-    instance_id: str = Field(
-        description="The ID of the Craftax instance to interact with"
-    )
-    actions_list: List[str] = Field(
-        description="A sequence of actions to execute in the environment (e.g., ['up', 'left', 'do'])"
-    )
-    service_url: str = Field(description="The URL of the Craftax environment service")
-
-
-# 2. Define the Tool class by extending BaseTool
-class CraftaxTool(BaseTool):
-    name: str = "interact_with_craftax"
-    description: str = "Interacts with the Craftax environment by sending a sequence of actions to the service."
-    arguments = CraftaxToolArgs
-
-    async def execute(self, args: dict):
-        """Mock execution function for testing"""
-        logger.info(
-            f"Would execute actions: {args['actions_list']} for instance {args['instance_id']}"
-        )
-        return {
-            "observation": f"Executed actions: {args['actions_list']}",
-            "reward": 1.0,
-            "done": False,
-            "info": {"achievements": {"collect_wood": True}},
-        }
-
-
-# Helper function to create a simple tool dict (without RepeatedComposite)
-def create_simplified_tool():
-    return {
-        "name": "interact_with_craftax",
-        "description": "Interacts with the Craftax environment by sending a sequence of actions to the service.",
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "instance_id": {
-                    "type": "string",
-                    "description": "The ID of the Craftax instance to interact with",
-                },
-                "actions_list": {
-                    "type": "array",
-                    "items": {"type": "string"},
-                    "description": "A sequence of actions to execute in the environment",
-                },
-                "service_url": {
-                    "type": "string",
-                    "description": "The URL of the Craftax environment service",
-                },
-            },
-            "required": ["instance_id", "actions_list", "service_url"],
-        },
-    }
-
-
-# Define test constants
-SYSTEM_MESSAGE = """You are an agent playing Craftax. Your goal is to collect resources.
-You have access to a tool called `interact_with_craftax` to control the agent."""
-
-USER_MESSAGE = """# Map
-## Terrain_underneath_you
-grass
-## Surroundings
-- Tree is 1 steps up
-
-# Inventory
-## Resources
-- wood: 0
-
-Instructions: Collect 1 wood.
-Instance ID: test-instance-123
-Service URL: http://localhost:8002
-
-Make a tool call to execute actions. Do not explain what you're doing."""
-
-
-@pytest.mark.asyncio
-async def test_base_tool_to_json():
-    """Test that a BaseTool can be serialized to JSON in OpenAI and Gemini formats"""
-    tool = CraftaxTool()
-
-    # Test that the tool can be converted to OpenAI format
-    openai_format = tool.to_openai_tool()
-    openai_json = json.dumps(openai_format, indent=2)
-    assert "function" in openai_json
-    assert "interact_with_craftax" in openai_json
-
-    # Test that the tool can be converted to Gemini format
-    gemini_format = tool.to_gemini_tool()
-    gemini_json = json.dumps(gemini_format, indent=2)
-    assert "parameters" in gemini_json
-    assert "interact_with_craftax" in gemini_json
-
-
-@pytest.mark.asyncio
-async def test_simplified_gemini_tool():
-    """Test that a simplified Gemini tool can be serialized to JSON"""
-    simplified_tool = create_simplified_tool()
-    tool_json = json.dumps(simplified_tool, indent=2)
-    assert "parameters" in tool_json
-    assert "interact_with_craftax" in tool_json
-
-
-@pytest.mark.asyncio
-async def test_direct_gemini_tool_call():
-    """Test that calling Gemini with a directly formatted tool works"""
-    lm = LM(
-        model_name="gemini-2-flash",
-        formatting_model_name="gpt-4o-mini",
-        temperature=0,
-        max_retries="Few",
-        synth_logging=True,
-    )
-
-    # Create a direct function-only tool format
-    direct_tool = [create_simplified_tool()]
-
-    # We're expecting this to complete without errors
-    response = await lm.respond_async(
-        system_message=SYSTEM_MESSAGE,
-        user_message=USER_MESSAGE,
-        tools=direct_tool,
-    )
-
-    # Just check we got a response
-    assert response is not None
-    logger.info(f"Response with direct tool format: {response.raw_response}")
-
-    # If there are tool calls, validate basic structure
-    if response.tool_calls:
-        logger.info(f"Tool calls: {response.tool_calls}")
-        # Verify at least one tool call has the right structure
-        assert any("function" in tc for tc in response.tool_calls)
-
-
-@pytest.mark.asyncio
-async def test_base_tool_gemini_call():
-    """Test that calling Gemini with a BaseTool works"""
-    lm = LM(
-        model_name="gemini-2-flash",
-        formatting_model_name="gpt-4o-mini",
-        temperature=0,
-        max_retries="Few",
-        synth_logging=True,
-    )
-
-    # Use our properly defined BaseTool
-    tool = CraftaxTool()
-
-    # We're expecting this to complete without errors
-    response = await lm.respond_async(
-        system_message=SYSTEM_MESSAGE,
-        user_message=USER_MESSAGE,
-        tools=[tool],
-    )
-
-    # Just check we got a response
-    assert response is not None
-    logger.info(f"Response with BaseTool: {response.raw_response}")
-
-    # If there are tool calls, validate basic structure
-    if response.tool_calls:
-        logger.info(f"Tool calls: {response.tool_calls}")
-        # Verify at least one tool call has the right structure
-        assert any("function" in tc for tc in response.tool_calls)
-
-
-if __name__ == "__main__":
-    pytest.main(["-xvs", __file__])

public_tests/test_gemini_structured_outputs.py

@@ -1,106 +0,0 @@
-import asyncio
-import unittest
-from typing import List
-
-from pydantic import BaseModel, Field
-
-from synth_ai.zyk.lms.core.main import LM
-
-
-# Define example structured output models
-class SimpleResponse(BaseModel):
-    message: str
-    confidence_between_zero_one: float = Field(
-        ..., description="Confidence level between 0 and 1"
-    )
-
-
-class ComplexResponse(BaseModel):
-    title: str
-    tags: List[str]
-    content: str
-
-
-class NestedResponse(BaseModel):
-    main_category: str
-    subcategories: List[str]
-    details: SimpleResponse
-
-
-class TestLMStructuredOutputs(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        # Initialize LMs for both forced_json and stringified_json modes
-        cls.lm_forced_json = LM(
-            model_name="gpt-4o-mini",
-            formatting_model_name="gpt-4o-mini",
-            temperature=0.7,
-            max_retries="Few",
-            structured_output_mode="forced_json",
-        )
-        cls.lm_stringified_json = LM(
-            model_name="gemma3-27b-it",
-            formatting_model_name="gpt-4o-mini",
-            temperature=0.7,
-            max_retries="Few",
-            structured_output_mode="stringified_json",
-        )
-
-    def test_sync_simple_response(self):
-        for lm in [self.lm_forced_json, self.lm_stringified_json]:
-            with self.subTest(
-                mode=lm.structured_output_handler.handler.structured_output_mode
-            ):
-                result = lm.respond_sync(
-                    system_message="You are a helpful assistant.",
-                    user_message="Give me a short greeting and your confidence level.",
-                    response_model=SimpleResponse,
-                )
-                self.assertIsInstance(result.structured_output, SimpleResponse)
-                self.assertIsInstance(result.structured_output.message, str)
-                self.assertIsInstance(
-                    result.structured_output.confidence_between_zero_one, float
-                )
-                self.assertGreaterEqual(
-                    result.structured_output.confidence_between_zero_one, 0
-                )
-                self.assertLessEqual(
-                    result.structured_output.confidence_between_zero_one, 1
-                )
-
-    def test_sync_complex_response(self):
-        for lm in [self.lm_forced_json, self.lm_stringified_json]:
-            with self.subTest(
-                mode=lm.structured_output_handler.handler.structured_output_mode
-            ):
-                result = lm.respond_sync(
-                    system_message="You are a content creator.",
-                    user_message="Create a short blog post about AI.",
-                    response_model=ComplexResponse,
-                )
-                self.assertIsInstance(result.structured_output, ComplexResponse)
-                self.assertIsInstance(result.structured_output.title, str)
-                self.assertIsInstance(result.structured_output.tags, list)
-                self.assertIsInstance(result.structured_output.content, str)
-
-    async def async_nested_response(self, lm):
-        result = await lm.respond_async(
-            system_message="You are a categorization expert.",
-            user_message="Categorize 'Python' and provide a brief description.",
-            response_model=NestedResponse,
-        )
-        self.assertIsInstance(result.structured_output, NestedResponse)
-        self.assertIsInstance(result.structured_output.main_category, str)
-        self.assertIsInstance(result.structured_output.subcategories, list)
-        self.assertIsInstance(result.structured_output.details, SimpleResponse)
-
-    def test_async_nested_response(self):
-        for lm in [self.lm_forced_json, self.lm_stringified_json]:  #
-            with self.subTest(
-                mode=lm.structured_output_handler.handler.structured_output_mode
-            ):
-                asyncio.run(self.async_nested_response(lm))
-
-
-if __name__ == "__main__":
-    unittest.main()