synth-ai 0.1.0.dev38__py3-none-any.whl → 0.1.0.dev49__py3-none-any.whl

public_tests/test_models.py

@@ -1,183 +0,0 @@
-import time
-
-import pytest
-
-from synth_ai.zyk import LM
-
-
-# Use pytest fixtures instead of unittest setup
-@pytest.fixture
-def model_instances():
-    """Initialize all model configurations for testing."""
-    models = {
-        # O3-mini standard
-        "o3-mini": LM(
-            model_name="o3-mini",
-            formatting_model_name="gpt-4o-mini",
-            temperature=0.7,
-        ),
-        # O3-mini with high reasoning
-        "o3-mini-high-reasoning": LM(
-            model_name="o3-mini",
-            formatting_model_name="gpt-4o-mini",
-            temperature=0.7,
-        ),
-        # Claude 3 Sonnet
-        "claude-3-7-sonnet-latest": LM(
-            model_name="claude-3-7-sonnet-latest",
-            formatting_model_name="gpt-4o-mini",
-            temperature=0.7,
-        ),
-        # Claude 3 Sonnet with high reasoning
-        "claude-3-7-sonnet-latest-high-reasoning": LM(
-            model_name="claude-3-7-sonnet-latest",
-            formatting_model_name="gpt-4o-mini",
-            temperature=0.7,
-        ),
-        # Gemini Flash
-        "gemini-2-flash": LM(
-            model_name="gemini-2-flash",
-            formatting_model_name="gpt-4o-mini",
-            temperature=0.7,
-        ),
-        # Gemma 3
-        "gemma3-27b-it": LM(
-            model_name="gemma3-27b-it",
-            formatting_model_name="gpt-4o-mini",
-            temperature=0.7,
-        ),
-        # GPT-4o mini
-        "gpt-4o-mini": LM(
-            model_name="gpt-4o-mini",
-            formatting_model_name="gpt-4o-mini",
-            temperature=0.7,
-        ),
-    }
-
-    # Set reasoning_effort to "high" for specific models
-    models["o3-mini-high-reasoning"].lm_config["reasoning_effort"] = "high"
-    models["claude-3-7-sonnet-latest-high-reasoning"].lm_config["reasoning_effort"] = (
-        "high"
-    )
-
-    return models
-
-
-# Convert tests to pytest style
-@pytest.mark.parametrize(
-    "model_name",
-    [
-        "o3-mini",
-        "o3-mini-high-reasoning",
-        "claude-3-7-sonnet-latest",
-        "claude-3-7-sonnet-latest-high-reasoning",
-        "gemini-2-flash",
-        "gemma3-27b-it",
-        "gpt-4o-mini",
-    ],
-)
-def test_model_simple_response(model_instances, model_name):
-    """Test that models can generate a simple response."""
-    lm = model_instances[model_name]
-    system_message = "You are a helpful assistant."
-    user_message = "What is the capital of France?"
-
-    print(f"\nTesting {model_name}...")
-    start_time = time.time()
-
-    response = lm.respond_sync(
-        system_message=system_message,
-        user_message=user_message,
-    )
-    elapsed = time.time() - start_time
-
-    print(f"Response time: {elapsed:.2f} seconds")
-    print(f"Response length: {len(response.raw_response)} characters")
-    print(f"Response sample: {response.raw_response[:100]}...")
-
-    # Basic validation
-    assert isinstance(response.raw_response, str)
-    assert len(response.raw_response) > 0
-    assert (
-        "Paris" in response.raw_response
-    ), f"Expected 'Paris' in response, but got: {response.raw_response[:200]}..."
-
-
-@pytest.mark.asyncio
-@pytest.mark.parametrize(
-    "model_name",
-    [
-   #     "o3-mini",
- #       "claude-3-7-sonnet-latest",
-        "claude-3-7-sonnet-latest-high-reasoning",
-        # "gemini-2-flash",
-        # "gemma3-27b-it",
-        # "gpt-4o-mini",
-    ],
-)
-async def test_reasoning_question(model_instances, model_name):
-    """Test models with a question that requires reasoning."""
-    lm = model_instances[model_name]
-    system_message = "You are a helpful assistant."
-    user_message = "If a train travels at 120 km/h and another train travels at 80 km/h in the opposite direction, how long will it take for them to be 500 km apart if they start 100 km apart?"
-
-    print(f"\nTesting {model_name} with reasoning question...")
-    start_time = time.time()
-
-    response = await lm.respond_async(
-        system_message=system_message,
-        user_message=user_message,
-    )
-    elapsed = time.time() - start_time
-
-    print(f"Response time: {elapsed:.2f} seconds")
-    print(f"Response length: {len(response.raw_response)} characters")
-    print(f"Response sample: {response.raw_response[:100]}...")
-
-    # Basic validation
-    assert isinstance(response.raw_response, str)
-    assert len(response.raw_response) > 0
-
-
-@pytest.mark.parametrize(
-    "model_name",
-    [
-        "o3-mini",
-       # "o3-mini",
-        #"claude-3-7-sonnet-latest",
-        "claude-3-7-sonnet-latest-high-reasoning",
-        # "gemini-2-flash",
-        # "gemma3-27b-it",
-        # "gpt-4o-mini",
-    ],
-)
-def test_model_context_and_factuality(model_instances, model_name):
-    """Test models for factuality with a context-based question."""
-    lm = model_instances[model_name]
-    system_message = "You are a helpful assistant."
-    context = """
-    The city of Atlantis was founded in 1968 by marine archaeologist Dr. Sophia Maris. 
-    It has a population of 37,500 residents and is known for its underwater research facilities. 
-    The current mayor is Dr. Robert Neptune who was elected in 2020.
-    """
-    user_message = f"Based on the following information, when was Atlantis founded and who is its current mayor?\n\n{context}"
-
-    print(f"\nTesting {model_name} for factuality...")
-
-    response = lm.respond_sync(
-        system_message=system_message,
-        user_message=user_message,
-    )
-
-    # Check if the response contains the correct information
-    assert (
-        "1968" in response.raw_response
-    ), f"Expected '1968' in response for founding year, but got: {response.raw_response[:200]}..."
-    assert (
-        "Robert Neptune" in response.raw_response
-    ), f"Expected 'Robert Neptune' in response for mayor, but got: {response.raw_response[:200]}..."
-
-
-if __name__ == "__main__":
-    # For direct script execution
-    pytest.main(["-xvs", __file__])

public_tests/test_openai_structured_outputs.py

@@ -1,106 +0,0 @@
-import asyncio
-import unittest
-from typing import List
-
-from pydantic import BaseModel, Field
-
-from synth_ai.zyk.lms.core.main import LM
-
-
-# Define example structured output models
-class SimpleResponse(BaseModel):
-    message: str
-    confidence_between_zero_one: float = Field(
-        ..., description="Confidence level between 0 and 1"
-    )
-
-
-class ComplexResponse(BaseModel):
-    title: str
-    tags: List[str]
-    content: str
-
-
-class NestedResponse(BaseModel):
-    main_category: str
-    subcategories: List[str]
-    details: SimpleResponse
-
-
-class TestLMStructuredOutputs(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        # Initialize LMs for both forced_json and stringified_json modes
-        cls.lm_forced_json = LM(
-            model_name="gpt-4o-mini",
-            formatting_model_name="gpt-4o-mini",
-            temperature=0.7,
-            max_retries="Few",
-            structured_output_mode="forced_json",
-        )
-        cls.lm_stringified_json = LM(
-            model_name="gpt-4o-mini",
-            formatting_model_name="gpt-4o-mini",
-            temperature=0.7,
-            max_retries="Few",
-            structured_output_mode="stringified_json",
-        )
-
-    def test_sync_simple_response(self):
-        for lm in [self.lm_forced_json, self.lm_stringified_json]:
-            with self.subTest(
-                mode=lm.structured_output_handler.handler.structured_output_mode
-            ):
-                result = lm.respond_sync(
-                    system_message="You are a helpful assistant.",
-                    user_message="Give me a short greeting and your confidence level.",
-                    response_model=SimpleResponse,
-                )
-                self.assertIsInstance(result.structured_output, SimpleResponse)
-                self.assertIsInstance(result.structured_output.message, str)
-                self.assertIsInstance(
-                    result.structured_output.confidence_between_zero_one, float
-                )
-                self.assertGreaterEqual(
-                    result.structured_output.confidence_between_zero_one, 0
-                )
-                self.assertLessEqual(
-                    result.structured_output.confidence_between_zero_one, 1
-                )
-
-    def test_sync_complex_response(self):
-        for lm in [self.lm_forced_json, self.lm_stringified_json]:
-            with self.subTest(
-                mode=lm.structured_output_handler.handler.structured_output_mode
-            ):
-                result = lm.respond_sync(
-                    system_message="You are a content creator.",
-                    user_message="Create a short blog post about AI.",
-                    response_model=ComplexResponse,
-                )
-                self.assertIsInstance(result.structured_output, ComplexResponse)
-                self.assertIsInstance(result.structured_output.title, str)
-                self.assertIsInstance(result.structured_output.tags, list)
-                self.assertIsInstance(result.structured_output.content, str)
-
-    async def async_nested_response(self, lm):
-        result = await lm.respond_async(
-            system_message="You are a categorization expert.",
-            user_message="Categorize 'Python' and provide a brief description.",
-            response_model=NestedResponse,
-        )
-        self.assertIsInstance(result.structured_output, NestedResponse)
-        self.assertIsInstance(result.structured_output.main_category, str)
-        self.assertIsInstance(result.structured_output.subcategories, list)
-        self.assertIsInstance(result.structured_output.details, SimpleResponse)
-
-    def test_async_nested_response(self):
-        for lm in [self.lm_forced_json, self.lm_stringified_json]:  #
-            with self.subTest(
-                mode=lm.structured_output_handler.handler.structured_output_mode
-            ):
-                asyncio.run(self.async_nested_response(lm))
-
-
-if __name__ == "__main__":
-    unittest.main()

public_tests/test_reasoning_effort.py

@@ -1,75 +0,0 @@
-import asyncio
-import pytest
-from synth_ai.zyk.lms.core.main import LM
-FORMATTING_MODEL_NAME = "gpt-4o-mini"
-
-# List of reasoning models to test
-# Note: Ensure these models are correctly configured and accessible in your environment
-# And that they are included in REASONING_MODELS in main.py
-REASONING_MODELS_TO_TEST = [
-    "o4-mini",
-    "claude-3-7-sonnet-latest",
-    "gemini-2.5-pro"
-]
-
-# Define effort levels (adjust if specific models use different terms)
-EFFORT_LEVELS = ["low", "medium", "high"]
-
-@pytest.mark.parametrize("model_name", REASONING_MODELS_TO_TEST)
-@pytest.mark.parametrize("effort", EFFORT_LEVELS)
-@pytest.mark.asyncio
-async def test_reasoning_effort_levels(model_name, effort):
-    """
-    Tests that the reasoning_effort parameter is accepted and calls succeed for various models and levels.
-    Note: This test primarily checks for successful API calls across effort levels.
-    Comparing output length or quality based on 'effort' is complex and model-specific.
-    Anthropic's 'thinking' budget might correlate, but OpenAI/others might handle 'effort' differently or ignore it.
-    """
-    print(f"\nTesting model: {model_name} with effort: {effort}")
-    lm = LM(
-        model_name=model_name,
-        formatting_model_name=FORMATTING_MODEL_NAME,
-        temperature=0,
-    )
-
-    system_prompt = "You are a helpful assistant designed to explain complex topics simply."
-    user_prompt = f"Explain the concept of quantum entanglement step by step using a simple analogy. Be concise if effort is low, detailed if high. Current effort: {effort}."
-
-    try:
-        result = await lm.respond_async(
-            system_message=system_prompt,
-            user_message=user_prompt,
-            reasoning_effort=effort, # Pass the effort level
-        )
-
-        response = result.raw_response
-
-        # Assert call succeeded and response is non-empty
-        assert isinstance(response, str), f"Model {model_name} (effort={effort}) failed. Response type: {type(response)}"
-        assert len(response) > 0, f"Model {model_name} (effort={effort}): Response is empty."
-
-        print(f"  Response length (effort={effort}): {len(response)}")
-        # print(f"  Response snippet: {response[:100]}...") # Optional: print snippet
-
-    except Exception as e:
-        pytest.fail(f"Model {model_name} (effort={effort}) raised an exception: {e}")
-
-# Optional: Add a separate test to compare lengths between low and high effort for specific models if needed.
-
-if __name__ == "__main__":
-    async def main():
-        print("Running effort tests directly...")
-        test_models = REASONING_MODELS_TO_TEST
-        effort_levels_to_run = EFFORT_LEVELS
-
-        all_tasks = []
-        for model in test_models:
-            for effort_level in effort_levels_to_run:
-                 # Create a task for each combination
-                 all_tasks.append(test_reasoning_effort_levels(model, effort_level))
-
-        # Run all tests concurrently (be mindful of rate limits)
-        await asyncio.gather(*all_tasks)
-        print("\nTest run finished.")
-
-    asyncio.run(main()) 

public_tests/test_reasoning_models.py

@@ -1,92 +0,0 @@
-import asyncio
-import os
-import time
-
-from pydantic import BaseModel
-
-from synth_ai.zyk import LM
-
-
-def get_openai_api_key():
-    # Get API key from environment variable
-    return os.environ.get("OPENAI_API_KEY")
-
-
-class TestOutput(BaseModel):
-    answer: str
-    reasoning: str
-
-
-async def test_reasoning_effort():
-    # Define a question that requires reasoning
-    question = "If a train travels at 120 km/h and another train travels at 80 km/h in the opposite direction, how long will it take for them to be 500 km apart if they start 100 km apart?"
-
-    print("Testing o3-mini with different reasoning_effort settings:")
-    print("-" * 60)
-
-    # Create an instance for HIGH reasoning
-    print("Testing with reasoning_effort='high'")
-    lm_high = LM(
-        model_name="o3-mini",
-        formatting_model_name="gpt-4o-mini",
-        temperature=1,
-    )
-    # Set reasoning_effort in lm_config
-    lm_high.lm_config["reasoning_effort"] = "high"
-
-    # Time the API call
-    start_time = time.time()
-    high_result = await lm_high.respond_async(
-        system_message="You are a helpful assistant.",
-        user_message=question,
-    )
-    high_time = time.time() - start_time
-
-    print(f"Time taken: {high_time:.2f} seconds")
-    print(f"Response length: {len(high_result.raw_response)} characters")
-    print("-" * 60)
-
-    # Create a separate instance for LOW reasoning
-    print("Testing with reasoning_effort='low'")
-    lm_low = LM(
-        model_name="o3-mini",
-        formatting_model_name="gpt-4o-mini",
-        temperature=1,
-    )
-    # Set reasoning_effort in lm_config
-    lm_low.lm_config["reasoning_effort"] = "low"
-
-    # Time the API call
-    start_time = time.time()
-    low_result = await lm_low.respond_async(
-        system_message="You are a helpful assistant.",
-        user_message=question,
-    )
-    low_time = time.time() - start_time
-
-    print(f"Time taken: {low_time:.2f} seconds")
-    print(f"Response length: {len(low_result.raw_response)} characters")
-    print("-" * 60)
-
-    # Print comparison
-    print("Results comparison:")
-    print(f"High reasoning time: {high_time:.2f} seconds")
-    print(f"Low reasoning time: {low_time:.2f} seconds")
-    print(
-        f"Difference: {high_time - low_time:.2f} seconds ({(high_time/low_time - 1)*100:.1f}% difference)"
-    )
-    print(f"High response length: {len(high_result.raw_response)} characters")
-    print(f"Low response length: {len(low_result.raw_response)} characters")
-    print(
-        f"Response length ratio: {len(high_result.raw_response)/len(low_result.raw_response):.2f}x"
-    )
-
-    # Print response samples
-    print("\nHIGH Response Sample (first 200 chars):")
-    print(high_result.raw_response[:200] + "...")
-    print("\nLOW Response Sample (first 200 chars):")
-    print(low_result.raw_response[:200] + "...")
-
-
-if __name__ == "__main__":
-    asyncio.run(test_reasoning_effort())

public_tests/test_recursive_structured_outputs.py

@@ -1,180 +0,0 @@
-import asyncio
-import unittest
-from typing import List
-
-from pydantic import BaseModel
-
-from synth_ai.zyk.lms.core.main import LM
-
-
-# Define example structured output models
-class SimpleResponse(BaseModel):
-    message: str
-    confidence: float
-
-
-class ComplexResponse(BaseModel):
-    title: str
-    tags: List[str]
-    content: str
-
-
-class NestedResponse(BaseModel):
-    main_category: str
-    subcategories: List[str]
-    details: SimpleResponse
-
-
-# Define nested structured output models
-class Address(BaseModel):
-    street: str
-    city: str
-    country: str
-
-
-class PersonalInfo(BaseModel):
-    name: str
-    age: int
-    address: Address
-
-
-class WorkInfo(BaseModel):
-    company: str
-    position: str
-    years_experience: int
-
-
-class NestedPersonResponse(BaseModel):
-    personal: PersonalInfo
-    work: WorkInfo
-    skills: List[str]
-
-
-class ProjectDetails(BaseModel):
-    name: str
-    description: str
-    technologies: List[str]
-
-
-class NestedPortfolioResponse(BaseModel):
-    developer: PersonalInfo
-    projects: List[ProjectDetails]
-    total_experience: int
-
-
-class NestedCompanyResponse(BaseModel):
-    name: str
-    founded: int
-    headquarters: Address
-    employees: List[PersonalInfo]
-    main_products: List[str]
-
-
-class TestLMStructuredOutputs(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        # Initialize the LM once for all tests
-        cls.lm = LM(
-            model_name="gpt-4o-mini",
-            formatting_model_name="gpt-4o-mini",
-            temperature=0.7,
-            max_retries="Few",
-            structured_output_mode="forced_json",
-        )
-
-    def test_sync_simple_response(self):
-        result = self.lm.respond_sync(
-            system_message="You are a helpful assistant.",
-            user_message="Give me a short greeting and your confidence level.",
-            response_model=SimpleResponse,
-        )
-        self.assertIsInstance(result.structured_output, SimpleResponse)
-        self.assertIsInstance(result.structured_output.message, str)
-        self.assertIsInstance(result.structured_output.confidence, float)
-        self.assertGreaterEqual(result.structured_output.confidence, 0)
-        self.assertLessEqual(result.structured_output.confidence, 1)
-
-    def test_sync_complex_response(self):
-        result = self.lm.respond_sync(
-            system_message="You are a content creator.",
-            user_message="Create a short blog post about AI.",
-            response_model=ComplexResponse,
-        )
-        self.assertIsInstance(result.structured_output, ComplexResponse)
-        self.assertIsInstance(result.structured_output.title, str)
-        self.assertIsInstance(result.structured_output.tags, list)
-        self.assertIsInstance(result.structured_output.content, str)
-
-    async def async_nested_response(self):
-        result = await self.lm.respond_async(
-            system_message="You are a categorization expert.",
-            user_message="Categorize 'Python' and provide a brief description.",
-            response_model=NestedResponse,
-        )
-        self.assertIsInstance(result.structured_output, NestedResponse)
-        self.assertIsInstance(result.structured_output.main_category, str)
-        self.assertIsInstance(result.structured_output.subcategories, list)
-        self.assertIsInstance(result.structured_output.details, SimpleResponse)
-
-    def test_async_nested_response(self):
-        asyncio.run(self.async_nested_response())
-
-
-class TestLMNestedStructuredOutputs(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        # Initialize the LM once for all tests
-        cls.lm = LM(
-            model_name="gpt-4o-mini",
-            formatting_model_name="gpt-4o-mini",
-            temperature=0.7,
-            max_retries="Few",
-            structured_output_mode="forced_json",
-        )
-
-    def test_sync_nested_person_response(self):
-        result = self.lm.respond_sync(
-            system_message="You are an HR assistant.",
-            user_message="Provide detailed information about a fictional employee named John Doe.",
-            response_model=NestedPersonResponse,
-        )
-        self.assertIsInstance(result.structured_output, NestedPersonResponse)
-        self.assertIsInstance(result.structured_output.personal, PersonalInfo)
-        self.assertIsInstance(result.structured_output.personal.address, Address)
-        self.assertIsInstance(result.structured_output.work, WorkInfo)
-        self.assertIsInstance(result.structured_output.skills, list)
-
-    def test_sync_nested_portfolio_response(self):
-        result = self.lm.respond_sync(
-            system_message="You are a portfolio manager.",
-            user_message="Create a portfolio for a fictional software developer with multiple projects.",
-            response_model=NestedPortfolioResponse,
-        )
-        self.assertIsInstance(result.structured_output, NestedPortfolioResponse)
-        self.assertIsInstance(result.structured_output.developer, PersonalInfo)
-        self.assertIsInstance(result.structured_output.developer.address, Address)
-        self.assertIsInstance(result.structured_output.projects, list)
-        for project in result.structured_output.projects:
-            self.assertIsInstance(project, ProjectDetails)
-        self.assertIsInstance(result.structured_output.total_experience, int)
-
-    async def async_nested_company_response(self):
-        result = await self.lm.respond_async(
-            system_message="You are a company information specialist.",
-            user_message="Provide detailed information about a fictional tech company.",
-            response_model=NestedCompanyResponse,
-        )
-        self.assertIsInstance(result.structured_output, NestedCompanyResponse)
-        self.assertIsInstance(result.structured_output.headquarters, Address)
-        self.assertIsInstance(result.structured_output.employees, list)
-        for employee in result.structured_output.employees:
-            self.assertIsInstance(employee, PersonalInfo)
-            self.assertIsInstance(employee.address, Address)
-        self.assertIsInstance(result.structured_output.main_products, list)
-
-    def test_async_nested_company_response(self):
-        asyncio.run(self.async_nested_company_response())
-
-
-if __name__ == "__main__":
-    unittest.main()