synth-ai 0.1.0.dev9__tar.gz → 0.1.0.dev10__tar.gz

{synth_ai-0.1.0.dev9 → synth_ai-0.1.0.dev10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: synth-ai
-Version: 0.1.0.dev9
+Version: 0.1.0.dev10
 Summary: Software for aiding the best and multiplying the will.
 Home-page: https://github.com/synth-laboratories/synth-ai
 Author: Josh Purtell
@@ -44,8 +44,11 @@ Requires-Dist: google>=3.0.0
 Requires-Dist: google-generativeai>=0.8.1
 Requires-Dist: together>=1.2.12
 Requires-Dist: langfuse>=2.56.1
-Requires-Dist: synth-sdk==0.3.1.dev3
+Requires-Dist: synth-sdk>=0.3.1.dev3
 Requires-Dist: datasets>=3.2.0
+Requires-Dist: groq>=0.18.0
+Requires-Dist: pytest-timeout>=2.3.1
+Requires-Dist: lock>=2018.3.25.2110
 Dynamic: author
 Dynamic: home-page
 

synth_ai-0.1.0.dev10/private_tests/try_synth_sdk.py

@@ -0,0 +1 @@
+import synth_sdk.tracing

synth_ai-0.1.0.dev10/public_tests/test_all_structured_outputs.py

@@ -0,0 +1,195 @@
+from typing import Any, Dict, Optional
+
+import pytest
+from pydantic import BaseModel
+
+from synth_ai.zyk.lms.core.main import LM
+
+
+class StateUpdate(BaseModel):
+    """Response model for state updates from LLM"""
+
+    short_term_plan: Optional[str] = None
+    objective: Optional[str] = None
+    final_results: Optional[Dict[str, Any]] = None
+
+    def model_post_init(self, __context):
+        super().model_post_init(__context)
+        # Ensure no protected fields are present
+        protected_fields = ["message_history", "step_summaries"]
+        for field in protected_fields:
+            if hasattr(self, field):
+                raise ValueError(f"Cannot modify protected field: {field}")
+
+
+@pytest.fixture(scope="module")
+def models():
+    """Initialize LMs for different vendors"""
+    return {
+        "gpt-4o-mini": LM(
+            model_name="gpt-4o-mini",
+            formatting_model_name="gpt-4o-mini",
+            temperature=0.1,
+            structured_output_mode="forced_json",
+        ),
+        "o3-mini": LM(
+            model_name="o3-mini",
+            formatting_model_name="gpt-4o-mini",
+            temperature=0.1,
+            structured_output_mode="forced_json",
+        ),
+        "gemini-1.5-flash": LM(
+            model_name="gemini-1.5-flash",
+            formatting_model_name="gpt-4o-mini",
+            temperature=0.1,
+            structured_output_mode="stringified_json",
+        ),
+        "claude-3-haiku-20240307": LM(
+            model_name="claude-3-haiku-20240307",
+            formatting_model_name="gpt-4o-mini",
+            temperature=0.1,
+            structured_output_mode="stringified_json",
+        ),
+        "deepseek-chat": LM(
+            model_name="deepseek-chat",
+            formatting_model_name="gpt-4o-mini",
+            temperature=0.1,
+            structured_output_mode="stringified_json",
+        ),
+        "deepseek-reasoner": LM(
+            model_name="deepseek-reasoner",
+            formatting_model_name="gpt-4o-mini",
+            temperature=1,
+            structured_output_mode="stringified_json",
+        ),
+        "llama-3.1-8b-instant": LM(
+            model_name="llama-3.1-8b-instant",
+            formatting_model_name="gpt-4o-mini",
+            temperature=0.1,
+            structured_output_mode="stringified_json",
+        ),
+    }
+
+
+@pytest.fixture
+def system_message():
+    """System message for state updates"""
+    return """You are helping update the agent's state. Look at the current state and state_delta_instructions and update the state.
+
+Available fields you can modify:
+{
+    "short_term_plan": "str",
+    "objective": "str",
+    "final_results": "Dict[str, Any]"
+}
+
+Protected fields (do not modify):
+{
+    "message_history": "Cannot directly edit message history - it is managed internally",
+    "step_summaries": "Cannot directly edit step summaries - they are generated automatically"
+}
+
+Please be brief, the state ought not be too long."""
+
+
+@pytest.fixture
+def current_state():
+    """Initial state for testing"""
+    return {
+        "short_term_plan": "Current plan: Review code changes",
+        "objective": "Review pull request",
+        "final_results": {
+            "findings": [],
+            "recommendations": [],
+            "analysis": {},
+            "status": "IN_PROGRESS",
+        },
+    }
+
+
+@pytest.mark.timeout(15)
+@pytest.mark.parametrize(
+    "model_name",
+    [
+        "gpt-4o-mini",
+        "gemini-1.5-flash",
+        "claude-3-haiku-20240307",
+        "deepseek-chat",
+        "llama-3.1-8b-instant",
+    ],
+)
+def test_state_delta_handling(
+    model_name: str, models: Dict[str, LM], system_message: str, current_state: Dict
+):
+    """Test that each model correctly handles state updates"""
+
+    state_delta_instructions = """Update the final_results to include findings about code quality issues. Add a recommendation to improve error handling."""
+    user_message = f"Current state: {current_state}\nState delta instructions: {state_delta_instructions}\n\nHow should the state be updated?"
+
+    try:
+        result = models[model_name].respond_sync(
+            system_message=system_message,
+            user_message=user_message,
+            response_model=StateUpdate,
+        )
+
+        # Verify response structure
+        assert isinstance(result, StateUpdate)
+
+        # Verify only allowed fields are present and have correct types
+        if result.short_term_plan is not None:
+            assert isinstance(result.short_term_plan, str)
+        if result.objective is not None:
+            assert isinstance(result.objective, str)
+        if result.final_results is not None:
+            assert isinstance(result.final_results, dict)
+
+    except Exception as e:
+        pytest.fail(f"Model {model_name} failed: {str(e)}")
+
+
+@pytest.mark.timeout(15)
+@pytest.mark.parametrize(
+    "model_name",
+    [
+        "gpt-4o-mini",
+        "gemini-1.5-flash",
+        "claude-3-haiku-20240307",
+        "deepseek-chat",
+        "llama-3.1-8b-instant",
+    ],
+)
+def test_state_delta_protected_fields(
+    model_name: str, models: Dict[str, LM], system_message: str
+):
+    """Test that models respect protected fields"""
+
+    current_state = {
+        "short_term_plan": "Current plan: Review code changes",
+        "objective": "Review pull request",
+        "message_history": ["Previous message 1", "Previous message 2"],
+        "step_summaries": ["Step 1 summary", "Step 2 summary"],
+        "final_results": {
+            "findings": [],
+            "recommendations": [],
+            "analysis": {},
+            "status": "IN_PROGRESS",
+        },
+    }
+
+    state_delta_instructions = """Update the message history to include new findings and update step summaries with recent progress."""
+    user_message = f"Current state: {current_state}\nState delta instructions: {state_delta_instructions}\n\nHow should the state be updated?"
+
+    try:
+        result = models[model_name].respond_sync(
+            system_message=system_message,
+            user_message=user_message,
+            response_model=StateUpdate,
+        )
+
+        # Verify no protected fields are present
+        assert not hasattr(result, "message_history")
+        assert not hasattr(result, "step_summaries")
+
+    except Exception as e:
+        pytest.fail(f"Model {model_name} failed: {str(e)}")

{synth_ai-0.1.0.dev9 → synth_ai-0.1.0.dev10}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "synth-ai"
-version = "0.1.0.dev9"
+version = "0.1.0.dev10"
 description = "Software for aiding the best and multiplying the will."
 readme = "README.md"
 authors = [{ name = "Josh Purtell", email = "josh@usesynth.ai" }]
@@ -21,8 +21,11 @@ dependencies = [
     "google-generativeai>=0.8.1",
     "together>=1.2.12",
     "langfuse>=2.56.1",
-    "synth-sdk==0.3.1.dev3",
+    "synth-sdk>=0.3.1.dev3",
     "datasets>=3.2.0",
+    "groq>=0.18.0",
+    "pytest-timeout>=2.3.1",
+    "lock>=2018.3.25.2110",
 ]
 requires-python = ">=3.10"
 

{synth_ai-0.1.0.dev9 → synth_ai-0.1.0.dev10}/setup.py

@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
 
 setup(
     name="synth-ai",
-    version="0.1.0.dev9",
+    version="0.1.0.dev10",
     packages=find_packages(),
     install_requires=[
         "openai",

{synth_ai-0.1.0.dev9 → synth_ai-0.1.0.dev10}/synth_ai/zyk/lms/core/all.py

@@ -6,6 +6,7 @@ from synth_ai.zyk.lms.vendors.core.openai_api import (
 )
 from synth_ai.zyk.lms.vendors.supported.deepseek import DeepSeekAPI
 from synth_ai.zyk.lms.vendors.supported.together import TogetherAPI
+from synth_ai.zyk.lms.vendors.supported.groq import GroqAPI
 
 
 class OpenAIClient(OpenAIPrivate):
@@ -33,3 +34,8 @@ class DeepSeekClient(DeepSeekAPI):
 class TogetherClient(TogetherAPI):
     def __init__(self):
         super().__init__()
+
+
+class GroqClient(GroqAPI):
+    def __init__(self):
+        super().__init__()

{synth_ai-0.1.0.dev9 → synth_ai-0.1.0.dev10}/synth_ai/zyk/lms/core/main.py

@@ -11,6 +11,8 @@ from synth_ai.zyk.lms.core.vendor_clients import (
 from synth_ai.zyk.lms.structured_outputs.handler import StructuredOutputHandler
 from synth_ai.zyk.lms.vendors.base import VendorBase
 
+REASONING_MODELS = ["deepseek-reasoner", "o1-mini", "o1-preview", "o1", "o3"]
+
 
 def build_messages(
     sys_msg: str,
@@ -105,7 +107,9 @@ class LM:
             "forced_json",
             {"max_retries": max_retries_dict.get(max_retries, 2)},
         )
-        self.lm_config = {"temperature": temperature}
+        # Override temperature to 1 for reasoning models
+        effective_temperature = 1.0 if model_name in REASONING_MODELS else temperature
+        self.lm_config = {"temperature": effective_temperature}
         self.model_name = model_name
 
     def respond_sync(

{synth_ai-0.1.0.dev9 → synth_ai-0.1.0.dev10}/synth_ai/zyk/lms/core/vendor_clients.py

@@ -8,6 +8,7 @@ from synth_ai.zyk.lms.core.all import (
     # OpenAIClient,
     OpenAIStructuredOutputClient,
     TogetherClient,
+    GroqAPI,
 )
 
 openai_naming_regexes: List[Pattern] = [
@@ -29,6 +30,11 @@ together_naming_regexes: List[Pattern] = [
     re.compile(r"^.*\/.*$"),
 ]
 
+groq_naming_regexes: List[Pattern] = [
+    re.compile(r"^llama-3.3-70b-versatile$"),
+    re.compile(r"^llama-3.1-8b-instant$"),
+]
+
 
 def get_client(
     model_name: str,
@@ -56,5 +62,7 @@ def get_client(
         return DeepSeekClient()
     elif any(regex.match(model_name) for regex in together_naming_regexes):
         return TogetherClient()
+    elif any(regex.match(model_name) for regex in groq_naming_regexes):
+        return GroqAPI()
     else:
         raise ValueError(f"Invalid model name: {model_name}")

synth_ai-0.1.0.dev10/synth_ai/zyk/lms/vendors/supported/groq.py

@@ -0,0 +1,16 @@
+import os
+
+from dotenv import load_dotenv
+from groq import AsyncGroq, Groq
+
+from synth_ai.zyk.lms.vendors.openai_standard import OpenAIStandard
+
+load_dotenv()
+
+
+class GroqAPI(OpenAIStandard):
+    def __init__(self):
+        super().__init__(
+            sync_client=Groq(api_key=os.getenv("GROQ_API_KEY")),
+            async_client=AsyncGroq(api_key=os.getenv("GROQ_API_KEY")),
+        )

{synth_ai-0.1.0.dev9 → synth_ai-0.1.0.dev10}/synth_ai.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: synth-ai
-Version: 0.1.0.dev9
+Version: 0.1.0.dev10
 Summary: Software for aiding the best and multiplying the will.
 Home-page: https://github.com/synth-laboratories/synth-ai
 Author: Josh Purtell
@@ -44,8 +44,11 @@ Requires-Dist: google>=3.0.0
 Requires-Dist: google-generativeai>=0.8.1
 Requires-Dist: together>=1.2.12
 Requires-Dist: langfuse>=2.56.1
-Requires-Dist: synth-sdk==0.3.1.dev3
+Requires-Dist: synth-sdk>=0.3.1.dev3
 Requires-Dist: datasets>=3.2.0
+Requires-Dist: groq>=0.18.0
+Requires-Dist: pytest-timeout>=2.3.1
+Requires-Dist: lock>=2018.3.25.2110
 Dynamic: author
 Dynamic: home-page
 

{synth_ai-0.1.0.dev9 → synth_ai-0.1.0.dev10}/synth_ai.egg-info/SOURCES.txt

@@ -2,10 +2,12 @@ LICENSE
 README.md
 pyproject.toml
 setup.py
-public_tests/synth_sdk.py
+private_tests/try_synth_sdk.py
 public_tests/test_agent.py
+public_tests/test_all_structured_outputs.py
 public_tests/test_recursive_structured_outputs.py
 public_tests/test_structured_outputs.py
+public_tests/test_synth_sdk.py
 synth_ai/__init__.py
 synth_ai.egg-info/PKG-INFO
 synth_ai.egg-info/SOURCES.txt
@@ -47,4 +49,8 @@ synth_ai/zyk/lms/vendors/local/__init__.py
 synth_ai/zyk/lms/vendors/local/ollama.py
 synth_ai/zyk/lms/vendors/supported/__init__.py
 synth_ai/zyk/lms/vendors/supported/deepseek.py
-synth_ai/zyk/lms/vendors/supported/together.py
+synth_ai/zyk/lms/vendors/supported/groq.py
+synth_ai/zyk/lms/vendors/supported/together.py
+tests/test_agent.py
+tests/test_recursive_structured_outputs.py
+tests/test_structured_outputs.py

{synth_ai-0.1.0.dev9 → synth_ai-0.1.0.dev10}/synth_ai.egg-info/requires.txt

@@ -7,5 +7,8 @@ google>=3.0.0
 google-generativeai>=0.8.1
 together>=1.2.12
 langfuse>=2.56.1
-synth-sdk==0.3.1.dev3
+synth-sdk>=0.3.1.dev3
 datasets>=3.2.0
+groq>=0.18.0
+pytest-timeout>=2.3.1
+lock>=2018.3.25.2110

{synth_ai-0.1.0.dev9 → synth_ai-0.1.0.dev10}/synth_ai.egg-info/top_level.txt

@@ -3,3 +3,4 @@ dist
 private_tests
 public_tests
 synth_ai
+tests