synth-ai 0.1.0.dev9__py3-none-any.whl → 0.1.0.dev10__py3-none-any.whl

private_tests/try_synth_sdk.py

@@ -0,0 +1 @@
+import synth_sdk.tracing

public_tests/test_all_structured_outputs.py

@@ -0,0 +1,195 @@
+from typing import Any, Dict, Optional
+
+import pytest
+from pydantic import BaseModel
+
+from synth_ai.zyk.lms.core.main import LM
+
+
+class StateUpdate(BaseModel):
+    """Response model for state updates from LLM"""
+
+    short_term_plan: Optional[str] = None
+    objective: Optional[str] = None
+    final_results: Optional[Dict[str, Any]] = None
+
+    def model_post_init(self, __context):
+        super().model_post_init(__context)
+        # Ensure no protected fields are present
+        protected_fields = ["message_history", "step_summaries"]
+        for field in protected_fields:
+            if hasattr(self, field):
+                raise ValueError(f"Cannot modify protected field: {field}")
+
+
+@pytest.fixture(scope="module")
+def models():
+    """Initialize LMs for different vendors"""
+    return {
+        "gpt-4o-mini": LM(
+            model_name="gpt-4o-mini",
+            formatting_model_name="gpt-4o-mini",
+            temperature=0.1,
+            structured_output_mode="forced_json",
+        ),
+        "o3-mini": LM(
+            model_name="o3-mini",
+            formatting_model_name="gpt-4o-mini",
+            temperature=0.1,
+            structured_output_mode="forced_json",
+        ),
+        "gemini-1.5-flash": LM(
+            model_name="gemini-1.5-flash",
+            formatting_model_name="gpt-4o-mini",
+            temperature=0.1,
+            structured_output_mode="stringified_json",
+        ),
+        "claude-3-haiku-20240307": LM(
+            model_name="claude-3-haiku-20240307",
+            formatting_model_name="gpt-4o-mini",
+            temperature=0.1,
+            structured_output_mode="stringified_json",
+        ),
+        "deepseek-chat": LM(
+            model_name="deepseek-chat",
+            formatting_model_name="gpt-4o-mini",
+            temperature=0.1,
+            structured_output_mode="stringified_json",
+        ),
+        "deepseek-reasoner": LM(
+            model_name="deepseek-reasoner",
+            formatting_model_name="gpt-4o-mini",
+            temperature=1,
+            structured_output_mode="stringified_json",
+        ),
+        "llama-3.1-8b-instant": LM(
+            model_name="llama-3.1-8b-instant",
+            formatting_model_name="gpt-4o-mini",
+            temperature=0.1,
+            structured_output_mode="stringified_json",
+        ),
+    }
+
+
+@pytest.fixture
+def system_message():
+    """System message for state updates"""
+    return """You are helping update the agent's state. Look at the current state and state_delta_instructions and update the state.
+
+Available fields you can modify:
+{
+    "short_term_plan": "str",
+    "objective": "str",
+    "final_results": "Dict[str, Any]"
+}
+
+Protected fields (do not modify):
+{
+    "message_history": "Cannot directly edit message history - it is managed internally",
+    "step_summaries": "Cannot directly edit step summaries - they are generated automatically"
+}
+
+Please be brief, the state ought not be too long."""
+
+
+@pytest.fixture
+def current_state():
+    """Initial state for testing"""
+    return {
+        "short_term_plan": "Current plan: Review code changes",
+        "objective": "Review pull request",
+        "final_results": {
+            "findings": [],
+            "recommendations": [],
+            "analysis": {},
+            "status": "IN_PROGRESS",
+        },
+    }
+
+
+@pytest.mark.timeout(15)
+@pytest.mark.parametrize(
+    "model_name",
+    [
+        "gpt-4o-mini",
+        "gemini-1.5-flash",
+        "claude-3-haiku-20240307",
+        "deepseek-chat",
+        "llama-3.1-8b-instant",
+    ],
+)
+def test_state_delta_handling(
+    model_name: str, models: Dict[str, LM], system_message: str, current_state: Dict
+):
+    """Test that each model correctly handles state updates"""
+
+    state_delta_instructions = """Update the final_results to include findings about code quality issues. Add a recommendation to improve error handling."""
+    user_message = f"Current state: {current_state}\nState delta instructions: {state_delta_instructions}\n\nHow should the state be updated?"
+
+    try:
+        result = models[model_name].respond_sync(
+            system_message=system_message,
+            user_message=user_message,
+            response_model=StateUpdate,
+        )
+
+        # Verify response structure
+        assert isinstance(result, StateUpdate)
+
+        # Verify only allowed fields are present and have correct types
+        if result.short_term_plan is not None:
+            assert isinstance(result.short_term_plan, str)
+        if result.objective is not None:
+            assert isinstance(result.objective, str)
+        if result.final_results is not None:
+            assert isinstance(result.final_results, dict)
+
+    except Exception as e:
+        pytest.fail(f"Model {model_name} failed: {str(e)}")
+
+
+@pytest.mark.timeout(15)
+@pytest.mark.parametrize(
+    "model_name",
+    [
+        "gpt-4o-mini",
+        "gemini-1.5-flash",
+        "claude-3-haiku-20240307",
+        "deepseek-chat",
+        "llama-3.1-8b-instant",
+    ],
+)
+def test_state_delta_protected_fields(
+    model_name: str, models: Dict[str, LM], system_message: str
+):
+    """Test that models respect protected fields"""
+
+    current_state = {
+        "short_term_plan": "Current plan: Review code changes",
+        "objective": "Review pull request",
+        "message_history": ["Previous message 1", "Previous message 2"],
+        "step_summaries": ["Step 1 summary", "Step 2 summary"],
+        "final_results": {
+            "findings": [],
+            "recommendations": [],
+            "analysis": {},
+            "status": "IN_PROGRESS",
+        },
+    }
+
+    state_delta_instructions = """Update the message history to include new findings and update step summaries with recent progress."""
+    user_message = f"Current state: {current_state}\nState delta instructions: {state_delta_instructions}\n\nHow should the state be updated?"
+
+    try:
+        result = models[model_name].respond_sync(
+            system_message=system_message,
+            user_message=user_message,
+            response_model=StateUpdate,
+        )
+
+        # Verify no protected fields are present
+        assert not hasattr(result, "message_history")
+        assert not hasattr(result, "step_summaries")
+
+    except Exception as e:
+        pytest.fail(f"Model {model_name} failed: {str(e)}")

synth_ai/zyk/lms/core/all.py

@@ -6,6 +6,7 @@ from synth_ai.zyk.lms.vendors.core.openai_api import (
 )
 from synth_ai.zyk.lms.vendors.supported.deepseek import DeepSeekAPI
 from synth_ai.zyk.lms.vendors.supported.together import TogetherAPI
+from synth_ai.zyk.lms.vendors.supported.groq import GroqAPI
 
 
 class OpenAIClient(OpenAIPrivate):
@@ -33,3 +34,8 @@ class DeepSeekClient(DeepSeekAPI):
 class TogetherClient(TogetherAPI):
     def __init__(self):
         super().__init__()
+
+
+class GroqClient(GroqAPI):
+    def __init__(self):
+        super().__init__()

synth_ai/zyk/lms/core/main.py

@@ -11,6 +11,8 @@ from synth_ai.zyk.lms.core.vendor_clients import (
 from synth_ai.zyk.lms.structured_outputs.handler import StructuredOutputHandler
 from synth_ai.zyk.lms.vendors.base import VendorBase
 
+REASONING_MODELS = ["deepseek-reasoner", "o1-mini", "o1-preview", "o1", "o3"]
+
 
 def build_messages(
     sys_msg: str,
@@ -105,7 +107,9 @@ class LM:
             "forced_json",
             {"max_retries": max_retries_dict.get(max_retries, 2)},
         )
-        self.lm_config = {"temperature": temperature}
+        # Override temperature to 1 for reasoning models
+        effective_temperature = 1.0 if model_name in REASONING_MODELS else temperature
+        self.lm_config = {"temperature": effective_temperature}
         self.model_name = model_name
 
     def respond_sync(

synth_ai/zyk/lms/core/vendor_clients.py

@@ -8,6 +8,7 @@ from synth_ai.zyk.lms.core.all import (
     # OpenAIClient,
     OpenAIStructuredOutputClient,
     TogetherClient,
+    GroqAPI,
 )
 
 openai_naming_regexes: List[Pattern] = [
@@ -29,6 +30,11 @@ together_naming_regexes: List[Pattern] = [
     re.compile(r"^.*\/.*$"),
 ]
 
+groq_naming_regexes: List[Pattern] = [
+    re.compile(r"^llama-3.3-70b-versatile$"),
+    re.compile(r"^llama-3.1-8b-instant$"),
+]
+
 
 def get_client(
     model_name: str,
@@ -56,5 +62,7 @@ def get_client(
         return DeepSeekClient()
     elif any(regex.match(model_name) for regex in together_naming_regexes):
         return TogetherClient()
+    elif any(regex.match(model_name) for regex in groq_naming_regexes):
+        return GroqAPI()
     else:
         raise ValueError(f"Invalid model name: {model_name}")

synth_ai/zyk/lms/vendors/supported/groq.py

@@ -0,0 +1,16 @@
+import os
+
+from dotenv import load_dotenv
+from groq import AsyncGroq, Groq
+
+from synth_ai.zyk.lms.vendors.openai_standard import OpenAIStandard
+
+load_dotenv()
+
+
+class GroqAPI(OpenAIStandard):
+    def __init__(self):
+        super().__init__(
+            sync_client=Groq(api_key=os.getenv("GROQ_API_KEY")),
+            async_client=AsyncGroq(api_key=os.getenv("GROQ_API_KEY")),
+        )

{synth_ai-0.1.0.dev9.dist-info → synth_ai-0.1.0.dev10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: synth-ai
-Version: 0.1.0.dev9
+Version: 0.1.0.dev10
 Summary: Software for aiding the best and multiplying the will.
 Home-page: https://github.com/synth-laboratories/synth-ai
 Author: Josh Purtell
@@ -44,8 +44,11 @@ Requires-Dist: google>=3.0.0
 Requires-Dist: google-generativeai>=0.8.1
 Requires-Dist: together>=1.2.12
 Requires-Dist: langfuse>=2.56.1
-Requires-Dist: synth-sdk==0.3.1.dev3
+Requires-Dist: synth-sdk>=0.3.1.dev3
 Requires-Dist: datasets>=3.2.0
+Requires-Dist: groq>=0.18.0
+Requires-Dist: pytest-timeout>=2.3.1
+Requires-Dist: lock>=2018.3.25.2110
 Dynamic: author
 Dynamic: home-page
 

{synth_ai-0.1.0.dev9.dist-info → synth_ai-0.1.0.dev10.dist-info}/RECORD

@@ -1,7 +1,9 @@
-public_tests/synth_sdk.py,sha256=fqkzyzLb_NW4k8EiP2mJ5HZk3lDTi1juyTf9Gv_9wfc,14238
+private_tests/try_synth_sdk.py,sha256=vk4lUEfpQfLACFl6Qw468t_lsuYxuoIIr05WRgWKGKY,24
 public_tests/test_agent.py,sha256=CjPPWuMWC_TzX1DkDald-bbAxgjXE-HPQvFhq2B--5k,22363
+public_tests/test_all_structured_outputs.py,sha256=SkvQq4paFVh2b3XfeZJ0ihd4LLKgVvVk2Yd4bIXZEEw,6307
 public_tests/test_recursive_structured_outputs.py,sha256=Ne-9XwnOxN7eSpGbNHOpegR-sRj589I84T6y8Z_4QnA,5781
 public_tests/test_structured_outputs.py,sha256=J7sfbGZ7OeB5ONIKpcCTymyayNyAdFfGokC1bcUrSx0,3651
+public_tests/test_synth_sdk.py,sha256=fqkzyzLb_NW4k8EiP2mJ5HZk3lDTi1juyTf9Gv_9wfc,14238
 synth_ai/__init__.py,sha256=2siivzLbT2r-EA7m91dcJB-6Vsurc5_sX3WiKf4_o8Y,198
 synth_ai/zyk/__init__.py,sha256=zoPor1PI2OrgpCu-MBLZXcX1jAbSgD9q0kqZpTghTcQ,60
 synth_ai/zyk/lms/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -14,10 +16,10 @@ synth_ai/zyk/lms/caching/handler.py,sha256=sewq5rRfqXHzCEiXvdckbuxYp9ze_EjVSndnU
 synth_ai/zyk/lms/caching/initialize.py,sha256=zZls6RKAax6Z-8oJInGaSg_RPN_fEZ6e_RCX64lMLJw,416
 synth_ai/zyk/lms/caching/persistent.py,sha256=mQmP1z0rWVYjxwso5zIwd51Df2dWZvdHonuqsOY6SFI,2075
 synth_ai/zyk/lms/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synth_ai/zyk/lms/core/all.py,sha256=oUplKT1AsTv9Uvx2SqVjGjOsa0ml5G_GJFdKp6rPiGs,894
+synth_ai/zyk/lms/core/all.py,sha256=kuuu0V1_DVJp8GtZztdwFiTwGVYIdJ946Y1V0D4zz8Y,1034
 synth_ai/zyk/lms/core/exceptions.py,sha256=K0BVdAzxVIchsvYZAaHEH1GAWBZvpxhFi-SPcJOjyPQ,205
-synth_ai/zyk/lms/core/main.py,sha256=LMPsr8fF93kRk5sts5Q9-acKyLsX4tmJEoifj7DQHvo,8786
-synth_ai/zyk/lms/core/vendor_clients.py,sha256=O2KWCFJ0XHWhd4-_UE-T0WQKKgz_SNT7X4nV9j1YEPM,1878
+synth_ai/zyk/lms/core/main.py,sha256=fdAPBjnyOfLBPtcD0D79tn0f7SrKv49zCP6IhiAeH9Y,9019
+synth_ai/zyk/lms/core/vendor_clients.py,sha256=tuYf9Jio9mnCajPc1dCwurdprknRGnbHv7wwHC5tIdY,2119
 synth_ai/zyk/lms/cost/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synth_ai/zyk/lms/cost/monitor.py,sha256=cSKIvw6WdPZIRubADWxQoh1MdB40T8-jjgfNUeUHIn0,5
 synth_ai/zyk/lms/cost/statefulness.py,sha256=TOsuXL8IjtKOYJ2aJQF8TwJVqn_wQ7AIwJJmdhMye7U,36
@@ -38,9 +40,13 @@ synth_ai/zyk/lms/vendors/local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5
 synth_ai/zyk/lms/vendors/local/ollama.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synth_ai/zyk/lms/vendors/supported/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synth_ai/zyk/lms/vendors/supported/deepseek.py,sha256=diFfdhPMO5bLFZxnYj7VT0v6jKTlOYESBkspUuVa2eY,529
+synth_ai/zyk/lms/vendors/supported/groq.py,sha256=Fbi7QvhdLx0F-VHO5PY-uIQlPR0bo3C9h1MvIOx8nz0,388
 synth_ai/zyk/lms/vendors/supported/together.py,sha256=Ni_jBqqGPN0PkkY-Ew64s3gNKk51k3FCpLSwlNhKbf0,342
-synth_ai-0.1.0.dev9.dist-info/LICENSE,sha256=ynhjRQUfqA_RdGRATApfFA_fBAy9cno04sLtLUqxVFM,1069
-synth_ai-0.1.0.dev9.dist-info/METADATA,sha256=aMcnA1M3zSQ1ZE8RzMgtmIdPSzRxor8gDhtwVzU9M4Q,2610
-synth_ai-0.1.0.dev9.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-synth_ai-0.1.0.dev9.dist-info/top_level.txt,sha256=MKoWqlbnW0ZKcm_eLzeCpgAihgL59ZrZZ8Q1HnZwHdg,22
-synth_ai-0.1.0.dev9.dist-info/RECORD,,
+tests/test_agent.py,sha256=CjPPWuMWC_TzX1DkDald-bbAxgjXE-HPQvFhq2B--5k,22363
+tests/test_recursive_structured_outputs.py,sha256=Ne-9XwnOxN7eSpGbNHOpegR-sRj589I84T6y8Z_4QnA,5781
+tests/test_structured_outputs.py,sha256=J7sfbGZ7OeB5ONIKpcCTymyayNyAdFfGokC1bcUrSx0,3651
+synth_ai-0.1.0.dev10.dist-info/LICENSE,sha256=ynhjRQUfqA_RdGRATApfFA_fBAy9cno04sLtLUqxVFM,1069
+synth_ai-0.1.0.dev10.dist-info/METADATA,sha256=0JR0iVQFHPMzRDsm8PxJMsAM76PYPHKONQRNZQ2LsHU,2712
+synth_ai-0.1.0.dev10.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+synth_ai-0.1.0.dev10.dist-info/top_level.txt,sha256=5GzJO9j-KbJ_4ppxhmCUa_qdhHM4-9cHHNU76yAI8do,42
+synth_ai-0.1.0.dev10.dist-info/RECORD,,

{synth_ai-0.1.0.dev9.dist-info → synth_ai-0.1.0.dev10.dist-info}/top_level.txt

@@ -1,2 +1,4 @@
+private_tests
 public_tests
 synth_ai
+tests

tests/test_agent.py

@@ -0,0 +1,538 @@
+import unittest
+from typing import List, Union
+
+from pydantic import BaseModel
+
+from synth_ai.zyk.lms.core.main import LM
+
+
+class UnitTestDict(BaseModel):
+    test_description: str
+    input_names: List[str]
+    input_types: List[str]
+    stringified_input_values: List[str]
+    assertion_condition: str
+    assertion_type: str  # Consider using Literal for specific assertion types
+
+
+class ActionArgument(BaseModel):
+    key: str
+    value: Union[str, int, float, bool, UnitTestDict]
+
+
+class ReAct(BaseModel):
+    reasoning: str
+    action_name: str
+    action_args: List[ActionArgument]  # Dict#[str, Dict]
+
+
+system = """
+<System Message> 
+# Premise
+You are a software engineer
+Here is some information about this setting
+<Setting Information>
+You are working to solve a computer science problem. You will need to submit a solution to the problem, which will be tested against a suite of hidden unit tests.
+</Setting Information>
+<Actions Available>
+<edit_submission>
+<action_context>
+Edit the submission code. Use this when you want to make changes to the current solution.
+</action_context>
+<action_arg_spec>
+{'first_line': <class 'int'>, 'last_line': <class 'int'>, 'new_code': <class 'str'>}
+</action_arg_spec>
+<action_description>
+Edit the submission code
+</action_description>
+
+</edit_submission>
+<add_submission>
+<action_context>
+Add the submission code. Use this when you want to start from scratch with a new solution.
+</action_context>
+<action_arg_spec>
+{'submission': <class 'str'>}
+</action_arg_spec>
+<action_description>
+Add the submission code
+</action_description>
+
+</add_submission>
+<add_unit_test>
+<action_context>
+Add a unit test. The unit test information you submit must be in the format of a BCBUnitTest: 
+
+class BCBUnitTest(BaseModel):
+    test_description: str
+    input_names: List[str]
+    input_types: List[str]
+    input_values: List[Any]
+    assertion_condition: str
+    assertion_type: Literal["assertTrue", "assertRaises"] = "assertTrue"
+
+
+ It will be parsed via BCBUnitTest(**unit_test_dict)
+
+
+
+# Some various notes:
+1. If an input should be of a type defined by a specific package, add the package name/alias to the type. E.g. "np.ndarray" or "pd.DataFrame". You still should fully define the value for the input_value field e.g. "pd.DataFrame({'a': [1, 2, 3]})"
+
+2. Unit tests will be compiled from the BCBUnitTest class as follows:
+    A. For AssertTrue type tests, the test will be compiled as follows:
+    ```python
+    def test_case(self):
+        # {{self.test_description}}
+
+        {{defs}}
+        result = {{function_name}}(**{{{{args}}}}})
+        self.{{self.assertion_type}}({{self.assertion_condition}})
+    ```
+    B. For AssertRaises type tests, the test will be compiled as follows:
+
+    ```python
+    def test_case(self):
+        # {{self.test_description}}
+        {{defs}}
+        with self.{{self.assertion_type}}({{self.assertion_condition}}):
+            {{function_name}}(**{{{{args}}}}})
+    ```
+
+    Provide information accordingly.
+
+</action_context>
+<action_arg_spec>
+{'unit_test_name': <class 'str'>, 'unit_test_dict': typing.Dict}
+</action_arg_spec>
+<action_description>
+Add a unit test
+</action_description>
+
+</add_unit_test>
+<remove_unit_test>
+<action_context>
+Remove a unit test
+</action_context>
+<action_arg_spec>
+{'unit_test_name': <class 'str'>}
+</action_arg_spec>
+<action_description>
+Remove a unit test
+</action_description>
+
+</remove_unit_test>
+<test_submission>
+<action_context>
+Test the submission
+</action_context>
+<action_arg_spec>
+{}
+</action_arg_spec>
+<action_description>
+Test the submission
+</action_description>
+
+</test_submission>
+<submit_solution>
+<action_context>
+Submit the solution
+</action_context>
+<action_arg_spec>
+{}
+</action_arg_spec>
+<action_description>
+Submit the solution
+</action_description>
+
+</submit_solution>
+
+</Actions Available>
+You'll be given your past actions/thoughts, along with recent raw observations from the environment
+The environment one step in the past is your current environment.
+
+# Objective
+Please complete the problem by drafting a solution, creating unit tests, improving the solution, and submitting the solution.
+
+# Constraints
+You will be given a code_prompt_for_answer, which contains imports and the function signature. Your solution must comprise code that can be appended to code_prompt_for_answer and run as a single script.
+ 
+"""
+
+user = """
+<User Message> 
+# Recent Actions / Thoughts
+
+# Recent Observations
+<1 environment step(s) in the past>{'action_result': None, 'environment_state': {'question': 'import pandas as pd\nimport numpy as np\n\n# Constants\nCOLUMNS = [\'column1\', \'column2\', \'column3\', \'column4\', \'column5\']\n\ndef task_func(df, dct):\n    '''\n    Replace certain values in a DataFrame with a dictionary mapping and calculate the Pearson correlation coefficient between each pair of columns.\n\n    Parameters:\n    df (DataFrame): The input DataFrame, containing numeric or categorical data.\n    dct (dict): A dictionary for replacing values in df, where keys are existing values and values are new values.\n\n    Returns:\n    DataFrame: A DataFrame with the correlation coefficients between each pair of columns. The format of the DataFrame is a square matrix with column and index labels matching the columns of the input DataFrame.\n    \n    Requirements:\n    - pandas\n    - numpy\n    \n    Note:\n    - This function operates on DataFrames containing numeric or categorical data that can be replaced with numeric values, as correlation calculations require numeric data.\n    - This function using pearson method to calculate the correlation matrix.\n    \n    Raises:\n    - This function will raise a ValueError is input df is not a DataFrame.\n        \n    Example:\n    >>> df = pd.DataFrame({\'A\': [1, 2, 3], \'B\': [4, 5, 6]})\n    >>> dct = {1: 10, 2: 20, 3: 30, 4: 40, 5: 50, 6: 60}\n    >>> correlation_matrix = task_func(df, dct)\n    >>> correlation_matrix.shape == (2, 2)\n    True\n    >>> np.allclose(correlation_matrix, np.array([[1.0, 1.0], [1.0, 1.0]]))\n    True\n    '''\n', 'code_prompt_for_answer': "import pandas as pd\nimport numpy as np\n# Constants\nCOLUMNS = ['column1', 'column2', 'column3', 'column4', 'column5']\ndef task_func(df, dct):\n", 'unit_tests_you_have_written': {}, 'current_solution': ''}}</1 environment step(s) in the past>
+
+Your next actions / thought:
+"""
+
+
+###
+
+hard_system = """
+# Premise
+You are a software engineer
+Here is some information about this setting
+<Setting Information>
+You are working to solve a computer science problem. You will need to submit a solution to the problem, which will be tested against a suite of hidden unit tests.
+</Setting Information>
+<Actions Available>
+<edit_submission>
+<action_context>
+Edit the submission code. Use this when you want to make changes to the current solution.
+</action_context>
+<action_arg_spec>
+{'first_line': <class 'int'>, 'last_line': <class 'int'>, 'new_code': <class 'str'>}
+</action_arg_spec>
+<action_description>
+Edit the submission code
+</action_description>
+
+</edit_submission>
+<add_submission>
+<action_context>
+Add the submission code. Use this when you want to start from scratch with a new solution.
+</action_context>
+<action_arg_spec>
+{'submission': <class 'str'>}
+</action_arg_spec>
+<action_description>
+Add the submission code
+</action_description>
+
+</add_submission>
+<add_unit_test>
+<action_context>
+Add a unit test. The unit test information you submit must be in the format of a BCBUnitTest: 
+
+class BCBUnitTest(BaseModel):
+    test_description: str
+    input_names: List[str]
+    input_types: List[str]
+    input_values: List[Any]
+    assertion_condition: str
+    assertion_type: Literal["assertTrue", "assertRaises"] = "assertTrue"
+
+
+ It will be parsed via BCBUnitTest(**unit_test_dict)
+
+
+
+# Some various notes:
+1. If an input should be of a type defined by a specific package, add the package name/alias to the type. E.g. "np.ndarray" or "pd.DataFrame". You still should fully define the value for the input_value field e.g. "pd.DataFrame({'a': [1, 2, 3]})"
+
+2. Unit tests will be compiled from the BCBUnitTest class as follows:
+    A. For AssertTrue type tests, the test will be compiled as follows:
+    ```python
+    def test_case(self):
+        # {{self.test_description}}
+
+        {{defs}}
+        result = {{function_name}}(**{{{{args}}}}})
+        self.{{self.assertion_type}}({{self.assertion_condition}})
+    ```
+    B. For AssertRaises type tests, the test will be compiled as follows:
+
+    ```python
+    def test_case(self):
+        # {{self.test_description}}
+        {{defs}}
+        with self.{{self.assertion_type}}({{self.assertion_condition}}):
+            {{function_name}}(**{{{{args}}}}})
+    ```
+
+    Provide information accordingly.
+
+</action_context>
+<action_arg_spec>
+{'unit_test_name': <class 'str'>, 'unit_test_dict': typing.Dict}
+</action_arg_spec>
+<action_description>
+Add a unit test
+</action_description>
+
+</add_unit_test>
+<remove_unit_test>
+<action_context>
+Remove a unit test
+</action_context>
+<action_arg_spec>
+{'unit_test_name': <class 'str'>}
+</action_arg_spec>
+<action_description>
+Remove a unit test
+</action_description>
+
+</remove_unit_test>
+<test_submission>
+<action_context>
+Test the submission
+</action_context>
+<action_arg_spec>
+{}
+</action_arg_spec>
+<action_description>
+Test the submission
+</action_description>
+
+</test_submission>
+<submit_solution>
+<action_context>
+Submit the solution
+</action_context>
+<action_arg_spec>
+{}
+</action_arg_spec>
+<action_description>
+Submit the solution
+</action_description>
+
+</submit_solution>
+
+</Actions Available>
+You'll be given your past actions/thoughts, along with recent raw observations from the environment
+The environment one step in the past is your current environment.
+
+# Objective
+Please complete the problem by drafting a solution, creating unit tests, improving the solution, and submitting the solution.
+
+# Constraints
+You will be given a code_prompt_for_answer, which contains imports and the function signature. Your solution must comprise code that can be appended to code_prompt_for_answer and run as a single script.
+ 
+
+<User Message> 
+# Recent Actions / Thoughts
+
+# Recent Observations
+<1 environment step(s) in the past>{'action_result': None, 'environment_state': {'question': 'import pandas as pd\nimport numpy as np\n\n# Constants\nCOLUMNS = [\'column1\', \'column2\', \'column3\', \'column4\', \'column5\']\n\ndef task_func(df, dct):\n   '''\n    Replace certain values in a DataFrame with a dictionary mapping and calculate the Pearson correlation coefficient between each pair of columns.\n\n    Parameters:\n    df (DataFrame): The input DataFrame, containing numeric or categorical data.\n    dct (dict): A dictionary for replacing values in df, where keys are existing values and values are new values.\n\n    Returns:\n    DataFrame: A DataFrame with the correlation coefficients between each pair of columns. The format of the DataFrame is a square matrix with column and index labels matching the columns of the input DataFrame.\n    \n    Requirements:\n    - pandas\n    - numpy\n    \n    Note:\n    - This function operates on DataFrames containing numeric or categorical data that can be replaced with numeric values, as correlation calculations require numeric data.\n    - This function using pearson method to calculate the correlation matrix.\n    \n    Raises:\n    - This function will raise a ValueError is input df is not a DataFrame.\n        \n    Example:\n    >>> df = pd.DataFrame({\'A\': [1, 2, 3], \'B\': [4, 5, 6]})\n    >>> dct = {1: 10, 2: 20, 3: 30, 4: 40, 5: 50, 6: 60}\n    >>> correlation_matrix = task_func(df, dct)\n    >>> correlation_matrix.shape == (2, 2)\n    True\n    >>> np.allclose(correlation_matrix, np.array([[1.0, 1.0], [1.0, 1.0]]))\n    True\n    '''\n', 'code_prompt_for_answer': "import pandas as pd\nimport numpy as np\n# Constants\nCOLUMNS = ['column1', 'column2', 'column3', 'column4', 'column5']\ndef task_func(df, dct):\n", 'unit_tests_you_have_written': {}, 'current_solution': ''}}</1 environment step(s) in the past>
+
+Your next actions / thought:  
+
+Structured output: reasoning="I need to implement the function 'task_func' that replaces values in a DataFrame based on a dictionary and calculates the Pearson correlation coefficient between the columns. I will also ensure to handle the case where the input is not a DataFrame by raising a ValueError." action_name='edit_submission' action_args=[ActionArgument(key='first_line', value=4), ActionArgument(key='last_line', value=4), ActionArgument(key='new_code', value="    if not isinstance(df, pd.DataFrame):\n        raise ValueError('Input must be a DataFrame')\n    df.replace(dct, inplace=True)\n    return df.corr(method='pearson')")]
+<System Message> 
+# Premise
+You are a software engineer
+Here is some information about this setting
+<Setting Information>
+You are working to solve a computer science problem. You will need to submit a solution to the problem, which will be tested against a suite of hidden unit tests.
+</Setting Information>
+<Actions Available>
+<edit_submission>
+<action_context>
+Edit the submission code. Use this when you want to make changes to the current solution.
+</action_context>
+<action_arg_spec>
+{'first_line': <class 'int'>, 'last_line': <class 'int'>, 'new_code': <class 'str'>}
+</action_arg_spec>
+<action_description>
+Edit the submission code
+</action_description>
+
+</edit_submission>
+<add_submission>
+<action_context>
+Add the submission code. Use this when you want to start from scratch with a new solution.
+</action_context>
+<action_arg_spec>
+{'submission': <class 'str'>}
+</action_arg_spec>
+<action_description>
+Add the submission code
+</action_description>
+
+</add_submission>
+<add_unit_test>
+<action_context>
+Add a unit test. The unit test information you submit must be in the format of a BCBUnitTest: 
+
+class BCBUnitTest(BaseModel):
+    test_description: str
+    input_names: List[str]
+    input_types: List[str]
+    input_values: List[Any]
+    assertion_condition: str
+    assertion_type: Literal["assertTrue", "assertRaises"] = "assertTrue"
+
+
+ It will be parsed via BCBUnitTest(**unit_test_dict)
+
+
+
+# Some various notes:
+1. If an input should be of a type defined by a specific package, add the package name/alias to the type. E.g. "np.ndarray" or "pd.DataFrame". You still should fully define the value for the input_value field e.g. "pd.DataFrame({'a': [1, 2, 3]})"
+
+2. Unit tests will be compiled from the BCBUnitTest class as follows:
+    A. For AssertTrue type tests, the test will be compiled as follows:
+    ```python
+    def test_case(self):
+        # {{self.test_description}}
+
+        {{defs}}
+        result = {{function_name}}(**{{{{args}}}}})
+        self.{{self.assertion_type}}({{self.assertion_condition}})
+    ```
+    B. For AssertRaises type tests, the test will be compiled as follows:
+
+    ```python
+    def test_case(self):
+        # {{self.test_description}}
+        {{defs}}
+        with self.{{self.assertion_type}}({{self.assertion_condition}}):
+            {{function_name}}(**{{{{args}}}}})
+    ```
+
+    Provide information accordingly.
+
+</action_context>
+<action_arg_spec>
+{'unit_test_name': <class 'str'>, 'unit_test_dict': typing.Dict}
+</action_arg_spec>
+<action_description>
+Add a unit test
+</action_description>
+
+</add_unit_test>
+<remove_unit_test>
+<action_context>
+Remove a unit test
+</action_context>
+<action_arg_spec>
+{'unit_test_name': <class 'str'>}
+</action_arg_spec>
+<action_description>
+Remove a unit test
+</action_description>
+
+</remove_unit_test>
+<test_submission>
+<action_context>
+Test the submission
+</action_context>
+<action_arg_spec>
+{}
+</action_arg_spec>
+<action_description>
+Test the submission
+</action_description>
+
+</test_submission>
+<submit_solution>
+<action_context>
+Submit the solution
+</action_context>
+<action_arg_spec>
+{}
+</action_arg_spec>
+<action_description>
+Submit the solution
+</action_description>
+
+</submit_solution>
+
+</Actions Available>
+You'll be given your past actions/thoughts, along with recent raw observations from the environment
+The environment one step in the past is your current environment.
+
+# Objective
+Please complete the problem by drafting a solution, creating unit tests, improving the solution, and submitting the solution.
+
+# Constraints
+You will be given a code_prompt_for_answer, which contains imports and the function signature. Your solution must comprise code that can be appended to code_prompt_for_answer and run as a single script.
+"""
+
+hard_user = """
+# Recent Actions / Thoughts
+<1 reasoning step(s) in the past>reasoning="I need to implement the function 'task_func' that replaces values in a DataFrame based on a dictionary and calculates the Pearson correlation coefficient between the columns. I will also ensure to handle the case where the input is not a DataFrame by raising a ValueError." action_name='edit_submission' action_args=[ActionArgument(key='first_line', value=4), ActionArgument(key='last_line', value=4), ActionArgument(key='new_code', value="    if not isinstance(df, pd.DataFrame):\n        raise ValueError('Input must be a DataFrame')\n    df.replace(dct, inplace=True)\n    return df.corr(method='pearson')")]</1 reasoning step(s) in the past>
+# Recent Observations
+<1 environment step(s) in the past>success=True result='Edited submission successfully'</1 environment step(s) in the past>
+<2 environment step(s) in the past>{'action_result': None, 'environment_state': {'question': 'import pandas as pd\nimport numpy as np\n\n# Constants\nCOLUMNS = [\'column1\', \'column2\', \'column3\', \'column4\', \'column5\']\n\ndef task_func(df, dct):\n    '''\n    Replace certain values in a DataFrame with a dictionary mapping and calculate the Pearson correlation coefficient between each pair of columns.\n\n    Parameters:\n    df (DataFrame): The input DataFrame, containing numeric or categorical data.\n    dct (dict): A dictionary for replacing values in df, where keys are existing values and values are new values.\n\n    Returns:\n    DataFrame: A DataFrame with the correlation coefficients between each pair of columns. The format of the DataFrame is a square matrix with column and index labels matching the columns of the input DataFrame.\n    \n    Requirements:\n    - pandas\n    - numpy\n    \n    Note:\n    - This function operates on DataFrames containing numeric or categorical data that can be replaced with numeric values, as correlation calculations require numeric data.\n    - This function using pearson method to calculate the correlation matrix.\n    \n    Raises:\n    - This function will raise a ValueError is input df is not a DataFrame.\n        \n    Example:\n    >>> df = pd.DataFrame({\'A\': [1, 2, 3], \'B\': [4, 5, 6]})\n    >>> dct = {1: 10, 2: 20, 3: 30, 4: 40, 5: 50, 6: 60}\n    >>> correlation_matrix = task_func(df, dct)\n    >>> correlation_matrix.shape == (2, 2)\n    True\n    >>> np.allclose(correlation_matrix, np.array([[1.0, 1.0], [1.0, 1.0]]))\n    True\n    '''\n', 'code_prompt_for_answer': "import pandas as pd\nimport numpy as np\n# Constants\nCOLUMNS = ['column1', 'column2', 'column3', 'column4', 'column5']\ndef task_func(df, dct):\n", 'unit_tests_you_have_written': {}, 'current_solution': ''}}</2 environment step(s) in the past>
+
+Your next actions / thought:
+"""
+
+
+class TestLMStructuredOutputs(unittest.TestCase):
+    # ... existing code ...
+
+    @classmethod
+    def setUpClass(cls):
+        # Initialize LMs for both forced_json and stringified_json modes
+        cls.lm_forced_json = LM(
+            model_name="gpt-4o-mini",
+            formatting_model_name="gpt-4o-mini",
+            temperature=0.0,
+            max_retries="Few",
+            structured_output_mode="forced_json",
+        )
+        cls.lm_stringified_json = LM(
+            model_name="gpt-4o-mini",
+            formatting_model_name="gpt-4o-mini",
+            temperature=0.0,
+            max_retries="Few",
+            structured_output_mode="stringified_json",
+        )
+
+    def test_sync_react_response_content(self):
+        system_message = system
+
+        user_message = user
+
+        for lm in [self.lm_forced_json, self.lm_stringified_json]:
+            with self.subTest(
+                mode=lm.structured_output_handler.handler.structured_output_mode
+            ):
+                result = lm.respond_sync(
+                    system_message=system_message,
+                    user_message=user_message,
+                    response_model=ReAct,
+                )
+                self.assertIsInstance(result, ReAct)
+                self.assertIsInstance(result.reasoning, str)
+                self.assertIsInstance(result.action_name, str)
+                self.assertIsInstance(result.action_args, list)
+                for arg in result.action_args:
+                    self.assertIsInstance(arg, ActionArgument)
+                    self.assertIsInstance(arg.key, str)
+                    # self.assertIsInstance(arg.value, str)
+
+    def test_sync_react_response_hard_content(self):
+        system_message = hard_system
+
+        user_message = hard_user
+
+        for lm in [self.lm_forced_json, self.lm_stringified_json]:
+            with self.subTest(
+                mode=lm.structured_output_handler.handler.structured_output_mode
+            ):
+                result = lm.respond_sync(
+                    system_message=system_message,
+                    user_message=user_message,
+                    response_model=ReAct,
+                )
+                self.assertIsInstance(result, ReAct)
+                self.assertIsInstance(result.reasoning, str)
+                self.assertIsInstance(result.action_name, str)
+                self.assertIsInstance(result.action_args, list)
+                for arg in result.action_args:
+                    self.assertIsInstance(arg, ActionArgument)
+                    self.assertIsInstance(arg.key, str)
+                    # self.assertIsInstance(arg.value, str)
+
+
+# use non-trivial fallback?
+
+if __name__ == "__main__":
+    # Create an instance of the test class
+    test_instance = TestLMStructuredOutputs()
+
+    # Set up the class (this would normally be done by unittest)
+    test_instance.setUpClass()
+
+    # Run the test methods
+    test_instance.test_sync_react_response_content()
+    test_instance.test_sync_react_response_hard_content()
+
+    print("All tests completed.")

tests/test_recursive_structured_outputs.py

@@ -0,0 +1,180 @@
+import asyncio
+import unittest
+from typing import List
+
+from pydantic import BaseModel, Field
+
+from synth_ai.zyk.lms.core.main import LM
+
+
+# Define example structured output models
+class SimpleResponse(BaseModel):
+    message: str
+    confidence: float
+
+
+class ComplexResponse(BaseModel):
+    title: str
+    tags: List[str]
+    content: str
+
+
+class NestedResponse(BaseModel):
+    main_category: str
+    subcategories: List[str]
+    details: SimpleResponse
+
+
+# Define nested structured output models
+class Address(BaseModel):
+    street: str
+    city: str
+    country: str
+
+
+class PersonalInfo(BaseModel):
+    name: str
+    age: int
+    address: Address
+
+
+class WorkInfo(BaseModel):
+    company: str
+    position: str
+    years_experience: int
+
+
+class NestedPersonResponse(BaseModel):
+    personal: PersonalInfo
+    work: WorkInfo
+    skills: List[str]
+
+
+class ProjectDetails(BaseModel):
+    name: str
+    description: str
+    technologies: List[str]
+
+
+class NestedPortfolioResponse(BaseModel):
+    developer: PersonalInfo
+    projects: List[ProjectDetails]
+    total_experience: int
+
+
+class NestedCompanyResponse(BaseModel):
+    name: str
+    founded: int
+    headquarters: Address
+    employees: List[PersonalInfo]
+    main_products: List[str]
+
+
+class TestLMStructuredOutputs(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        # Initialize the LM once for all tests
+        cls.lm = LM(
+            model_name="gpt-4o-mini",
+            formatting_model_name="gpt-4o-mini",
+            temperature=0.7,
+            max_retries="Few",
+            structured_output_mode="forced_json",
+        )
+
+    def test_sync_simple_response(self):
+        result = self.lm.respond_sync(
+            system_message="You are a helpful assistant.",
+            user_message="Give me a short greeting and your confidence level.",
+            response_model=SimpleResponse,
+        )
+        self.assertIsInstance(result, SimpleResponse)
+        self.assertIsInstance(result.message, str)
+        self.assertIsInstance(result.confidence, float)
+        self.assertGreaterEqual(result.confidence, 0)
+        self.assertLessEqual(result.confidence, 1)
+
+    def test_sync_complex_response(self):
+        result = self.lm.respond_sync(
+            system_message="You are a content creator.",
+            user_message="Create a short blog post about AI.",
+            response_model=ComplexResponse,
+        )
+        self.assertIsInstance(result, ComplexResponse)
+        self.assertIsInstance(result.title, str)
+        self.assertIsInstance(result.tags, list)
+        self.assertIsInstance(result.content, str)
+
+    async def async_nested_response(self):
+        result = await self.lm.respond_async(
+            system_message="You are a categorization expert.",
+            user_message="Categorize 'Python' and provide a brief description.",
+            response_model=NestedResponse,
+        )
+        self.assertIsInstance(result, NestedResponse)
+        self.assertIsInstance(result.main_category, str)
+        self.assertIsInstance(result.subcategories, list)
+        self.assertIsInstance(result.details, SimpleResponse)
+
+    def test_async_nested_response(self):
+        asyncio.run(self.async_nested_response())
+
+
+class TestLMNestedStructuredOutputs(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        # Initialize the LM once for all tests
+        cls.lm = LM(
+            model_name="gpt-4o-mini",
+            formatting_model_name="gpt-4o-mini",
+            temperature=0.7,
+            max_retries="Few",
+            structured_output_mode="forced_json",
+        )
+
+    def test_sync_nested_person_response(self):
+        result = self.lm.respond_sync(
+            system_message="You are an HR assistant.",
+            user_message="Provide detailed information about a fictional employee named John Doe.",
+            response_model=NestedPersonResponse,
+        )
+        self.assertIsInstance(result, NestedPersonResponse)
+        self.assertIsInstance(result.personal, PersonalInfo)
+        self.assertIsInstance(result.personal.address, Address)
+        self.assertIsInstance(result.work, WorkInfo)
+        self.assertIsInstance(result.skills, list)
+
+    def test_sync_nested_portfolio_response(self):
+        result = self.lm.respond_sync(
+            system_message="You are a portfolio manager.",
+            user_message="Create a portfolio for a fictional software developer with multiple projects.",
+            response_model=NestedPortfolioResponse,
+        )
+        self.assertIsInstance(result, NestedPortfolioResponse)
+        self.assertIsInstance(result.developer, PersonalInfo)
+        self.assertIsInstance(result.developer.address, Address)
+        self.assertIsInstance(result.projects, list)
+        for project in result.projects:
+            self.assertIsInstance(project, ProjectDetails)
+        self.assertIsInstance(result.total_experience, int)
+
+    async def async_nested_company_response(self):
+        result = await self.lm.respond_async(
+            system_message="You are a company information specialist.",
+            user_message="Provide detailed information about a fictional tech company.",
+            response_model=NestedCompanyResponse,
+        )
+        self.assertIsInstance(result, NestedCompanyResponse)
+        self.assertIsInstance(result.headquarters, Address)
+        self.assertIsInstance(result.employees, list)
+        for employee in result.employees:
+            self.assertIsInstance(employee, PersonalInfo)
+            self.assertIsInstance(employee.address, Address)
+        self.assertIsInstance(result.main_products, list)
+
+    def test_async_nested_company_response(self):
+        asyncio.run(self.async_nested_company_response())
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_structured_outputs.py

@@ -0,0 +1,100 @@
+import asyncio
+import unittest
+from typing import List
+
+from pydantic import BaseModel, Field
+
+from synth_ai.zyk.lms.core.main import LM
+
+
+# Define example structured output models
+class SimpleResponse(BaseModel):
+    message: str
+    confidence_between_zero_one: float = Field(
+        ..., description="Confidence level between 0 and 1"
+    )
+
+
+class ComplexResponse(BaseModel):
+    title: str
+    tags: List[str]
+    content: str
+
+
+class NestedResponse(BaseModel):
+    main_category: str
+    subcategories: List[str]
+    details: SimpleResponse
+
+
+class TestLMStructuredOutputs(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        # Initialize LMs for both forced_json and stringified_json modes
+        cls.lm_forced_json = LM(
+            model_name="gpt-4o-mini",
+            formatting_model_name="gpt-4o-mini",
+            temperature=0.7,
+            max_retries="Few",
+            structured_output_mode="forced_json",
+        )
+        cls.lm_stringified_json = LM(
+            model_name="gpt-4o-mini",
+            formatting_model_name="gpt-4o-mini",
+            temperature=0.7,
+            max_retries="Few",
+            structured_output_mode="stringified_json",
+        )
+
+    def test_sync_simple_response(self):
+        for lm in [self.lm_forced_json, self.lm_stringified_json]:
+            with self.subTest(
+                mode=lm.structured_output_handler.handler.structured_output_mode
+            ):
+                result = lm.respond_sync(
+                    system_message="You are a helpful assistant.",
+                    user_message="Give me a short greeting and your confidence level.",
+                    response_model=SimpleResponse,
+                )
+                self.assertIsInstance(result, SimpleResponse)
+                self.assertIsInstance(result.message, str)
+                self.assertIsInstance(result.confidence_between_zero_one, float)
+                self.assertGreaterEqual(result.confidence_between_zero_one, 0)
+                self.assertLessEqual(result.confidence_between_zero_one, 1)
+
+    def test_sync_complex_response(self):
+        for lm in [self.lm_forced_json, self.lm_stringified_json]:
+            with self.subTest(
+                mode=lm.structured_output_handler.handler.structured_output_mode
+            ):
+                result = lm.respond_sync(
+                    system_message="You are a content creator.",
+                    user_message="Create a short blog post about AI.",
+                    response_model=ComplexResponse,
+                )
+                self.assertIsInstance(result, ComplexResponse)
+                self.assertIsInstance(result.title, str)
+                self.assertIsInstance(result.tags, list)
+                self.assertIsInstance(result.content, str)
+
+    async def async_nested_response(self, lm):
+        result = await lm.respond_async(
+            system_message="You are a categorization expert.",
+            user_message="Categorize 'Python' and provide a brief description.",
+            response_model=NestedResponse,
+        )
+        self.assertIsInstance(result, NestedResponse)
+        self.assertIsInstance(result.main_category, str)
+        self.assertIsInstance(result.subcategories, list)
+        self.assertIsInstance(result.details, SimpleResponse)
+
+    def test_async_nested_response(self):
+        for lm in [self.lm_forced_json, self.lm_stringified_json]:  #
+            with self.subTest(
+                mode=lm.structured_output_handler.handler.structured_output_mode
+            ):
+                asyncio.run(self.async_nested_response(lm))
+
+
+if __name__ == "__main__":
+    unittest.main()