synth-ai 0.1.0.dev50__py3-none-any.whl → 0.1.0.dev51__py3-none-any.whl

synth_ai/zyk/lms/structured_outputs/inject.py

@@ -0,0 +1,314 @@
+import json
+from typing import (
+    Any,
+    Dict,
+    List,
+    Optional,
+    Tuple,
+    Type,
+    get_type_hints,
+    get_args,
+    get_origin,
+    Union,
+    Literal,
+)
+from pydantic import BaseModel
+import warnings
+
+
+def generate_type_map() -> Dict[Any, str]:
+    base_types = {
+        int: "int",
+        float: "float",
+        bool: "bool",
+        str: "str",
+        Any: "Any",
+    }
+
+    collection_types = {
+        List: "List",
+        Dict: "Dict",
+        Optional: "Optional",
+    }
+
+    type_map = {}
+    for base_type, name in base_types.items():
+        type_map[base_type] = name
+        for collection, collection_name in collection_types.items():
+            if collection is Optional:
+                type_map[Optional[base_type]] = name
+            elif collection is Dict:
+                # Handle generic Dict type
+                type_map[Dict] = "Dict[Any,Any]"
+                # Provide both key and value types for Dict
+                type_map[Dict[base_type, base_type]] = (
+                    f"{collection_name}[{name},{name}]"
+                )
+                # Handle Dict[Any, Any] explicitly
+                type_map[Dict[Any, Any]] = "Dict[Any,Any]"
+            else:
+                type_map[collection[base_type]] = f"{collection_name}[{name}]"
+    return type_map
+
+
+def generate_example_dict() -> Dict[str, Any]:
+    example_values = {
+        "str": "<Your type-str response here>",
+        "int": "<Your type-int response here>",
+        "float": "<Your type-float response here>",
+        "bool": "<Your type-bool response here>",
+        "Any": "<Your response here (infer the type from context)>",
+    }
+
+    example_dict = {}
+    for key, value in example_values.items():
+        example_dict[key] = value
+        example_dict[f"List[{key}]"] = [value]
+        example_dict[f"List[List[{key}]]"] = [[value]]
+        if key == "Dict[str,str]":
+            example_dict[f"Dict[str,{key.split('[')[1]}"] = {value: value}
+        elif key.startswith("Dict"):
+            example_dict[key] = {value: value}
+
+    # Add example for Dict[Any,Any]
+    example_dict["Dict[Any,Any]"] = {"<key>": "<value>"}
+
+    return example_dict
+
+
+base_type_examples = {
+    int: ("int", 42),
+    float: ("float", 3.14),
+    bool: ("bool", True),
+    str: ("str", "example"),
+    Any: ("Any", "<Any value>"),
+}
+
+
+def get_type_string(type_hint):
+    origin = get_origin(type_hint)
+    args = get_args(type_hint)
+
+    if origin is None:
+        if isinstance(type_hint, type) and issubclass(type_hint, BaseModel):
+            # For Pydantic models, create a dictionary of field types
+            field_types = {}
+            for field_name, field_info in type_hint.model_fields.items():
+                field_type = get_type_string(field_info.annotation)
+                # Check for Literal type by looking at the origin
+                if get_origin(field_info.annotation) is Literal:
+                    literal_args = get_args(field_info.annotation)
+                    field_type = f"Literal[{repr(literal_args[0])}]"
+                field_types[field_name] = field_type
+            return f"{type_hint.__name__}({', '.join(f'{k}: {v}' for k, v in field_types.items())})"
+        else:
+            return base_type_examples.get(type_hint, ("Unknown", "unknown"))[0]
+    elif origin in (list, List):
+        elem_type = get_type_string(args[0])
+        return f"List[{elem_type}]"
+    elif origin in (dict, Dict):
+        key_type = get_type_string(args[0])
+        value_type = get_type_string(args[1])
+        return f"Dict[{key_type}, {value_type}]"
+    elif origin is Union:
+        non_none_types = [t for t in args if t is not type(None)]
+        if len(non_none_types) == 1:
+            return f"Optional[{get_type_string(non_none_types[0])}]"
+        else:
+            # For unions of Pydantic models (like tool calls), show each variant
+            union_types = []
+            for t in non_none_types:
+                if isinstance(t, type) and issubclass(t, BaseModel):
+                    # Include discriminator field if present
+                    discriminator = None
+                    for field_name, field_info in t.model_fields.items():
+                        if get_origin(field_info.annotation) is Literal:
+                            literal_args = get_args(field_info.annotation)
+                            discriminator = f"{field_name}={repr(literal_args[0])}"
+                            break
+                    type_str = t.__name__
+                    if discriminator:
+                        type_str += f"({discriminator})"
+                    union_types.append(type_str)
+                else:
+                    union_types.append(get_type_string(t))
+            return f"Union[{', '.join(union_types)}]"
+    elif origin is Literal:
+        # Handle Literal type directly
+        return f"Literal[{repr(args[0])}]"
+    else:
+        return "Any"
+
+
+def get_example_value(type_hint):
+    origin = get_origin(type_hint)
+    args = get_args(type_hint)
+
+    if origin is None:
+        if isinstance(type_hint, type) and issubclass(type_hint, BaseModel):
+            example = {}
+            union_docs = []
+            for field_name, field_info in type_hint.model_fields.items():
+                field_value, field_docs = get_example_value(field_info.annotation)
+                if field_docs:
+                    union_docs.extend(field_docs)
+
+                # Handle literal fields by checking origin
+                if get_origin(field_info.annotation) is Literal:
+                    literal_args = get_args(field_info.annotation)
+                    field_value = literal_args[0]
+
+                # Include field description if available
+                if field_info.description:
+                    example[field_name] = {
+                        "value": field_value,
+                        "description": field_info.description,
+                    }
+                else:
+                    example[field_name] = field_value
+            return example, union_docs
+        else:
+            return base_type_examples.get(type_hint, ("Unknown", "unknown"))[1], []
+    elif origin in (list, List):
+        value, docs = get_example_value(args[0])
+        return [value], docs
+    elif origin in (dict, Dict):
+        if not args or len(args) < 2:
+            warnings.warn(
+                f"Dictionary type hint {type_hint} missing type arguments. "
+                "Defaulting to Dict[str, Any].",
+                UserWarning,
+            )
+            return {"example_key": "<Any value>"}, []  # Default for Dict[str, Any]
+        key_value, key_docs = get_example_value(args[0])
+        value_value, value_docs = get_example_value(args[1])
+        return {key_value: value_value}, key_docs + value_docs
+    elif origin is Union:
+        non_none_types = [t for t in args if t is not type(None)]
+        # For unions of tool calls, use the first one but preserve the discriminator
+        first_type = non_none_types[0]
+        union_docs = []
+
+        if all(
+            isinstance(t, type) and issubclass(t, BaseModel) for t in non_none_types
+        ):
+            # Generate examples for all union variants
+            for t in non_none_types:
+                example = {}
+                for field_name, field_info in t.model_fields.items():
+                    field_value, _ = get_example_value(field_info.annotation)
+                    if get_origin(field_info.annotation) is Literal:
+                        literal_args = get_args(field_info.annotation)
+                        field_value = literal_args[0]
+                    example[field_name] = field_value
+                union_docs.append(f"\nExample {t.__name__}:")
+                union_docs.append(json.dumps(example, indent=2))
+
+        # Return first type as main example
+        if isinstance(first_type, type) and issubclass(first_type, BaseModel):
+            example, _ = get_example_value(first_type)
+            # Ensure tool_type or other discriminator is preserved
+            for field_name, field_info in first_type.model_fields.items():
+                if get_origin(field_info.annotation) is Literal:
+                    literal_args = get_args(field_info.annotation)
+                    if (
+                        isinstance(example[field_name], dict)
+                        and "value" in example[field_name]
+                    ):
+                        example[field_name]["value"] = literal_args[0]
+                    else:
+                        example[field_name] = literal_args[0]
+            return example, union_docs
+        main_example, docs = get_example_value(first_type)
+        return main_example, docs + union_docs
+    elif origin is Literal:
+        # Handle Literal type directly
+        return args[0], []
+    else:
+        return "<Unknown>", []
+
+
+def add_json_instructions_to_messages(
+    system_message,
+    user_message,
+    response_model: Optional[Type[BaseModel]] = None,
+    previously_failed_error_messages: List[str] = [],
+) -> Tuple[str, str]:
+    if response_model:
+        type_hints = get_type_hints(response_model)
+        # print("Type hints", type_hints)
+        stringified_fields = {}
+        union_docs = []
+        for key, type_hint in type_hints.items():
+            # print("Key", key, "Type hint", type_hint)
+            example_value, docs = get_example_value(type_hint)
+            union_docs.extend(docs)
+            field = response_model.model_fields[key]  # Updated for Pydantic v2
+
+            # Adjusted for Pydantic v2
+            field_description = (
+                field.description if hasattr(field, "description") else None
+            )
+
+            if field_description:
+                stringified_fields[key] = (example_value, field_description)
+            else:
+                stringified_fields[key] = example_value
+        example_json = json.dumps(
+            {
+                k: v[0] if isinstance(v, tuple) else v
+                for k, v in stringified_fields.items()
+            },
+            indent=4,
+        )
+        description_comments = "\n".join(
+            f"// {k}: {v[1]}"
+            for k, v in stringified_fields.items()
+            if isinstance(v, tuple)
+        )
+
+        # print("Example JSON", example_json)
+        # print("Description comments", description_comments)
+        # print("Union documentation", "\n".join(union_docs))
+        # raise Exception("Stop here")
+
+        system_message += f"""\n\n
+Please deliver your response in the following JSON format:
+```json
+{example_json}
+```
+{description_comments}
+"""
+    if len(union_docs) != 0:
+        system_message += f"""\n\n
+NOTE - the above example included a union type. Here are some additional examples for that union type:
+{chr(10).join(union_docs)}
+"""
+    if len(previously_failed_error_messages) != 0:
+        system_message += f"""\n\n
+Please take special care to follow the format exactly.
+Keep in mind the following:
+- Always use double quotes for strings
+
+Here are some error traces from previous attempts:
+{previously_failed_error_messages}
+"""
+    return system_message, user_message
+
+
+def inject_structured_output_instructions(
+    messages: List[Dict[str, str]],
+    response_model: Optional[Type[BaseModel]] = None,
+    previously_failed_error_messages: List[str] = [],
+) -> List[Dict[str, str]]:
+    prev_system_message_content = messages[0]["content"]
+    prev_user_message_content = messages[1]["content"]
+    system_message, user_message = add_json_instructions_to_messages(
+        prev_system_message_content,
+        prev_user_message_content,
+        response_model,
+        previously_failed_error_messages,
+    )
+    messages[0]["content"] = system_message
+    messages[1]["content"] = user_message
+    return messages

synth_ai/zyk/lms/structured_outputs/rehabilitate.py

@@ -0,0 +1,187 @@
+import ast
+import json
+import logging
+import re
+from typing import Dict, List, Type, Union
+
+from pydantic import BaseModel
+
+from synth_ai.zyk.lms.vendors.base import VendorBase
+from synth_ai.zyk.lms.vendors.core.openai_api import OpenAIStructuredOutputClient
+
+
+def pull_out_structured_output(
+    response_raw: str, response_model: Type[BaseModel]
+) -> BaseModel:
+    logger = logging.getLogger(__name__)
+    #logger.debug(f"Raw response received: {response_raw}")
+
+    assert isinstance(
+        response_raw, str
+    ), f"Response raw is not a string: {type(response_raw)}"
+
+    # Use regex to extract JSON content within ```json ... ```
+    json_pattern = re.compile(r"```json\s*(\{.*\})\s*```", re.DOTALL)
+    match = json_pattern.search(response_raw)
+    if match:
+        response_prepared = match.group(1).strip()
+    else:
+        # Fallback to existing parsing if no code fencing is found
+        if "```" in response_raw:
+            response_prepared = response_raw.split("```")[1].strip()
+        else:
+            response_prepared = response_raw.strip()
+
+    # Replace "null" with '"None"' if needed (ensure this aligns with your data)
+    response_prepared = response_prepared.replace("null", '"None"')
+
+    logger.debug(f"Prepared response for JSON parsing: {response_prepared}")
+
+    try:
+        response = json.loads(response_prepared)
+        final = response_model(**response)
+    except json.JSONDecodeError as e:
+        # Attempt to parse using ast.literal_eval as a fallback
+        response_prepared = response_prepared.replace("\n", "").replace("\\n", "")
+        response_prepared = response_prepared.replace('\\"', '"')
+        try:
+            response = ast.literal_eval(response_prepared)
+            final = response_model(**response)
+        except Exception as inner_e:
+            raise ValueError(
+                f"Failed to parse response as {response_model}: {inner_e} - {response_prepared}"
+            )
+    except Exception as e:
+        raise ValueError(
+            f"Failed to parse response as {response_model}: {e} - {response_prepared}"
+        )
+    assert isinstance(final, BaseModel), "Structured output must be a Pydantic model"
+    return final
+
+
+def fix_errant_stringified_json_sync(
+    response_raw: str,
+    response_model: Type[BaseModel],
+    models: List[str] = ["gpt-4o-mini", "gpt-4o"],
+) -> BaseModel:
+    try:
+        return pull_out_structured_output(response_raw, response_model)
+    except ValueError as e:
+        # print("")
+        mini_client = OpenAIStructuredOutputClient()
+        messages = [
+            {
+                "role": "system",
+                "content": "An AI attempted to generate stringified json that could be extracted into the provided Pydantic model. This json cannot be extracted, and an error message is provided to elucidate why. Please review the information and return a corrected response. Do not materially change the content of the response, just formatting where necessary.",
+            },
+            {
+                "role": "user",
+                "content": f"# Errant Attempt\n{response_raw}\n# Response Model\n {response_model.model_json_schema()}\n# Error Message\n {str(e)}.",
+            },
+        ]
+        for model in models:
+            if model == "gpt-4o":
+                print(
+                    "Warning - using gpt-4o for structured output correction - this could add up over time (latency, cost)"
+                )
+            try:
+                fixed_response = mini_client._hit_api_sync(
+                    model, messages=messages, lm_config={"temperature": 0.0}
+                )
+                return pull_out_structured_output(fixed_response, response_model)
+            except Exception as e:
+                pass
+        raise ValueError("Failed to fix response using any model")
+
+
+async def fix_errant_stringified_json_async(
+    response_raw: str,
+    response_model: Type[BaseModel],
+    models: List[str] = ["gpt-4o-mini", "gpt-4o"],
+) -> BaseModel:
+    try:
+        return pull_out_structured_output(response_raw, response_model)
+    except ValueError as e:
+        mini_client = OpenAIStructuredOutputClient()
+        messages = [
+            {
+                "role": "system",
+                "content": "An AI attempted to generate stringified json that could be extracted into the provided Pydantic model. This json cannot be extracted, and an error message is provided to elucidate why. Please review the information and return a corrected response. Do not materially change the content of the response, just formatting where necessary.",
+            },
+            {
+                "role": "user",
+                "content": f"# Errant Attempt\n{response_raw}\n# Response Model\n {response_model.model_json_schema()}\n# Error Message\n {str(e)}.",
+            },
+        ]
+        for model in models:
+            if model == "gpt-4o":
+                print(
+                    "Warning - using gpt-4o for structured output correction - this could add up over time (latency, cost)"
+                )
+            try:
+                fixed_response = await mini_client._hit_api_async(
+                    model, messages=messages, lm_config={"temperature": 0.0}
+                )
+                return pull_out_structured_output(fixed_response, response_model)
+            except Exception as e:
+                pass
+        raise ValueError("Failed to fix response using any model")
+
+
+async def fix_errant_forced_async(
+    messages: List[Dict],
+    response_raw: str,
+    response_model: Type[BaseModel],
+    model: str,
+) -> BaseModel:
+    try:
+        return pull_out_structured_output(response_raw, response_model)
+    except ValueError as e:
+        client = OpenAIStructuredOutputClient()
+        messages = [
+            {
+                "role": "system",
+                "content": "An AI attempted to generate stringified json that could be extracted into the provided Pydantic model. This json cannot be extracted, and an error message is provided to elucidate why. Please review the information and return a corrected response. Do not materially change the content of the response, just formatting where necessary.",
+            },
+            {
+                "role": "user",
+                "content": f"<previous messages>\n<system_message>{messages[0]['content']}</system_message>\n<user_message>{messages[1]['content']}</user_message></previous messages> # Errant Attempt\n{response_raw}\n# Response Model\n {response_model.model_json_schema()}\n# Error Message\n {str(e)}.",
+            },
+        ]
+        # print("Broken response:")
+        # print(response_raw)
+        fixed_response = await client._hit_api_async_structured_output(
+            model=model,
+            messages=messages,
+            response_model=response_model,
+            temperature=0.0,
+        )
+        # print("Fixed response:")
+        # print(fixed_response)
+        return fixed_response.structured_output
+
+
+def fix_errant_forced_sync(
+    response_raw: str,
+    response_model: Type[BaseModel],
+    model: str,
+) -> BaseModel:
+    client = OpenAIStructuredOutputClient()
+    messages = [
+        {
+            "role": "system",
+            "content": "An AI attempted to generate a response that could be extracted into the provided Pydantic model. This response cannot be extracted. Please review the information and return a corrected response.",
+        },
+        {
+            "role": "user",
+            "content": f"# Errant Attempt\n{response_raw}\n# Response Model\n {response_model.model_json_schema()}",
+        },
+    ]
+    # print("Broken response:")
+    # print(response_raw)
+    fixed_response = client._hit_api_sync_structured_output(
+        model=model, messages=messages, response_model=response_model, temperature=0.0
+    )
+    # print("Fixed response:")
+    # print(fixed_response)
+    return fixed_response.structured_output

synth_ai/zyk/lms/tools/base.py

@@ -0,0 +1,118 @@
+from typing import Type
+
+from pydantic import BaseModel
+
+
+class BaseTool(BaseModel):
+    name: str
+    arguments: Type[BaseModel]
+    description: str = ""
+    strict: bool = True
+
+    def to_openai_tool(self):
+        schema = self.arguments.model_json_schema()
+        schema["additionalProperties"] = False
+
+        if "properties" in schema:
+            for prop_name, prop_schema in schema["properties"].items():
+                if prop_schema.get("type") == "array":
+                    items_schema = prop_schema.get("items", {})
+                    if not isinstance(items_schema, dict) or not items_schema.get("type"):
+                        prop_schema["items"] = {"type": "string"}
+
+                elif "anyOf" in prop_schema:
+                    for sub_schema in prop_schema["anyOf"]:
+                        if isinstance(sub_schema, dict) and sub_schema.get("type") == "array":
+                            items_schema = sub_schema.get("items", {})
+                            if not isinstance(items_schema, dict) or not items_schema.get("type"):
+                                sub_schema["items"] = {"type": "string"}
+
+        return {
+            "type": "function",
+            "function": {
+                "name": self.name,
+                "description": self.description,
+                "parameters": schema,
+                "strict": self.strict,
+            },
+        }
+
+    def to_anthropic_tool(self):
+        schema = self.arguments.model_json_schema()
+        schema["additionalProperties"] = False
+
+        return {
+            "name": self.name,
+            "description": self.description,
+            "input_schema": {
+                "type": "object",
+                "properties": schema["properties"],
+                "required": schema.get("required", []),
+            },
+        }
+
+    def to_mistral_tool(self):
+        schema = self.arguments.model_json_schema()
+        properties = {}
+        for prop_name, prop in schema.get("properties", {}).items():
+            prop_type = prop["type"]
+            if prop_type == "array" and "items" in prop:
+                properties[prop_name] = {
+                    "type": "array",
+                    "items": prop["items"],
+                    "description": prop.get("description", ""),
+                }
+                continue
+
+            properties[prop_name] = {
+                "type": prop_type,
+                "description": prop.get("description", ""),
+            }
+            if "enum" in prop:
+                properties[prop_name]["enum"] = prop["enum"]
+
+        parameters = {
+            "type": "object",
+            "properties": properties,
+            "required": schema.get("required", []),
+            "additionalProperties": False,
+        }
+        return {
+            "type": "function",
+            "function": {
+                "name": self.name,
+                "description": self.description,
+                "parameters": parameters,
+            },
+        }
+
+    def to_gemini_tool(self):
+        schema = self.arguments.model_json_schema()
+        # Convert Pydantic schema types to Gemini schema types
+        properties = {}
+        for name, prop in schema["properties"].items():
+            prop_type = prop.get("type", "string")
+            if prop_type == "array" and "items" in prop:
+                properties[name] = {
+                    "type": "array",
+                    "items": prop["items"],
+                    "description": prop.get("description", ""),
+                }
+                continue
+
+            properties[name] = {
+                "type": prop_type,
+                "description": prop.get("description", ""),
+            }
+            if "enum" in prop:
+                properties[name]["enum"] = prop["enum"]
+
+        return {
+            "name": self.name,
+            "description": self.description,
+            "parameters": {
+                "type": "object",
+                "properties": properties,
+                "required": schema.get("required", []),
+            },
+        }

synth_ai/zyk/lms/vendors/base.py

@@ -0,0 +1,31 @@
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel
+
+
+class BaseLMResponse(BaseModel):
+    raw_response: str
+    structured_output: Optional[BaseModel] = None
+    tool_calls: Optional[List[Dict]] = None
+
+
+class VendorBase(ABC):
+    used_for_structured_outputs: bool = False
+    exceptions_to_retry: List[Exception] = []
+
+    @abstractmethod
+    async def _hit_api_async(
+        self,
+        messages: List[Dict[str, Any]],
+        response_model_override: Optional[BaseModel] = None,
+    ) -> str:
+        pass
+
+    @abstractmethod
+    def _hit_api_sync(
+        self,
+        messages: List[Dict[str, Any]],
+        response_model_override: Optional[BaseModel] = None,
+    ) -> str:
+        pass