synth-ai 0.1.0.dev4__py3-none-any.whl → 0.1.0.dev6__py3-none-any.whl

synth_ai/zyk/lms/core/main.py

@@ -0,0 +1,245 @@
+from typing import Any, Dict, List, Literal, Optional
+
+from pydantic import BaseModel, Field
+
+from synth_ai.zyk.lms.core.exceptions import StructuredOutputCoercionFailureException
+from synth_ai.zyk.lms.core.vendor_clients import (
+    anthropic_naming_regexes,
+    get_client,
+    openai_naming_regexes,
+)
+from synth_ai.zyk.lms.structured_outputs.handler import StructuredOutputHandler
+from synth_ai.zyk.lms.vendors.base import VendorBase
+
+
+def build_messages(
+    sys_msg: str,
+    user_msg: str,
+    images_bytes: List = [],
+    model_name: Optional[str] = None,
+) -> List[Dict]:
+    if len(images_bytes) > 0 and any(
+        regex.match(model_name) for regex in openai_naming_regexes
+    ):
+        return [
+            {"role": "system", "content": sys_msg},
+            {
+                "role": "user",
+                "content": [{"type": "text", "text": user_msg}]
+                + [
+                    {
+                        "type": "image_url",
+                        "image_url": {"url": f"data:image/jpeg;base64,{image_bytes}"},
+                    }
+                    for image_bytes in images_bytes
+                ],
+            },
+        ]
+    elif len(images_bytes) > 0 and any(
+        regex.match(model_name) for regex in anthropic_naming_regexes
+    ):
+        system_info = {"role": "system", "content": sys_msg}
+        user_info = {
+            "role": "user",
+            "content": [{"type": "text", "text": user_msg}]
+            + [
+                {
+                    "type": "image",
+                    "source": {
+                        "type": "base64",
+                        "media_type": "image/png",
+                        "data": image_bytes,
+                    },
+                }
+                for image_bytes in images_bytes
+            ],
+        }
+        return [system_info, user_info]
+    elif len(images_bytes) > 0:
+        raise ValueError("Images are not yet supported for this model")
+    else:
+        return [
+            {"role": "system", "content": sys_msg},
+            {"role": "user", "content": user_msg},
+        ]
+
+
+class LM:
+    # if str
+    model_name: str
+    client: VendorBase
+    lm_config: Dict[str, Any]
+    structured_output_handler: StructuredOutputHandler
+
+    def __init__(
+        self,
+        model_name: str,
+        formatting_model_name: str,
+        temperature: float,
+        max_retries: Literal["None", "Few", "Many"] = "Few",
+        structured_output_mode: Literal[
+            "stringified_json", "forced_json"
+        ] = "stringified_json",
+        synth_logging: bool = True,
+    ):
+        # print("Structured output mode", structured_output_mode)
+        self.client = get_client(
+            model_name,
+            with_formatting=structured_output_mode == "forced_json",
+            synth_logging=synth_logging,
+        )
+        # print(self.client.__class__)
+
+        formatting_client = get_client(formatting_model_name, with_formatting=True)
+
+        max_retries_dict = {"None": 0, "Few": 2, "Many": 5}
+        self.structured_output_handler = StructuredOutputHandler(
+            self.client,
+            formatting_client,
+            structured_output_mode,
+            {"max_retries": max_retries_dict.get(max_retries, 2)},
+        )
+        self.backup_structured_output_handler = StructuredOutputHandler(
+            self.client,
+            formatting_client,
+            "forced_json",
+            {"max_retries": max_retries_dict.get(max_retries, 2)},
+        )
+        self.lm_config = {"temperature": temperature}
+        self.model_name = model_name
+
+    def respond_sync(
+        self,
+        system_message: Optional[str] = None,
+        user_message: Optional[str] = None,
+        messages: Optional[List[Dict]] = None,
+        images_as_bytes: List[Any] = [],
+        response_model: Optional[BaseModel] = None,
+        use_ephemeral_cache_only: bool = False,
+    ):
+        assert (system_message is None) == (
+            user_message is None
+        ), "Must provide both system_message and user_message or neither"
+        assert (
+            (messages is None) != (system_message is None)
+        ), "Must provide either messages or system_message/user_message pair, but not both"
+
+        if messages is None:
+            messages = build_messages(
+                system_message, user_message, images_as_bytes, self.model_name
+            )
+
+        if response_model:
+            try:
+                return self.structured_output_handler.call_sync(
+                    messages,
+                    model=self.model_name,
+                    lm_config=self.lm_config,
+                    response_model=response_model,
+                    use_ephemeral_cache_only=use_ephemeral_cache_only,
+                )
+            except StructuredOutputCoercionFailureException:
+                # print("Falling back to backup handler")
+                return self.backup_structured_output_handler.call_sync(
+                    messages,
+                    model=self.model_name,
+                    lm_config=self.lm_config,
+                    response_model=response_model,
+                    use_ephemeral_cache_only=use_ephemeral_cache_only,
+                )
+        else:
+            return self.client._hit_api_sync(
+                messages=messages,
+                model=self.model_name,
+                lm_config=self.lm_config,
+                use_ephemeral_cache_only=use_ephemeral_cache_only,
+            )
+
+    async def respond_async(
+        self,
+        system_message: Optional[str] = None,
+        user_message: Optional[str] = None,
+        messages: Optional[List[Dict]] = None,
+        images_as_bytes: List[Any] = [],
+        response_model: Optional[BaseModel] = None,
+        use_ephemeral_cache_only: bool = False,
+    ):
+        # "In respond_async")
+        assert (system_message is None) == (
+            user_message is None
+        ), "Must provide both system_message and user_message or neither"
+        assert (
+            (messages is None) != (system_message is None)
+        ), "Must provide either messages or system_message/user_message pair, but not both"
+
+        if messages is None:
+            messages = build_messages(
+                system_message, user_message, images_as_bytes, self.model_name
+            )
+
+        if response_model:
+            try:
+                # "Trying structured output handler")
+                return await self.structured_output_handler.call_async(
+                    messages,
+                    model=self.model_name,
+                    lm_config=self.lm_config,
+                    response_model=response_model,
+                    use_ephemeral_cache_only=use_ephemeral_cache_only,
+                )
+            except StructuredOutputCoercionFailureException:
+                # print("Falling back to backup handler")
+                return await self.backup_structured_output_handler.call_async(
+                    messages,
+                    model=self.model_name,
+                    lm_config=self.lm_config,
+                    response_model=response_model,
+                    use_ephemeral_cache_only=use_ephemeral_cache_only,
+                )
+        else:
+            # print("Calling API no response model")
+            return await self.client._hit_api_async(
+                messages=messages,
+                model=self.model_name,
+                lm_config=self.lm_config,
+                use_ephemeral_cache_only=use_ephemeral_cache_only,
+            )
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    # Update json instructions to handle nested pydantic?
+    class Thought(BaseModel):
+        argument_keys: List[str] = Field(description="The keys of the arguments")
+        argument_values: List[str] = Field(
+            description="Stringified JSON for the values of the arguments"
+        )
+
+    class TestModel(BaseModel):
+        emotion: str = Field(description="The emotion expressed")
+        concern: str = Field(description="The concern expressed")
+        action: str = Field(description="The action to be taken")
+        thought: Thought = Field(description="The thought process")
+
+        class Config:
+            schema_extra = {"required": ["thought", "emotion", "concern", "action"]}
+
+    lm = LM(
+        model_name="gpt-4o-mini",
+        formatting_model_name="gpt-4o-mini",
+        temperature=1,
+        max_retries="Few",
+        structured_output_mode="forced_json",
+    )
+    print(
+        asyncio.run(
+            lm.respond_async(
+                system_message="You are a helpful assistant ",
+                user_message="Hello, how are you?",
+                images_as_bytes=[],
+                response_model=TestModel,
+                use_ephemeral_cache_only=False,
+            )
+        )
+    )

synth_ai/zyk/lms/core/vendor_clients.py

@@ -0,0 +1,60 @@
+import re
+from typing import Any, List, Pattern
+
+from synth_ai.zyk.lms.core.all import (
+    AnthropicClient,
+    DeepSeekClient,
+    GeminiClient,
+    # OpenAIClient,
+    OpenAIStructuredOutputClient,
+    TogetherClient,
+)
+
+openai_naming_regexes: List[Pattern] = [
+    re.compile(r"^(ft:)?(o[1,3](-.*)?|gpt-.*)$"),
+]
+openai_formatting_model_regexes: List[Pattern] = [
+    re.compile(r"^(ft:)?gpt-4o(-.*)?$"),
+]
+anthropic_naming_regexes: List[Pattern] = [
+    re.compile(r"^claude-.*$"),
+]
+gemini_naming_regexes: List[Pattern] = [
+    re.compile(r"^gemini-.*$"),
+]
+deepseek_naming_regexes: List[Pattern] = [
+    re.compile(r"^deepseek-.*$"),
+]
+together_naming_regexes: List[Pattern] = [
+    re.compile(r"^.*\/.*$"),
+]
+
+
+def get_client(
+    model_name: str,
+    with_formatting: bool = False,
+    synth_logging: bool = True,
+) -> Any:
+    # print("With formatting", with_formatting)
+    if any(regex.match(model_name) for regex in openai_naming_regexes):
+        # print("Returning OpenAIStructuredOutputClient")
+        return OpenAIStructuredOutputClient(
+            synth_logging=synth_logging,
+        )
+    elif any(regex.match(model_name) for regex in anthropic_naming_regexes):
+        if with_formatting:
+            client = AnthropicClient()
+            client._hit_api_async_structured_output = OpenAIStructuredOutputClient(
+                synth_logging=synth_logging
+            )._hit_api_async
+            return client
+        else:
+            return AnthropicClient()
+    elif any(regex.match(model_name) for regex in gemini_naming_regexes):
+        return GeminiClient()
+    elif any(regex.match(model_name) for regex in deepseek_naming_regexes):
+        return DeepSeekClient()
+    elif any(regex.match(model_name) for regex in together_naming_regexes):
+        return TogetherClient()
+    else:
+        raise ValueError(f"Invalid model name: {model_name}")

synth_ai/zyk/lms/cost/monitor.py

@@ -0,0 +1 @@
+#TODO

synth_ai/zyk/lms/cost/statefulness.py

@@ -0,0 +1 @@
+# Maybe some kind of ephemeral cache

synth_ai/zyk/lms/structured_outputs/handler.py

@@ -0,0 +1,388 @@
+import time
+from abc import ABC, abstractmethod
+from typing import Any, Callable, Dict, List, Literal, Optional, Union
+
+from pydantic import BaseModel
+
+from synth_ai.zyk.lms.core.exceptions import StructuredOutputCoercionFailureException
+from synth_ai.zyk.lms.structured_outputs.inject import (
+    inject_structured_output_instructions,
+)
+from synth_ai.zyk.lms.structured_outputs.rehabilitate import (
+    fix_errant_forced_async,
+    fix_errant_forced_sync,
+    fix_errant_stringified_json_async,
+    fix_errant_stringified_json_sync,
+    pull_out_structured_output,
+)
+from synth_ai.zyk.lms.vendors.base import VendorBase
+from synth_ai.zyk.lms.vendors.constants import SPECIAL_BASE_TEMPS
+
+
+class StructuredHandlerBase(ABC):
+    core_client: VendorBase
+    retry_client: VendorBase
+    handler_params: Dict[str, Any]
+    structured_output_mode: Literal["stringified_json", "forced_json"]
+
+    def __init__(
+        self,
+        core_client: VendorBase,
+        retry_client: VendorBase,
+        handler_params: Optional[Dict[str, Any]] = None,
+        structured_output_mode: Literal[
+            "stringified_json", "forced_json"
+        ] = "stringified_json",
+    ):
+        self.core_client = core_client
+        self.retry_client = retry_client
+        self.handler_params = (
+            handler_params if handler_params is not None else {"retries": 3}
+        )
+        self.structured_output_mode = structured_output_mode
+
+    async def call_async(
+        self,
+        messages: List[Dict[str, Any]],
+        model: str,
+        response_model: BaseModel,
+        temperature: float = 0.0,
+        use_ephemeral_cache_only: bool = False,
+    ) -> BaseModel:
+        if temperature == 0.0:
+            temperature = SPECIAL_BASE_TEMPS.get(model, 0.0)
+        # print("Calling from base")
+        return await self._process_call_async(
+            messages=messages,
+            model=model,
+            response_model=response_model,
+            api_call_method=self.core_client._hit_api_async_structured_output
+            if (not not response_model and self.structured_output_mode == "forced_json")
+            else self.core_client._hit_api_async,
+            temperature=temperature,
+            use_ephemeral_cache_only=use_ephemeral_cache_only,
+        )
+
+    def call_sync(
+        self,
+        messages: List[Dict[str, Any]],
+        response_model: BaseModel,
+        model: str,
+        temperature: float = 0.0,
+        use_ephemeral_cache_only: bool = False,
+    ) -> BaseModel:
+        if temperature == 0.0:
+            temperature = SPECIAL_BASE_TEMPS.get(model, 0.0)
+        return self._process_call_sync(
+            messages=messages,
+            model=model,
+            response_model=response_model,
+            api_call_method=self.core_client._hit_api_sync_structured_output
+            if (not not response_model and self.structured_output_mode == "forced_json")
+            else self.core_client._hit_api_sync,
+            temperature=temperature,
+            use_ephemeral_cache_only=use_ephemeral_cache_only,
+        )
+
+    @abstractmethod
+    async def _process_call_async(
+        self,
+        messages: List[Dict[str, Any]],
+        model: str,
+        response_model: BaseModel,
+        api_call_method,
+        use_ephemeral_cache_only: bool = False,
+    ) -> BaseModel:
+        pass
+
+    @abstractmethod
+    def _process_call_sync(
+        self,
+        messages: List[Dict[str, Any]],
+        model: str,
+        response_model: BaseModel,
+        api_call_method,
+        use_ephemeral_cache_only: bool = False,
+    ) -> BaseModel:
+        pass
+
+
+class StringifiedJSONHandler(StructuredHandlerBase):
+    core_client: VendorBase
+    retry_client: VendorBase
+    handler_params: Dict[str, Any]
+
+    def __init__(
+        self,
+        core_client: VendorBase,
+        retry_client: VendorBase,
+        handler_params: Dict[str, Any] = {"retries": 3},
+    ):
+        super().__init__(
+            core_client,
+            retry_client,
+            handler_params,
+            structured_output_mode="stringified_json",
+        )
+
+    async def _process_call_async(
+        self,
+        messages: List[Dict[str, Any]],
+        model: str,
+        response_model: BaseModel,
+        temperature: float,
+        api_call_method: Callable,
+        use_ephemeral_cache_only: bool = False,
+    ) -> BaseModel:
+        # print("In _process_call_async")
+        assert isinstance(
+            api_call_method, Callable
+        ), "api_call_method must be a callable"
+        assert (
+            response_model is not None
+        ), "Don't use this handler for unstructured outputs"
+        remaining_retries = self.handler_params.get("retries", 2)
+        previously_failed_error_messages = []
+        structured_output = None
+
+        while remaining_retries > 0:
+            messages_with_json_formatting_instructions = (
+                inject_structured_output_instructions(
+                    messages=messages,
+                    response_model=response_model,
+                    previously_failed_error_messages=previously_failed_error_messages,
+                )
+            )
+            t0 = time.time()
+            raw_text_response_or_cached_hit = await api_call_method(
+                messages=messages_with_json_formatting_instructions,
+                model=model,
+                lm_config={"response_model": None, "temperature": temperature},
+                use_ephemeral_cache_only=use_ephemeral_cache_only,
+            )
+            # print(f"Time to get response: {time.time() - t0}")
+            if not isinstance(raw_text_response_or_cached_hit, str):
+                return raw_text_response_or_cached_hit
+            else:
+                raw_text_response = raw_text_response_or_cached_hit
+            try:
+                structured_output = pull_out_structured_output(
+                    raw_text_response, response_model
+                )
+                break
+            # except Exception as e:
+            #     try:
+            #         structured_output = await fix_errant_stringified_json_async(raw_text_response, response_model)
+            #         break
+            except Exception as e:
+                try:
+                    # t0 = time.time()
+                    # print(f"Got error {e}, attempting to fix")
+                    structured_output = await fix_errant_forced_async(
+                        messages_with_json_formatting_instructions,
+                        raw_text_response,
+                        response_model,
+                        "gpt-4o-mini",
+                    )
+
+                    # print(f"Time to fix: {time.time() - t0}")
+                    break
+                except Exception as e:
+                    previously_failed_error_messages.append(
+                        f"Generated attempt and got error. Attempt:\n\n{raw_text_response}\n\nError:\n\n{e}"
+                    )
+                    remaining_retries -= 1
+        if structured_output is None:
+            raise StructuredOutputCoercionFailureException(
+                "Failed to get structured output"
+            )
+        return structured_output
+
+    def _process_call_sync(
+        self,
+        messages: List[Dict[str, Any]],
+        model: str,
+        response_model: BaseModel,
+        temperature: float,
+        api_call_method: Callable,
+        use_ephemeral_cache_only: bool = False,
+    ) -> BaseModel:
+        assert isinstance(
+            api_call_method, Callable
+        ), "api_call_method must be a callable"
+        assert (
+            response_model is not None
+        ), "Don't use this handler for unstructured outputs"
+        remaining_retries = self.handler_params.get("retries", 2)
+        previously_failed_error_messages = []
+        structured_output = None
+
+        while remaining_retries > 0:
+            messages_with_json_formatting_instructions = (
+                inject_structured_output_instructions(
+                    messages=messages,
+                    response_model=response_model,
+                    previously_failed_error_messages=previously_failed_error_messages,
+                )
+            )
+            # t0 = time.time()
+            raw_text_response_or_cached_hit = api_call_method(
+                messages=messages_with_json_formatting_instructions,
+                model=model,
+                lm_config={"response_model": None, "temperature": temperature},
+                use_ephemeral_cache_only=use_ephemeral_cache_only,
+            )
+            # print(f"Time to get response: {time.time() - t0}")
+            if not isinstance(raw_text_response_or_cached_hit, str):
+                return raw_text_response_or_cached_hit
+            else:
+                raw_text_response = raw_text_response_or_cached_hit
+            try:
+                structured_output = pull_out_structured_output(
+                    raw_text_response, response_model
+                )
+                break
+            # except Exception:
+            #     try:
+            #         structured_output = fix_errant_stringified_json_sync(raw_text_response, response_model)
+            #         break
+            except Exception as e:
+                try:
+                    # t0 = time.time()
+                    # print(f"Got error {e}, attempting to fix")
+                    structured_output = fix_errant_forced_sync(
+                        raw_text_response, response_model, "gpt-4o-mini"
+                    )
+                    break
+                    # print(f"Time to fix: {time.time() - t0}")
+                except Exception as e:
+                    previously_failed_error_messages.append(
+                        f"Generated attempt and got error. Attempt:\n\n{raw_text_response}\n\nError:\n\n{e}"
+                    )
+                    remaining_retries -= 1
+        if structured_output is None:
+            raise StructuredOutputCoercionFailureException(
+                "Failed to get structured output"
+            )
+        return structured_output
+
+
+class ForcedJSONHandler(StructuredHandlerBase):
+    core_client: VendorBase
+    retry_client: VendorBase
+    handler_params: Dict[str, Any]
+
+    def __init__(
+        self,
+        core_client: VendorBase,
+        retry_client: VendorBase,
+        handler_params: Dict[str, Any] = {},
+    ):
+        super().__init__(
+            core_client,
+            retry_client,
+            handler_params,
+            structured_output_mode="forced_json",
+        )
+
+    async def _process_call_async(
+        self,
+        messages: List[Dict[str, Any]],
+        model: str,
+        response_model: BaseModel,
+        api_call_method: Callable,
+        temperature: float = 0.0,
+        use_ephemeral_cache_only: bool = False,
+    ) -> BaseModel:
+        # print("Forced JSON")
+        assert (
+            response_model is not None
+        ), "Don't use this handler for unstructured outputs"
+        return await api_call_method(
+            messages=messages,
+            model=model,
+            response_model=response_model,
+            temperature=temperature,
+            use_ephemeral_cache_only=use_ephemeral_cache_only,
+        )
+
+    def _process_call_sync(
+        self,
+        messages: List[Dict[str, Any]],
+        model: str,
+        response_model: BaseModel,
+        api_call_method: Callable,
+        temperature: float = 0.0,
+        use_ephemeral_cache_only: bool = False,
+    ) -> BaseModel:
+        assert (
+            response_model is not None
+        ), "Don't use this handler for unstructured outputs"
+        return api_call_method(
+            messages=messages,
+            model=model,
+            response_model=response_model,
+            temperature=temperature,
+            use_ephemeral_cache_only=use_ephemeral_cache_only,
+        )
+
+
+class StructuredOutputHandler:
+    handler: Union[StringifiedJSONHandler, ForcedJSONHandler]
+    mode: Literal["stringified_json", "forced_json"]
+    handler_params: Dict[str, Any]
+
+    def __init__(
+        self,
+        core_client: VendorBase,
+        retry_client: VendorBase,
+        mode: Literal["stringified_json", "forced_json"],
+        handler_params: Dict[str, Any] = {},
+    ):
+        self.mode = mode
+        if self.mode == "stringified_json":
+            self.handler = StringifiedJSONHandler(
+                core_client, retry_client, handler_params
+            )
+        elif self.mode == "forced_json":
+            # print("Forced JSON")
+            self.handler = ForcedJSONHandler(core_client, retry_client, handler_params)
+        else:
+            raise ValueError(f"Invalid mode: {mode}")
+
+    async def call_async(
+        self,
+        messages: List[Dict[str, Any]],
+        model: str,
+        response_model: BaseModel,
+        use_ephemeral_cache_only: bool = False,
+        lm_config: Dict[str, Any] = {},
+    ) -> BaseModel:
+        # print("Output handler call async")
+        return await self.handler.call_async(
+            messages=messages,
+            model=model,
+            response_model=response_model,
+            temperature=lm_config.get(
+                "temperature", SPECIAL_BASE_TEMPS.get(model, 0.0)
+            ),
+            use_ephemeral_cache_only=use_ephemeral_cache_only,
+        )
+
+    def call_sync(
+        self,
+        messages: List[Dict[str, Any]],
+        model: str,
+        response_model: BaseModel,
+        use_ephemeral_cache_only: bool = False,
+        lm_config: Dict[str, Any] = {},
+    ) -> BaseModel:
+        return self.handler.call_sync(
+            messages=messages,
+            model=model,
+            response_model=response_model,
+            temperature=lm_config.get(
+                "temperature", SPECIAL_BASE_TEMPS.get(model, 0.0)
+            ),
+            use_ephemeral_cache_only=use_ephemeral_cache_only,
+        )