letta-nightly 0.1.7.dev20240924104148__py3-none-any.whl

letta/llm_api/anthropic.py

@@ -0,0 +1,383 @@
+import json
+import re
+from typing import List, Optional, Union
+
+import requests
+
+from letta.schemas.message import Message
+from letta.schemas.openai.chat_completion_request import ChatCompletionRequest, Tool
+from letta.schemas.openai.chat_completion_response import (
+    ChatCompletionResponse,
+    Choice,
+    FunctionCall,
+)
+from letta.schemas.openai.chat_completion_response import (
+    Message as ChoiceMessage,  # NOTE: avoid conflict with our own Letta Message datatype
+)
+from letta.schemas.openai.chat_completion_response import ToolCall, UsageStatistics
+from letta.utils import get_utc_time, smart_urljoin
+
+BASE_URL = "https://api.anthropic.com/v1"
+
+
+# https://docs.anthropic.com/claude/docs/models-overview
+# Sadly hardcoded
+MODEL_LIST = [
+    {
+        "name": "claude-3-opus-20240229",
+        "context_window": 200000,
+    },
+    {
+        "name": "claude-3-sonnet-20240229",
+        "context_window": 200000,
+    },
+    {
+        "name": "claude-3-haiku-20240307",
+        "context_window": 200000,
+    },
+]
+
+DUMMY_FIRST_USER_MESSAGE = "User initializing bootup sequence."
+
+
+def antropic_get_model_context_window(url: str, api_key: Union[str, None], model: str) -> int:
+    for model_dict in anthropic_get_model_list(url=url, api_key=api_key):
+        if model_dict["name"] == model:
+            return model_dict["context_window"]
+    raise ValueError(f"Can't find model '{model}' in Anthropic model list")
+
+
+def anthropic_get_model_list(url: str, api_key: Union[str, None]) -> dict:
+    """https://docs.anthropic.com/claude/docs/models-overview"""
+
+    # NOTE: currently there is no GET /models, so we need to hardcode
+    return MODEL_LIST
+
+
+def convert_tools_to_anthropic_format(tools: List[Tool], inner_thoughts_in_kwargs: Optional[bool] = True) -> List[dict]:
+    """See: https://docs.anthropic.com/claude/docs/tool-use
+
+    OpenAI style:
+      "tools": [{
+        "type": "function",
+        "function": {
+            "name": "find_movies",
+            "description": "find ....",
+            "parameters": {
+              "type": "object",
+              "properties": {
+                 PARAM: {
+                   "type": PARAM_TYPE,  # eg "string"
+                   "description": PARAM_DESCRIPTION,
+                 },
+                 ...
+              },
+              "required": List[str],
+            }
+        }
+      }
+      ]
+
+    Anthropic style:
+      "tools": [{
+        "name": "find_movies",
+        "description": "find ....",
+        "input_schema": {
+          "type": "object",
+          "properties": {
+             PARAM: {
+               "type": PARAM_TYPE,  # eg "string"
+               "description": PARAM_DESCRIPTION,
+             },
+             ...
+          },
+          "required": List[str],
+        }
+      }
+      ]
+
+      Two small differences:
+        - 1 level less of nesting
+        - "parameters" -> "input_schema"
+    """
+    tools_dict_list = []
+    for tool in tools:
+        tools_dict_list.append(
+            {
+                "name": tool.function.name,
+                "description": tool.function.description,
+                "input_schema": tool.function.parameters,
+            }
+        )
+    return tools_dict_list
+
+
+def merge_tool_results_into_user_messages(messages: List[dict]):
+    """Anthropic API doesn't allow role 'tool'->'user' sequences
+
+    Example HTTP error:
+    messages: roles must alternate between "user" and "assistant", but found multiple "user" roles in a row
+
+    From: https://docs.anthropic.com/claude/docs/tool-use
+    You may be familiar with other APIs that return tool use as separate from the model's primary output,
+    or which use a special-purpose tool or function message role.
+    In contrast, Anthropic's models and API are built around alternating user and assistant messages,
+    where each message is an array of rich content blocks: text, image, tool_use, and tool_result.
+    """
+
+    # TODO walk through the messages list
+    # When a dict (dict_A) with 'role' == 'user' is followed by a dict with 'role' == 'user' (dict B), do the following
+    # dict_A["content"] = dict_A["content"] + dict_B["content"]
+
+    # The result should be a new merged_messages list that doesn't have any back-to-back dicts with 'role' == 'user'
+    merged_messages = []
+    if not messages:
+        return merged_messages
+
+    # Start with the first message in the list
+    current_message = messages[0]
+
+    for next_message in messages[1:]:
+        if current_message["role"] == "user" and next_message["role"] == "user":
+            # Merge contents of the next user message into current one
+            current_content = (
+                current_message["content"]
+                if isinstance(current_message["content"], list)
+                else [{"type": "text", "text": current_message["content"]}]
+            )
+            next_content = (
+                next_message["content"]
+                if isinstance(next_message["content"], list)
+                else [{"type": "text", "text": next_message["content"]}]
+            )
+            merged_content = current_content + next_content
+            current_message["content"] = merged_content
+        else:
+            # Append the current message to result as it's complete
+            merged_messages.append(current_message)
+            # Move on to the next message
+            current_message = next_message
+
+    # Append the last processed message to the result
+    merged_messages.append(current_message)
+
+    return merged_messages
+
+
+def remap_finish_reason(stop_reason: str) -> str:
+    """Remap Anthropic's 'stop_reason' to OpenAI 'finish_reason'
+
+    OpenAI: 'stop', 'length', 'function_call', 'content_filter', null
+    see: https://platform.openai.com/docs/guides/text-generation/chat-completions-api
+
+    From: https://docs.anthropic.com/claude/reference/migrating-from-text-completions-to-messages#stop-reason
+
+    Messages have a stop_reason of one of the following values:
+        "end_turn": The conversational turn ended naturally.
+        "stop_sequence": One of your specified custom stop sequences was generated.
+        "max_tokens": (unchanged)
+
+    """
+    if stop_reason == "end_turn":
+        return "stop"
+    elif stop_reason == "stop_sequence":
+        return "stop"
+    elif stop_reason == "max_tokens":
+        return "length"
+    elif stop_reason == "tool_use":
+        return "function_call"
+    else:
+        raise ValueError(f"Unexpected stop_reason: {stop_reason}")
+
+
+def strip_xml_tags(string: str, tag: Optional[str]) -> str:
+    if tag is None:
+        return string
+    # Construct the regular expression pattern to find the start and end tags
+    tag_pattern = f"<{tag}.*?>|</{tag}>"
+    # Use the regular expression to replace the tags with an empty string
+    return re.sub(tag_pattern, "", string)
+
+
+def convert_anthropic_response_to_chatcompletion(
+    response_json: dict,  # REST response from Google AI API
+    inner_thoughts_xml_tag: Optional[str] = None,
+) -> ChatCompletionResponse:
+    """
+    Example response from Claude 3:
+    response.json = {
+        'id': 'msg_01W1xg9hdRzbeN2CfZM7zD2w',
+        'type': 'message',
+        'role': 'assistant',
+        'content': [
+            {
+                'type': 'text',
+                'text': "<thinking>Analyzing user login event. This is Chad's first
+    interaction with me. I will adjust my personality and rapport accordingly.</thinking>"
+            },
+            {
+                'type':
+                'tool_use',
+                'id': 'toolu_01Ka4AuCmfvxiidnBZuNfP1u',
+                'name': 'core_memory_append',
+                'input': {
+                    'name': 'human',
+                    'content': 'Chad is logging in for the first time. I will aim to build a warm
+    and welcoming rapport.',
+                    'request_heartbeat': True
+                }
+            }
+        ],
+        'model': 'claude-3-haiku-20240307',
+        'stop_reason': 'tool_use',
+        'stop_sequence': None,
+        'usage': {
+            'input_tokens': 3305,
+            'output_tokens': 141
+        }
+    }
+    """
+    prompt_tokens = response_json["usage"]["input_tokens"]
+    completion_tokens = response_json["usage"]["output_tokens"]
+
+    finish_reason = remap_finish_reason(response_json["stop_reason"])
+
+    if isinstance(response_json["content"], list):
+        # inner mono + function call
+        # TODO relax asserts
+        assert len(response_json["content"]) == 2, response_json
+        assert response_json["content"][0]["type"] == "text", response_json
+        assert response_json["content"][1]["type"] == "tool_use", response_json
+        content = strip_xml_tags(string=response_json["content"][0]["text"], tag=inner_thoughts_xml_tag)
+        tool_calls = [
+            ToolCall(
+                id=response_json["content"][1]["id"],
+                type="function",
+                function=FunctionCall(
+                    name=response_json["content"][1]["name"],
+                    arguments=json.dumps(response_json["content"][1]["input"], indent=2),
+                ),
+            )
+        ]
+    else:
+        # just inner mono
+        content = strip_xml_tags(string=response_json["content"], tag=inner_thoughts_xml_tag)
+        tool_calls = None
+
+    assert response_json["role"] == "assistant", response_json
+    choice = Choice(
+        index=0,
+        finish_reason=finish_reason,
+        message=ChoiceMessage(
+            role=response_json["role"],
+            content=content,
+            tool_calls=tool_calls,
+        ),
+    )
+
+    return ChatCompletionResponse(
+        id=response_json["id"],
+        choices=[choice],
+        created=get_utc_time(),
+        model=response_json["model"],
+        usage=UsageStatistics(
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=prompt_tokens + completion_tokens,
+        ),
+    )
+
+
+def anthropic_chat_completions_request(
+    url: str,
+    api_key: str,
+    data: ChatCompletionRequest,
+    inner_thoughts_xml_tag: Optional[str] = "thinking",
+) -> ChatCompletionResponse:
+    """https://docs.anthropic.com/claude/docs/tool-use"""
+    from letta.utils import printd
+
+    url = smart_urljoin(url, "messages")
+    headers = {
+        "Content-Type": "application/json",
+        "x-api-key": api_key,
+        # NOTE: beta headers for tool calling
+        "anthropic-version": "2023-06-01",
+        "anthropic-beta": "tools-2024-04-04",
+    }
+
+    # convert the tools
+    anthropic_tools = None if data.tools is None else convert_tools_to_anthropic_format(data.tools)
+
+    # pydantic -> dict
+    data = data.model_dump(exclude_none=True)
+
+    if "functions" in data:
+        raise ValueError(f"'functions' unexpected in Anthropic API payload")
+
+    # If tools == None, strip from the payload
+    if "tools" in data and data["tools"] is None:
+        data.pop("tools")
+        data.pop("tool_choice", None)  # extra safe,  should exist always (default="auto")
+    # Remap to our converted tools
+    if anthropic_tools is not None:
+        data["tools"] = anthropic_tools
+
+    # Move 'system' to the top level
+    # 'messages: Unexpected role "system". The Messages API accepts a top-level `system` parameter, not "system" as an input message role.'
+    assert data["messages"][0]["role"] == "system", f"Expected 'system' role in messages[0]:\n{data['messages'][0]}"
+    data["system"] = data["messages"][0]["content"]
+    data["messages"] = data["messages"][1:]
+
+    # set `content` to None if missing
+    for message in data["messages"]:
+        if "content" not in message:
+            message["content"] = None
+
+    # Convert to Anthropic format
+
+    msg_objs = [Message.dict_to_message(user_id=None, agent_id=None, openai_message_dict=m) for m in data["messages"]]
+    data["messages"] = [m.to_anthropic_dict(inner_thoughts_xml_tag=inner_thoughts_xml_tag) for m in msg_objs]
+
+    # Handling Anthropic special requirement for 'user' message in front
+    # messages: first message must use the "user" role'
+    if data["messages"][0]["role"] != "user":
+        data["messages"] = [{"role": "user", "content": DUMMY_FIRST_USER_MESSAGE}] + data["messages"]
+
+    # Handle Anthropic's restriction on alternating user/assistant messages
+    data["messages"] = merge_tool_results_into_user_messages(data["messages"])
+
+    # Anthropic also wants max_tokens in the input
+    # It's also part of ChatCompletions
+    assert "max_tokens" in data, data
+
+    # Remove extra fields used by OpenAI but not Anthropic
+    data.pop("frequency_penalty", None)
+    data.pop("logprobs", None)
+    data.pop("n", None)
+    data.pop("top_p", None)
+    data.pop("presence_penalty", None)
+    data.pop("user", None)
+    data.pop("tool_choice", None)
+
+    printd(f"Sending request to {url}")
+    try:
+        response = requests.post(url, headers=headers, json=data)
+        printd(f"response = {response}")
+        response.raise_for_status()  # Raises HTTPError for 4XX/5XX status
+        response = response.json()  # convert to dict from string
+        printd(f"response.json = {response}")
+        response = convert_anthropic_response_to_chatcompletion(response_json=response, inner_thoughts_xml_tag=inner_thoughts_xml_tag)
+        return response
+    except requests.exceptions.HTTPError as http_err:
+        # Handle HTTP errors (e.g., response 4XX, 5XX)
+        printd(f"Got HTTPError, exception={http_err}, payload={data}")
+        raise http_err
+    except requests.exceptions.RequestException as req_err:
+        # Handle other requests-related errors (e.g., connection error)
+        printd(f"Got RequestException, exception={req_err}")
+        raise req_err
+    except Exception as e:
+        # Handle other potential errors
+        printd(f"Got unknown Exception, exception={e}")
+        raise e

letta/llm_api/azure_openai.py

@@ -0,0 +1,155 @@
+from typing import Union
+
+import requests
+
+from letta.schemas.openai.chat_completion_response import ChatCompletionResponse
+from letta.schemas.openai.embedding_response import EmbeddingResponse
+from letta.utils import smart_urljoin
+
+MODEL_TO_AZURE_ENGINE = {
+    "gpt-4-1106-preview": "gpt-4",
+    "gpt-4": "gpt-4",
+    "gpt-4-32k": "gpt-4-32k",
+    "gpt-3.5": "gpt-35-turbo",
+    "gpt-3.5-turbo": "gpt-35-turbo",
+    "gpt-3.5-turbo-16k": "gpt-35-turbo-16k",
+}
+
+
+def clean_azure_endpoint(raw_endpoint_name: str) -> str:
+    """Make sure the endpoint is of format 'https://YOUR_RESOURCE_NAME.openai.azure.com'"""
+    if raw_endpoint_name is None:
+        raise ValueError(raw_endpoint_name)
+    endpoint_address = raw_endpoint_name.strip("/").replace(".openai.azure.com", "")
+    endpoint_address = endpoint_address.replace("http://", "")
+    endpoint_address = endpoint_address.replace("https://", "")
+    return endpoint_address
+
+
+def azure_openai_get_model_list(url: str, api_key: Union[str, None], api_version: str) -> dict:
+    """https://learn.microsoft.com/en-us/rest/api/azureopenai/models/list?view=rest-azureopenai-2023-05-15&tabs=HTTP"""
+    from letta.utils import printd
+
+    # https://xxx.openai.azure.com/openai/models?api-version=xxx
+    url = smart_urljoin(url, "openai")
+    url = smart_urljoin(url, f"models?api-version={api_version}")
+
+    headers = {"Content-Type": "application/json"}
+    if api_key is not None:
+        headers["api-key"] = f"{api_key}"
+
+    printd(f"Sending request to {url}")
+    try:
+        response = requests.get(url, headers=headers)
+        response.raise_for_status()  # Raises HTTPError for 4XX/5XX status
+        response = response.json()  # convert to dict from string
+        printd(f"response = {response}")
+        return response
+    except requests.exceptions.HTTPError as http_err:
+        # Handle HTTP errors (e.g., response 4XX, 5XX)
+        try:
+            response = response.json()
+        except:
+            pass
+        printd(f"Got HTTPError, exception={http_err}, response={response}")
+        raise http_err
+    except requests.exceptions.RequestException as req_err:
+        # Handle other requests-related errors (e.g., connection error)
+        try:
+            response = response.json()
+        except:
+            pass
+        printd(f"Got RequestException, exception={req_err}, response={response}")
+        raise req_err
+    except Exception as e:
+        # Handle other potential errors
+        try:
+            response = response.json()
+        except:
+            pass
+        printd(f"Got unknown Exception, exception={e}, response={response}")
+        raise e
+
+
+def azure_openai_chat_completions_request(
+    resource_name: str, deployment_id: str, api_version: str, api_key: str, data: dict
+) -> ChatCompletionResponse:
+    """https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#chat-completions"""
+    from letta.utils import printd
+
+    assert resource_name is not None, "Missing required field when calling Azure OpenAI"
+    assert deployment_id is not None, "Missing required field when calling Azure OpenAI"
+    assert api_version is not None, "Missing required field when calling Azure OpenAI"
+    assert api_key is not None, "Missing required field when calling Azure OpenAI"
+
+    resource_name = clean_azure_endpoint(resource_name)
+    url = f"https://{resource_name}.openai.azure.com/openai/deployments/{deployment_id}/chat/completions?api-version={api_version}"
+    headers = {"Content-Type": "application/json", "api-key": f"{api_key}"}
+
+    # If functions == None, strip from the payload
+    if "functions" in data and data["functions"] is None:
+        data.pop("functions")
+        data.pop("function_call", None)  # extra safe,  should exist always (default="auto")
+
+    if "tools" in data and data["tools"] is None:
+        data.pop("tools")
+        data.pop("tool_choice", None)  # extra safe,  should exist always (default="auto")
+
+    printd(f"Sending request to {url}")
+    try:
+        data["messages"] = [i.to_openai_dict() for i in data["messages"]]
+        response = requests.post(url, headers=headers, json=data)
+        printd(f"response = {response}")
+        response.raise_for_status()  # Raises HTTPError for 4XX/5XX status
+        response = response.json()  # convert to dict from string
+        printd(f"response.json = {response}")
+        # NOTE: azure openai does not include "content" in the response when it is None, so we need to add it
+        if "content" not in response["choices"][0].get("message"):
+            response["choices"][0]["message"]["content"] = None
+        response = ChatCompletionResponse(**response)  # convert to 'dot-dict' style which is the openai python client default
+        return response
+    except requests.exceptions.HTTPError as http_err:
+        # Handle HTTP errors (e.g., response 4XX, 5XX)
+        printd(f"Got HTTPError, exception={http_err}, payload={data}")
+        raise http_err
+    except requests.exceptions.RequestException as req_err:
+        # Handle other requests-related errors (e.g., connection error)
+        printd(f"Got RequestException, exception={req_err}")
+        raise req_err
+    except Exception as e:
+        # Handle other potential errors
+        printd(f"Got unknown Exception, exception={e}")
+        raise e
+
+
+def azure_openai_embeddings_request(
+    resource_name: str, deployment_id: str, api_version: str, api_key: str, data: dict
+) -> EmbeddingResponse:
+    """https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#embeddings"""
+    from letta.utils import printd
+
+    resource_name = clean_azure_endpoint(resource_name)
+    url = f"https://{resource_name}.openai.azure.com/openai/deployments/{deployment_id}/embeddings?api-version={api_version}"
+    headers = {"Content-Type": "application/json", "api-key": f"{api_key}"}
+
+    printd(f"Sending request to {url}")
+    try:
+        response = requests.post(url, headers=headers, json=data)
+        printd(f"response = {response}")
+        response.raise_for_status()  # Raises HTTPError for 4XX/5XX status
+        response = response.json()  # convert to dict from string
+        printd(f"response.json = {response}")
+        response = EmbeddingResponse(**response)  # convert to 'dot-dict' style which is the openai python client default
+        return response
+    except requests.exceptions.HTTPError as http_err:
+        # Handle HTTP errors (e.g., response 4XX, 5XX)
+        printd(f"Got HTTPError, exception={http_err}, payload={data}")
+        raise http_err
+    except requests.exceptions.RequestException as req_err:
+        # Handle other requests-related errors (e.g., connection error)
+        printd(f"Got RequestException, exception={req_err}")
+        raise req_err
+    except Exception as e:
+        # Handle other potential errors
+        printd(f"Got unknown Exception, exception={e}")
+        raise e