letta-nightly 0.5.2.dev20241118104226__tar.gz → 0.5.2.dev20241119104253__tar.gz

{letta_nightly-0.5.2.dev20241118104226 → letta_nightly-0.5.2.dev20241119104253}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: letta-nightly
-Version: 0.5.2.dev20241118104226
+Version: 0.5.2.dev20241119104253
 Summary: Create LLM agents with long-term memory and custom tools
 License: Apache License
 Author: Letta Team

{letta_nightly-0.5.2.dev20241118104226 → letta_nightly-0.5.2.dev20241119104253}/letta/agent.py

@@ -1584,7 +1584,8 @@ class Agent(BaseAgent):
 
     def count_tokens(self) -> int:
         """Count the tokens in the current context window"""
-        return self.get_context_window().context_window_size_current
+        context_window_breakdown = self.get_context_window()
+        return context_window_breakdown.context_window_size_current
 
 
 def save_agent(agent: Agent, ms: MetadataStore):

{letta_nightly-0.5.2.dev20241118104226 → letta_nightly-0.5.2.dev20241119104253}/letta/constants.py

@@ -19,7 +19,7 @@ IN_CONTEXT_MEMORY_KEYWORD = "CORE_MEMORY"
 TOOL_CALL_ID_MAX_LEN = 29
 
 # minimum context window size
-MIN_CONTEXT_WINDOW = 4000
+MIN_CONTEXT_WINDOW = 4096
 
 # embeddings
 MAX_EMBEDDING_DIM = 4096  # maximum supported embeding size - do NOT change or else DBs will need to be reset

{letta_nightly-0.5.2.dev20241118104226 → letta_nightly-0.5.2.dev20241119104253}/letta/functions/functions.py

@@ -3,7 +3,7 @@ import inspect
 import os
 from textwrap import dedent  # remove indentation
 from types import ModuleType
-from typing import Optional
+from typing import Optional, List
 
 from letta.constants import CLI_WARNING_PREFIX
 from letta.functions.schema_generator import generate_schema
@@ -15,6 +15,7 @@ def derive_openai_json_schema(source_code: str, name: Optional[str] = None) -> d
         # Define a custom environment with necessary imports
         env = {
             "Optional": Optional,  # Add any other required imports here
+            "List": List
         }
 
         env.update(globals())

{letta_nightly-0.5.2.dev20241118104226 → letta_nightly-0.5.2.dev20241119104253}/letta/llm_api/llm_api_tools.py

@@ -25,6 +25,7 @@ from letta.local_llm.constants import (
     INNER_THOUGHTS_KWARG,
     INNER_THOUGHTS_KWARG_DESCRIPTION,
 )
+from letta.local_llm.utils import num_tokens_from_functions, num_tokens_from_messages
 from letta.schemas.llm_config import LLMConfig
 from letta.schemas.message import Message
 from letta.schemas.openai.chat_completion_request import (
@@ -33,6 +34,7 @@ from letta.schemas.openai.chat_completion_request import (
     cast_message_to_subtype,
 )
 from letta.schemas.openai.chat_completion_response import ChatCompletionResponse
+from letta.settings import ModelSettings
 from letta.streaming_interface import (
     AgentChunkStreamingInterface,
     AgentRefreshStreamingInterface,
@@ -122,10 +124,19 @@ def create(
     """Return response to chat completion with backoff"""
     from letta.utils import printd
 
+    # Count the tokens first, if there's an overflow exit early by throwing an error up the stack
+    # NOTE: we want to include a specific substring in the error message to trigger summarization
+    messages_oai_format = [m.to_openai_dict() for m in messages]
+    prompt_tokens = num_tokens_from_messages(messages=messages_oai_format, model=llm_config.model)
+    function_tokens = num_tokens_from_functions(functions=functions, model=llm_config.model) if functions else 0
+    if prompt_tokens + function_tokens > llm_config.context_window:
+        raise Exception(f"Request exceeds maximum context length ({prompt_tokens + function_tokens} > {llm_config.context_window} tokens)")
+
     if not model_settings:
         from letta.settings import model_settings
 
         model_settings = model_settings
+        assert isinstance(model_settings, ModelSettings)
 
     printd(f"Using model {llm_config.model_endpoint_type}, endpoint: {llm_config.model_endpoint}")
 
@@ -326,6 +337,33 @@ def create(
 
         return response
 
+    elif llm_config.model_endpoint_type == "together":
+        """TogetherAI endpoint that goes via /completions instead of /chat/completions"""
+
+        if stream:
+            raise NotImplementedError(f"Streaming not yet implemented for TogetherAI (via the /completions endpoint).")
+
+        if model_settings.together_api_key is None and llm_config.model_endpoint == "https://api.together.ai/v1/completions":
+            raise ValueError(f"TogetherAI key is missing from letta config file")
+
+        return get_chat_completion(
+            model=llm_config.model,
+            messages=messages,
+            functions=functions,
+            functions_python=functions_python,
+            function_call=function_call,
+            context_window=llm_config.context_window,
+            endpoint=llm_config.model_endpoint,
+            endpoint_type="vllm",  # NOTE: use the vLLM path through /completions
+            wrapper=llm_config.model_wrapper,
+            user=str(user_id),
+            # hint
+            first_message=first_message,
+            # auth-related
+            auth_type="bearer_token",  # NOTE: Together expects bearer token auth
+            auth_key=model_settings.together_api_key,
+        )
+
     # local model
     else:
         if stream:

{letta_nightly-0.5.2.dev20241118104226 → letta_nightly-0.5.2.dev20241119104253}/letta/llm_api/openai.py

@@ -536,7 +536,6 @@ def openai_chat_completions_request(
             tool["function"] = convert_to_structured_output(tool["function"])
 
     response_json = make_post_request(url, headers, data)
-
     return ChatCompletionResponse(**response_json)
 
 

{letta_nightly-0.5.2.dev20241118104226 → letta_nightly-0.5.2.dev20241119104253}/letta/local_llm/utils.py

@@ -94,7 +94,10 @@ def num_tokens_from_functions(functions: List[dict], model: str = "gpt-4"):
     num_tokens = 0
     for function in functions:
         function_tokens = len(encoding.encode(function["name"]))
-        function_tokens += len(encoding.encode(function["description"]))
+        if function["description"]:
+            function_tokens += len(encoding.encode(function["description"]))
+        else:
+            raise ValueError(f"Function {function['name']} has no description, function: {function}")
 
         if "parameters" in function:
             parameters = function["parameters"]
@@ -184,6 +187,7 @@ def num_tokens_from_messages(messages: List[dict], model: str = "gpt-4") -> int:
         https://community.openai.com/t/how-to-calculate-the-tokens-when-using-function-call/266573/11
     """
     try:
+        # Attempt to search for the encoding based on the model string
         encoding = tiktoken.encoding_for_model(model)
     except KeyError:
         # print("Warning: model not found. Using cl100k_base encoding.")
@@ -228,7 +232,13 @@ def num_tokens_from_messages(messages: List[dict], model: str = "gpt-4") -> int:
                     # num_tokens += len(encoding.encode(value["arguments"]))
 
                 else:
-                    num_tokens += len(encoding.encode(value))
+                    if value is None:
+                        # raise ValueError(f"Message has null value: {key} with value: {value} - message={message}")
+                        warnings.warn(f"Message has null value: {key} with value: {value} - message={message}")
+                    else:
+                        if not isinstance(value, str):
+                            raise ValueError(f"Message has non-string value: {key} with value: {value} - message={message}")
+                        num_tokens += len(encoding.encode(value))
 
                 if key == "name":
                     num_tokens += tokens_per_name

{letta_nightly-0.5.2.dev20241118104226 → letta_nightly-0.5.2.dev20241119104253}/letta/providers.py

@@ -2,7 +2,7 @@ from typing import List, Optional
 
 from pydantic import BaseModel, Field, model_validator
 
-from letta.constants import LLM_MAX_TOKENS
+from letta.constants import LLM_MAX_TOKENS, MIN_CONTEXT_WINDOW
 from letta.llm_api.azure_openai import (
     get_azure_chat_completions_endpoint,
     get_azure_embeddings_endpoint,
@@ -67,10 +67,15 @@ class OpenAIProvider(Provider):
         extra_params = {"supported_parameters": "tools"} if "openrouter.ai" in self.base_url else None
         response = openai_get_model_list(self.base_url, api_key=self.api_key, extra_params=extra_params)
 
-        assert "data" in response, f"OpenAI model query response missing 'data' field: {response}"
+        # TogetherAI's response is missing the 'data' field
+        # assert "data" in response, f"OpenAI model query response missing 'data' field: {response}"
+        if "data" in response:
+            data = response["data"]
+        else:
+            data = response
 
         configs = []
-        for model in response["data"]:
+        for model in data:
             assert "id" in model, f"OpenAI model missing 'id' field: {model}"
             model_name = model["id"]
 
@@ -82,6 +87,32 @@ class OpenAIProvider(Provider):
 
             if not context_window_size:
                 continue
+
+            # TogetherAI includes the type, which we can use to filter out embedding models
+            if self.base_url == "https://api.together.ai/v1":
+                if "type" in model and model["type"] != "chat":
+                    continue
+
+                # for TogetherAI, we need to skip the models that don't support JSON mode / function calling
+                # requests.exceptions.HTTPError: HTTP error occurred: 400 Client Error: Bad Request for url: https://api.together.ai/v1/chat/completions | Status code: 400, Message: {
+                #   "error": {
+                #     "message": "mistralai/Mixtral-8x7B-v0.1 is not supported for JSON mode/function calling",
+                #     "type": "invalid_request_error",
+                #     "param": null,
+                #     "code": "constraints_model"
+                #   }
+                # }
+                if "config" not in model:
+                    continue
+                if "chat_template" not in model["config"]:
+                    continue
+                if model["config"]["chat_template"] is None:
+                    continue
+                if "tools" not in model["config"]["chat_template"]:
+                    continue
+                # if "config" in data and "chat_template" in data["config"] and "tools" not in data["config"]["chat_template"]:
+                # continue
+
             configs.append(
                 LLMConfig(model=model_name, model_endpoint_type="openai", model_endpoint=self.base_url, context_window=context_window_size)
             )
@@ -325,6 +356,113 @@ class GroqProvider(OpenAIProvider):
         raise NotImplementedError
 
 
+class TogetherProvider(OpenAIProvider):
+    """TogetherAI provider that uses the /completions API
+
+    TogetherAI can also be used via the /chat/completions API
+    by settings OPENAI_API_KEY and OPENAI_API_BASE to the TogetherAI API key
+    and API URL, however /completions is preferred because their /chat/completions
+    function calling support is limited.
+    """
+
+    name: str = "together"
+    base_url: str = "https://api.together.ai/v1"
+    api_key: str = Field(..., description="API key for the TogetherAI API.")
+    default_prompt_formatter: str = Field(..., description="Default prompt formatter (aka model wrapper) to use on vLLM /completions API.")
+
+    def list_llm_models(self) -> List[LLMConfig]:
+        from letta.llm_api.openai import openai_get_model_list
+
+        response = openai_get_model_list(self.base_url, api_key=self.api_key)
+
+        # TogetherAI's response is missing the 'data' field
+        # assert "data" in response, f"OpenAI model query response missing 'data' field: {response}"
+        if "data" in response:
+            data = response["data"]
+        else:
+            data = response
+
+        configs = []
+        for model in data:
+            assert "id" in model, f"TogetherAI model missing 'id' field: {model}"
+            model_name = model["id"]
+
+            if "context_length" in model:
+                # Context length is returned in OpenRouter as "context_length"
+                context_window_size = model["context_length"]
+            else:
+                context_window_size = self.get_model_context_window_size(model_name)
+
+            # We need the context length for embeddings too
+            if not context_window_size:
+                continue
+
+            # Skip models that are too small for Letta
+            if context_window_size <= MIN_CONTEXT_WINDOW:
+                continue
+
+            # TogetherAI includes the type, which we can use to filter for embedding models
+            if "type" in model and model["type"] not in ["chat", "language"]:
+                continue
+
+            configs.append(
+                LLMConfig(
+                    model=model_name,
+                    model_endpoint_type="together",
+                    model_endpoint=self.base_url,
+                    model_wrapper=self.default_prompt_formatter,
+                    context_window=context_window_size,
+                )
+            )
+
+        return configs
+
+    def list_embedding_models(self) -> List[EmbeddingConfig]:
+        # TODO renable once we figure out how to pass API keys through properly
+        return []
+
+        # from letta.llm_api.openai import openai_get_model_list
+
+        # response = openai_get_model_list(self.base_url, api_key=self.api_key)
+
+        # # TogetherAI's response is missing the 'data' field
+        # # assert "data" in response, f"OpenAI model query response missing 'data' field: {response}"
+        # if "data" in response:
+        #     data = response["data"]
+        # else:
+        #     data = response
+
+        # configs = []
+        # for model in data:
+        #     assert "id" in model, f"TogetherAI model missing 'id' field: {model}"
+        #     model_name = model["id"]
+
+        #     if "context_length" in model:
+        #         # Context length is returned in OpenRouter as "context_length"
+        #         context_window_size = model["context_length"]
+        #     else:
+        #         context_window_size = self.get_model_context_window_size(model_name)
+
+        #     if not context_window_size:
+        #         continue
+
+        #     # TogetherAI includes the type, which we can use to filter out embedding models
+        #     if "type" in model and model["type"] not in ["embedding"]:
+        #         continue
+
+        #     configs.append(
+        #         EmbeddingConfig(
+        #             embedding_model=model_name,
+        #             embedding_endpoint_type="openai",
+        #             embedding_endpoint=self.base_url,
+        #             embedding_dim=context_window_size,
+        #             embedding_chunk_size=300,  # TODO: change?
+        #         )
+        #     )
+
+        # return configs
+
+
 class GoogleAIProvider(Provider):
     # gemini
     api_key: str = Field(..., description="API key for the Google AI API.")

{letta_nightly-0.5.2.dev20241118104226 → letta_nightly-0.5.2.dev20241119104253}/letta/schemas/llm_config.py

@@ -35,6 +35,7 @@ class LLMConfig(BaseModel):
         "vllm",
         "hugging-face",
         "mistral",
+        "together",  # completions endpoint
     ] = Field(..., description="The endpoint type for the model.")
     model_endpoint: Optional[str] = Field(None, description="The endpoint for the model.")
     model_wrapper: Optional[str] = Field(None, description="The wrapper for the model.")

{letta_nightly-0.5.2.dev20241118104226 → letta_nightly-0.5.2.dev20241119104253}/letta/schemas/openai/chat_completion_response.py

@@ -46,6 +46,7 @@ class Choice(BaseModel):
     index: int
     message: Message
     logprobs: Optional[Dict[str, Union[List[MessageContentLogProb], None]]] = None
+    seed: Optional[int] = None  # found in TogetherAI
 
 
 class UsageStatistics(BaseModel):

{letta_nightly-0.5.2.dev20241118104226 → letta_nightly-0.5.2.dev20241119104253}/letta/server/server.py

@@ -49,6 +49,7 @@ from letta.providers import (
     OllamaProvider,
     OpenAIProvider,
     Provider,
+    TogetherProvider,
     VLLMChatCompletionsProvider,
     VLLMCompletionsProvider,
 )
@@ -303,7 +304,18 @@ class SyncServer(Server):
                 )
             )
         if model_settings.groq_api_key:
-            self._enabled_providers.append(GroqProvider(api_key=model_settings.groq_api_key))
+            self._enabled_providers.append(
+                GroqProvider(
+                    api_key=model_settings.groq_api_key,
+                )
+            )
+        if model_settings.together_api_key:
+            self._enabled_providers.append(
+                TogetherProvider(
+                    api_key=model_settings.together_api_key,
+                    default_prompt_formatter=model_settings.default_prompt_formatter,
+                )
+            )
         if model_settings.vllm_api_base:
             # vLLM exposes both a /chat/completions and a /completions endpoint
             self._enabled_providers.append(

{letta_nightly-0.5.2.dev20241118104226 → letta_nightly-0.5.2.dev20241119104253}/letta/services/tool_manager.py

@@ -70,6 +70,10 @@ class ToolManager:
             pydantic_tool.organization_id = actor.organization_id
             tool_data = pydantic_tool.model_dump()
             tool = ToolModel(**tool_data)
+            # The description is most likely auto-generated via the json_schema,
+            # so copy it over into the top-level description field
+            if tool.description is None:
+                tool.description = tool.json_schema.get("description", None)
             tool.create(session, actor=actor)
 
         return tool.to_pydantic()

{letta_nightly-0.5.2.dev20241118104226 → letta_nightly-0.5.2.dev20241119104253}/letta/settings.py

@@ -43,6 +43,9 @@ class ModelSettings(BaseSettings):
     # google ai
     gemini_api_key: Optional[str] = None
 
+    # together
+    together_api_key: Optional[str] = None
+
     # vLLM
     vllm_api_base: Optional[str] = None
 

{letta_nightly-0.5.2.dev20241118104226 → letta_nightly-0.5.2.dev20241119104253}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "letta-nightly"
-version = "0.5.2.dev20241118104226"
+version = "0.5.2.dev20241119104253"
 packages = [
     {include = "letta"}
 ]