letta-nightly 0.4.1.dev20241010104112__tar.gz → 0.4.1.dev20241011104054__tar.gz

{letta_nightly-0.4.1.dev20241010104112 → letta_nightly-0.4.1.dev20241011104054}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: letta-nightly
-Version: 0.4.1.dev20241010104112
+Version: 0.4.1.dev20241011104054
 Summary: Create LLM agents with long-term memory and custom tools
 License: Apache License
 Author: Letta Team

{letta_nightly-0.4.1.dev20241010104112 → letta_nightly-0.4.1.dev20241011104054}/letta/cli/cli.py

@@ -14,7 +14,9 @@ from letta.constants import CLI_WARNING_PREFIX, LETTA_DIR
 from letta.local_llm.constants import ASSISTANT_MESSAGE_CLI_SYMBOL
 from letta.log import get_logger
 from letta.metadata import MetadataStore
+from letta.schemas.embedding_config import EmbeddingConfig
 from letta.schemas.enums import OptionState
+from letta.schemas.llm_config import LLMConfig
 from letta.schemas.memory import ChatMemory, Memory
 from letta.server.server import logger as server_logger
 
@@ -233,25 +235,46 @@ def run(
         # choose from list of llm_configs
         llm_configs = client.list_llm_configs()
         llm_options = [llm_config.model for llm_config in llm_configs]
+
+        # TODO move into LLMConfig as a class method?
+        def prettify_llm_config(llm_config: LLMConfig) -> str:
+            return f"{llm_config.model}" + f" ({llm_config.model_endpoint})" if llm_config.model_endpoint else ""
+
+        llm_choices = [questionary.Choice(title=prettify_llm_config(llm_config), value=llm_config) for llm_config in llm_configs]
+
         # select model
         if len(llm_options) == 0:
             raise ValueError("No LLM models found. Please enable a provider.")
         elif len(llm_options) == 1:
             llm_model_name = llm_options[0]
         else:
-            llm_model_name = questionary.select("Select LLM model:", choices=llm_options).ask()
+            llm_model_name = questionary.select("Select LLM model:", choices=llm_choices).ask().model
         llm_config = [llm_config for llm_config in llm_configs if llm_config.model == llm_model_name][0]
 
         # choose form list of embedding configs
         embedding_configs = client.list_embedding_configs()
         embedding_options = [embedding_config.embedding_model for embedding_config in embedding_configs]
+
+        # TODO move into EmbeddingConfig as a class method?
+        def prettify_embed_config(embedding_config: EmbeddingConfig) -> str:
+            return (
+                f"{embedding_config.embedding_model}" + f" ({embedding_config.embedding_endpoint})"
+                if embedding_config.embedding_endpoint
+                else ""
+            )
+
+        embedding_choices = [
+            questionary.Choice(title=prettify_embed_config(embedding_config), value=embedding_config)
+            for embedding_config in embedding_configs
+        ]
+
         # select model
         if len(embedding_options) == 0:
             raise ValueError("No embedding models found. Please enable a provider.")
         elif len(embedding_options) == 1:
             embedding_model_name = embedding_options[0]
         else:
-            embedding_model_name = questionary.select("Select embedding model:", choices=embedding_options).ask()
+            embedding_model_name = questionary.select("Select embedding model:", choices=embedding_choices).ask().embedding_model
         embedding_config = [
             embedding_config for embedding_config in embedding_configs if embedding_config.embedding_model == embedding_model_name
         ][0]

{letta_nightly-0.4.1.dev20241010104112 → letta_nightly-0.4.1.dev20241011104054}/letta/constants.py

@@ -75,6 +75,27 @@ NON_USER_MSG_PREFIX = "[This is an automated system message hidden from the user
 LLM_MAX_TOKENS = {
     "DEFAULT": 8192,
     ## OpenAI models: https://platform.openai.com/docs/models/overview
+    # "o1-preview
+    "chatgpt-4o-latest": 128000,
+    # "o1-preview-2024-09-12
+    "gpt-4o-2024-08-06": 128000,
+    "gpt-4-turbo-preview": 128000,
+    "gpt-4o": 128000,
+    "gpt-3.5-turbo-instruct": 16385,
+    "gpt-4-0125-preview": 128000,
+    "gpt-3.5-turbo-0125": 16385,
+    # "babbage-002": 128000,
+    # "davinci-002": 128000,
+    "gpt-4-turbo-2024-04-09": 128000,
+    # "gpt-4o-realtime-preview-2024-10-01
+    "gpt-4-turbo": 8192,
+    "gpt-4o-2024-05-13": 128000,
+    # "o1-mini
+    # "o1-mini-2024-09-12
+    # "gpt-3.5-turbo-instruct-0914
+    "gpt-4o-mini": 128000,
+    # "gpt-4o-realtime-preview
+    "gpt-4o-mini-2024-07-18": 128000,
     # gpt-4
     "gpt-4-1106-preview": 128000,
     "gpt-4": 8192,

{letta_nightly-0.4.1.dev20241010104112 → letta_nightly-0.4.1.dev20241011104054}/letta/llm_api/openai.py

@@ -41,7 +41,9 @@ from letta.utils import smart_urljoin
 OPENAI_SSE_DONE = "[DONE]"
 
 
-def openai_get_model_list(url: str, api_key: Union[str, None], fix_url: Optional[bool] = False) -> dict:
+def openai_get_model_list(
+    url: str, api_key: Union[str, None], fix_url: Optional[bool] = False, extra_params: Optional[dict] = None
+) -> dict:
     """https://platform.openai.com/docs/api-reference/models/list"""
     from letta.utils import printd
 
@@ -60,7 +62,8 @@ def openai_get_model_list(url: str, api_key: Union[str, None], fix_url: Optional
 
     printd(f"Sending request to {url}")
     try:
-        response = requests.get(url, headers=headers)
+        # TODO add query param "tool" to be true
+        response = requests.get(url, headers=headers, params=extra_params)
         response.raise_for_status()  # Raises HTTPError for 4XX/5XX status
         response = response.json()  # convert to dict from string
         printd(f"response = {response}")

{letta_nightly-0.4.1.dev20241010104112 → letta_nightly-0.4.1.dev20241011104054}/letta/providers.py

@@ -53,23 +53,40 @@ class LettaProvider(Provider):
 class OpenAIProvider(Provider):
     name: str = "openai"
     api_key: str = Field(..., description="API key for the OpenAI API.")
-    base_url: str = "https://api.openai.com/v1"
+    base_url: str = Field(..., description="Base URL for the OpenAI API.")
 
     def list_llm_models(self) -> List[LLMConfig]:
         from letta.llm_api.openai import openai_get_model_list
 
-        response = openai_get_model_list(self.base_url, api_key=self.api_key)
-        model_options = [obj["id"] for obj in response["data"]]
+        # Some hardcoded support for OpenRouter (so that we only get models with tool calling support)...
+        # See: https://openrouter.ai/docs/requests
+        extra_params = {"supported_parameters": "tools"} if "openrouter.ai" in self.base_url else None
+        response = openai_get_model_list(self.base_url, api_key=self.api_key, extra_params=extra_params)
+
+        assert "data" in response, f"OpenAI model query response missing 'data' field: {response}"
 
         configs = []
-        for model_name in model_options:
-            context_window_size = self.get_model_context_window_size(model_name)
+        for model in response["data"]:
+            assert "id" in model, f"OpenAI model missing 'id' field: {model}"
+            model_name = model["id"]
+
+            if "context_length" in model:
+                # Context length is returned in OpenRouter as "context_length"
+                context_window_size = model["context_length"]
+            else:
+                context_window_size = self.get_model_context_window_size(model_name)
 
             if not context_window_size:
                 continue
             configs.append(
                 LLMConfig(model=model_name, model_endpoint_type="openai", model_endpoint=self.base_url, context_window=context_window_size)
             )
+
+        # for OpenAI, sort in reverse order
+        if self.base_url == "https://api.openai.com/v1":
+            # alphnumeric sort
+            configs.sort(key=lambda x: x.model, reverse=True)
+
         return configs
 
     def list_embedding_models(self) -> List[EmbeddingConfig]:

{letta_nightly-0.4.1.dev20241010104112 → letta_nightly-0.4.1.dev20241011104054}/letta/server/server.py

@@ -50,6 +50,7 @@ from letta.providers import (
     LettaProvider,
     OllamaProvider,
     OpenAIProvider,
+    Provider,
     VLLMProvider,
 )
 from letta.schemas.agent import AgentState, AgentType, CreateAgent, UpdateAgentState
@@ -261,9 +262,9 @@ class SyncServer(Server):
         self.add_default_tools(module_name="base")
 
         # collect providers (always has Letta as a default)
-        self._enabled_providers = [LettaProvider()]
+        self._enabled_providers: List[Provider] = [LettaProvider()]
         if model_settings.openai_api_key:
-            self._enabled_providers.append(OpenAIProvider(api_key=model_settings.openai_api_key))
+            self._enabled_providers.append(OpenAIProvider(api_key=model_settings.openai_api_key, base_url=model_settings.openai_api_base))
         if model_settings.anthropic_api_key:
             self._enabled_providers.append(AnthropicProvider(api_key=model_settings.anthropic_api_key))
         if model_settings.ollama_base_url:

{letta_nightly-0.4.1.dev20241010104112 → letta_nightly-0.4.1.dev20241011104054}/letta/settings.py

@@ -11,7 +11,7 @@ class ModelSettings(BaseSettings):
 
     # openai
     openai_api_key: Optional[str] = None
-    # TODO: provide overriding BASE_URL?
+    openai_api_base: Optional[str] = "https://api.openai.com/v1"
 
     # groq
     groq_api_key: Optional[str] = None

{letta_nightly-0.4.1.dev20241010104112 → letta_nightly-0.4.1.dev20241011104054}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "letta-nightly"
-version = "0.4.1.dev20241010104112"
+version = "0.4.1.dev20241011104054"
 packages = [
     {include = "letta"}
 ]