letta-nightly 0.4.1.dev20241010104112__py3-none-any.whl → 0.4.1.dev20241012104008__py3-none-any.whl

letta/agent.py

@@ -481,7 +481,7 @@ class Agent(BaseAgent):
                 first_message=first_message,
                 # streaming
                 stream=stream,
-                stream_inferface=self.interface,
+                stream_interface=self.interface,
                 # putting inner thoughts in func args or not
                 inner_thoughts_in_kwargs_option=inner_thoughts_in_kwargs_option,
             )

letta/agent_store/db.py

@@ -18,13 +18,14 @@ from sqlalchemy import (
     select,
     text,
 )
-from sqlalchemy.orm import declarative_base, mapped_column
+from sqlalchemy.orm import mapped_column
 from sqlalchemy.orm.session import close_all_sessions
 from sqlalchemy.sql import func
 from sqlalchemy_json import MutableJson
 from tqdm import tqdm
 
 from letta.agent_store.storage import StorageConnector, TableType
+from letta.base import Base
 from letta.config import LettaConfig
 from letta.constants import MAX_EMBEDDING_DIM
 from letta.metadata import EmbeddingConfigColumn, ToolCallColumn
@@ -35,7 +36,6 @@ from letta.schemas.openai.chat_completions import ToolCall
 from letta.schemas.passage import Passage
 from letta.settings import settings
 
-Base = declarative_base()
 config = LettaConfig()
 
 
@@ -560,3 +560,9 @@ class SQLLiteStorageConnector(SQLStorageConnector):
 
             # Commit the changes to the database
             session.commit()
+
+
+def attach_base():
+    # This should be invoked in server.py to make sure Base gets initialized properly
+    # DO NOT REMOVE
+    print("Initializing database...")

letta/base.py

@@ -0,0 +1,3 @@
+from sqlalchemy.ext.declarative import declarative_base
+
+Base = declarative_base()

letta/cli/cli.py

@@ -233,25 +233,32 @@ def run(
         # choose from list of llm_configs
         llm_configs = client.list_llm_configs()
         llm_options = [llm_config.model for llm_config in llm_configs]
+        llm_choices = [questionary.Choice(title=llm_config.pretty_print(), value=llm_config) for llm_config in llm_configs]
+
         # select model
         if len(llm_options) == 0:
             raise ValueError("No LLM models found. Please enable a provider.")
         elif len(llm_options) == 1:
             llm_model_name = llm_options[0]
         else:
-            llm_model_name = questionary.select("Select LLM model:", choices=llm_options).ask()
+            llm_model_name = questionary.select("Select LLM model:", choices=llm_choices).ask().model
         llm_config = [llm_config for llm_config in llm_configs if llm_config.model == llm_model_name][0]
 
         # choose form list of embedding configs
         embedding_configs = client.list_embedding_configs()
         embedding_options = [embedding_config.embedding_model for embedding_config in embedding_configs]
+
+        embedding_choices = [
+            questionary.Choice(title=embedding_config.pretty_print(), value=embedding_config) for embedding_config in embedding_configs
+        ]
+
         # select model
         if len(embedding_options) == 0:
             raise ValueError("No embedding models found. Please enable a provider.")
         elif len(embedding_options) == 1:
             embedding_model_name = embedding_options[0]
         else:
-            embedding_model_name = questionary.select("Select embedding model:", choices=embedding_options).ask()
+            embedding_model_name = questionary.select("Select embedding model:", choices=embedding_choices).ask().embedding_model
         embedding_config = [
             embedding_config for embedding_config in embedding_configs if embedding_config.embedding_model == embedding_model_name
         ][0]

letta/constants.py

@@ -75,6 +75,27 @@ NON_USER_MSG_PREFIX = "[This is an automated system message hidden from the user
 LLM_MAX_TOKENS = {
     "DEFAULT": 8192,
     ## OpenAI models: https://platform.openai.com/docs/models/overview
+    # "o1-preview
+    "chatgpt-4o-latest": 128000,
+    # "o1-preview-2024-09-12
+    "gpt-4o-2024-08-06": 128000,
+    "gpt-4-turbo-preview": 128000,
+    "gpt-4o": 128000,
+    "gpt-3.5-turbo-instruct": 16385,
+    "gpt-4-0125-preview": 128000,
+    "gpt-3.5-turbo-0125": 16385,
+    # "babbage-002": 128000,
+    # "davinci-002": 128000,
+    "gpt-4-turbo-2024-04-09": 128000,
+    # "gpt-4o-realtime-preview-2024-10-01
+    "gpt-4-turbo": 8192,
+    "gpt-4o-2024-05-13": 128000,
+    # "o1-mini
+    # "o1-mini-2024-09-12
+    # "gpt-3.5-turbo-instruct-0914
+    "gpt-4o-mini": 128000,
+    # "gpt-4o-realtime-preview
+    "gpt-4o-mini-2024-07-18": 128000,
     # gpt-4
     "gpt-4-1106-preview": 128000,
     "gpt-4": 8192,

letta/llm_api/azure_openai.py

@@ -79,7 +79,7 @@ def azure_openai_chat_completions_request(
         data.pop("tools")
         data.pop("tool_choice", None)  # extra safe,  should exist always (default="auto")
 
-    url = get_azure_chat_completions_endpoint(model_settings.azure_base_url, llm_config.model, model_settings.api_version)
+    url = get_azure_chat_completions_endpoint(model_settings.azure_base_url, llm_config.model, model_settings.azure_api_version)
     response_json = make_post_request(url, headers, data)
     # NOTE: azure openai does not include "content" in the response when it is None, so we need to add it
     if "content" not in response_json["choices"][0].get("message"):

letta/llm_api/helpers.py

@@ -153,6 +153,7 @@ def unpack_inner_thoughts_from_kwargs(choice: Choice, inner_thoughts_key: str) -
                 return new_choice
             else:
                 warnings.warn(f"Did not find inner thoughts in tool call: {str(tool_call)}")
+                return choice
 
         except json.JSONDecodeError as e:
             warnings.warn(f"Failed to strip inner thoughts from kwargs: {e}")

letta/llm_api/llm_api_tools.py

@@ -70,6 +70,10 @@ def retry_with_exponential_backoff(
                 return func(*args, **kwargs)
 
             except requests.exceptions.HTTPError as http_err:
+
+                if not hasattr(http_err, "response") or not http_err.response:
+                    raise
+
                 # Retry on specified errors
                 if http_err.response.status_code in error_codes:
                     # Increment retries
@@ -115,7 +119,7 @@ def create(
     use_tool_naming: bool = True,
     # streaming?
     stream: bool = False,
-    stream_inferface: Optional[Union[AgentRefreshStreamingInterface, AgentChunkStreamingInterface]] = None,
+    stream_interface: Optional[Union[AgentRefreshStreamingInterface, AgentChunkStreamingInterface]] = None,
     # TODO move to llm_config?
     # if unspecified (None), default to something we've tested
     inner_thoughts_in_kwargs_option: OptionState = OptionState.DEFAULT,
@@ -149,19 +153,19 @@ def create(
 
         if stream:  # Client requested token streaming
             data.stream = True
-            assert isinstance(stream_inferface, AgentChunkStreamingInterface) or isinstance(
-                stream_inferface, AgentRefreshStreamingInterface
-            ), type(stream_inferface)
+            assert isinstance(stream_interface, AgentChunkStreamingInterface) or isinstance(
+                stream_interface, AgentRefreshStreamingInterface
+            ), type(stream_interface)
             response = openai_chat_completions_process_stream(
                 url=llm_config.model_endpoint,  # https://api.openai.com/v1 -> https://api.openai.com/v1/chat/completions
                 api_key=model_settings.openai_api_key,
                 chat_completion_request=data,
-                stream_inferface=stream_inferface,
+                stream_interface=stream_interface,
             )
         else:  # Client did not request token streaming (expect a blocking backend response)
             data.stream = False
-            if isinstance(stream_inferface, AgentChunkStreamingInterface):
-                stream_inferface.stream_start()
+            if isinstance(stream_interface, AgentChunkStreamingInterface):
+                stream_interface.stream_start()
             try:
                 response = openai_chat_completions_request(
                     url=llm_config.model_endpoint,  # https://api.openai.com/v1 -> https://api.openai.com/v1/chat/completions
@@ -169,8 +173,8 @@ def create(
                     chat_completion_request=data,
                 )
             finally:
-                if isinstance(stream_inferface, AgentChunkStreamingInterface):
-                    stream_inferface.stream_end()
+                if isinstance(stream_interface, AgentChunkStreamingInterface):
+                    stream_interface.stream_end()
 
         if inner_thoughts_in_kwargs:
             response = unpack_all_inner_thoughts_from_kwargs(response=response, inner_thoughts_key=INNER_THOUGHTS_KWARG)
@@ -317,8 +321,8 @@ def create(
         # They mention that none of the messages can have names, but it seems to not error out (for now)
 
         data.stream = False
-        if isinstance(stream_inferface, AgentChunkStreamingInterface):
-            stream_inferface.stream_start()
+        if isinstance(stream_interface, AgentChunkStreamingInterface):
+            stream_interface.stream_start()
         try:
             # groq uses the openai chat completions API, so this component should be reusable
             assert model_settings.groq_api_key is not None, "Groq key is missing"
@@ -328,8 +332,8 @@ def create(
                 chat_completion_request=data,
             )
         finally:
-            if isinstance(stream_inferface, AgentChunkStreamingInterface):
-                stream_inferface.stream_end()
+            if isinstance(stream_interface, AgentChunkStreamingInterface):
+                stream_interface.stream_end()
 
         if inner_thoughts_in_kwargs:
             response = unpack_all_inner_thoughts_from_kwargs(response=response, inner_thoughts_key=INNER_THOUGHTS_KWARG)

letta/llm_api/openai.py

@@ -41,7 +41,9 @@ from letta.utils import smart_urljoin
 OPENAI_SSE_DONE = "[DONE]"
 
 
-def openai_get_model_list(url: str, api_key: Union[str, None], fix_url: Optional[bool] = False) -> dict:
+def openai_get_model_list(
+    url: str, api_key: Union[str, None], fix_url: Optional[bool] = False, extra_params: Optional[dict] = None
+) -> dict:
     """https://platform.openai.com/docs/api-reference/models/list"""
     from letta.utils import printd
 
@@ -59,8 +61,10 @@ def openai_get_model_list(url: str, api_key: Union[str, None], fix_url: Optional
         headers["Authorization"] = f"Bearer {api_key}"
 
     printd(f"Sending request to {url}")
+    response = None
     try:
-        response = requests.get(url, headers=headers)
+        # TODO add query param "tool" to be true
+        response = requests.get(url, headers=headers, params=extra_params)
         response.raise_for_status()  # Raises HTTPError for 4XX/5XX status
         response = response.json()  # convert to dict from string
         printd(f"response = {response}")
@@ -68,7 +72,8 @@ def openai_get_model_list(url: str, api_key: Union[str, None], fix_url: Optional
     except requests.exceptions.HTTPError as http_err:
         # Handle HTTP errors (e.g., response 4XX, 5XX)
         try:
-            response = response.json()
+            if response:
+                response = response.json()
         except:
             pass
         printd(f"Got HTTPError, exception={http_err}, response={response}")
@@ -76,7 +81,8 @@ def openai_get_model_list(url: str, api_key: Union[str, None], fix_url: Optional
     except requests.exceptions.RequestException as req_err:
         # Handle other requests-related errors (e.g., connection error)
         try:
-            response = response.json()
+            if response:
+                response = response.json()
         except:
             pass
         printd(f"Got RequestException, exception={req_err}, response={response}")
@@ -84,7 +90,8 @@ def openai_get_model_list(url: str, api_key: Union[str, None], fix_url: Optional
     except Exception as e:
         # Handle other potential errors
         try:
-            response = response.json()
+            if response:
+                response = response.json()
         except:
             pass
         printd(f"Got unknown Exception, exception={e}, response={response}")
@@ -154,7 +161,7 @@ def openai_chat_completions_process_stream(
     url: str,
     api_key: str,
     chat_completion_request: ChatCompletionRequest,
-    stream_inferface: Optional[Union[AgentChunkStreamingInterface, AgentRefreshStreamingInterface]] = None,
+    stream_interface: Optional[Union[AgentChunkStreamingInterface, AgentRefreshStreamingInterface]] = None,
     create_message_id: bool = True,
     create_message_datetime: bool = True,
 ) -> ChatCompletionResponse:
@@ -164,7 +171,7 @@ def openai_chat_completions_process_stream(
     on the chunks received from the OpenAI-compatible server POST SSE response.
     """
     assert chat_completion_request.stream == True
-    assert stream_inferface is not None, "Required"
+    assert stream_interface is not None, "Required"
 
     # Count the prompt tokens
     # TODO move to post-request?
@@ -217,8 +224,8 @@ def openai_chat_completions_process_stream(
         ),
     )
 
-    if stream_inferface:
-        stream_inferface.stream_start()
+    if stream_interface:
+        stream_interface.stream_start()
 
     n_chunks = 0  # approx == n_tokens
     try:
@@ -227,17 +234,17 @@ def openai_chat_completions_process_stream(
         ):
             assert isinstance(chat_completion_chunk, ChatCompletionChunkResponse), type(chat_completion_chunk)
 
-            if stream_inferface:
-                if isinstance(stream_inferface, AgentChunkStreamingInterface):
-                    stream_inferface.process_chunk(
+            if stream_interface:
+                if isinstance(stream_interface, AgentChunkStreamingInterface):
+                    stream_interface.process_chunk(
                         chat_completion_chunk,
                         message_id=chat_completion_response.id if create_message_id else chat_completion_chunk.id,
                         message_date=chat_completion_response.created if create_message_datetime else chat_completion_chunk.created,
                     )
-                elif isinstance(stream_inferface, AgentRefreshStreamingInterface):
-                    stream_inferface.process_refresh(chat_completion_response)
+                elif isinstance(stream_interface, AgentRefreshStreamingInterface):
+                    stream_interface.process_refresh(chat_completion_response)
                 else:
-                    raise TypeError(stream_inferface)
+                    raise TypeError(stream_interface)
 
             if chunk_idx == 0:
                 # initialize the choice objects which we will increment with the deltas
@@ -311,13 +318,13 @@ def openai_chat_completions_process_stream(
             n_chunks += 1
 
     except Exception as e:
-        if stream_inferface:
-            stream_inferface.stream_end()
+        if stream_interface:
+            stream_interface.stream_end()
         print(f"Parsing ChatCompletion stream failed with error:\n{str(e)}")
         raise e
     finally:
-        if stream_inferface:
-            stream_inferface.stream_end()
+        if stream_interface:
+            stream_interface.stream_end()
 
     # make sure we didn't leave temp stuff in
     assert all([c.finish_reason != TEMP_STREAM_FINISH_REASON for c in chat_completion_response.choices])

letta/local_llm/vllm/api.py

@@ -3,7 +3,7 @@ from urllib.parse import urljoin
 from letta.local_llm.settings.settings import get_completions_settings
 from letta.local_llm.utils import count_tokens, post_json_auth_request
 
-WEBUI_API_SUFFIX = "/v1/completions"
+WEBUI_API_SUFFIX = "/completions"
 
 
 def get_vllm_completion(endpoint, auth_type, auth_key, model, prompt, context_window, user, grammar=None):

letta/metadata.py

@@ -14,11 +14,10 @@ from sqlalchemy import (
     String,
     TypeDecorator,
     desc,
-    func,
 )
-from sqlalchemy.orm import declarative_base
 from sqlalchemy.sql import func
 
+from letta.base import Base
 from letta.config import LettaConfig
 from letta.schemas.agent import AgentState
 from letta.schemas.api_key import APIKey
@@ -28,6 +27,8 @@ from letta.schemas.enums import JobStatus
 from letta.schemas.job import Job
 from letta.schemas.llm_config import LLMConfig
 from letta.schemas.memory import Memory
+
+# from letta.schemas.message import Message, Passage, Record, RecordType, ToolCall
 from letta.schemas.openai.chat_completions import ToolCall, ToolCallFunction
 from letta.schemas.organization import Organization
 from letta.schemas.source import Source
@@ -36,8 +37,6 @@ from letta.schemas.user import User
 from letta.settings import settings
 from letta.utils import enforce_types, get_utc_time, printd
 
-Base = declarative_base()
-
 
 class LLMConfigColumn(TypeDecorator):
     """Custom type for storing LLMConfig as JSON"""

letta/providers.py

@@ -14,14 +14,18 @@ from letta.schemas.llm_config import LLMConfig
 
 class Provider(BaseModel):
 
-    def list_llm_models(self):
+    def list_llm_models(self) -> List[LLMConfig]:
         return []
 
-    def list_embedding_models(self):
+    def list_embedding_models(self) -> List[EmbeddingConfig]:
         return []
 
-    def get_model_context_window(self, model_name: str):
-        pass
+    def get_model_context_window(self, model_name: str) -> Optional[int]:
+        raise NotImplementedError
+
+    def provider_tag(self) -> str:
+        """String representation of the provider for display purposes"""
+        raise NotImplementedError
 
 
 class LettaProvider(Provider):
@@ -53,23 +57,40 @@ class LettaProvider(Provider):
 class OpenAIProvider(Provider):
     name: str = "openai"
     api_key: str = Field(..., description="API key for the OpenAI API.")
-    base_url: str = "https://api.openai.com/v1"
+    base_url: str = Field(..., description="Base URL for the OpenAI API.")
 
     def list_llm_models(self) -> List[LLMConfig]:
         from letta.llm_api.openai import openai_get_model_list
 
-        response = openai_get_model_list(self.base_url, api_key=self.api_key)
-        model_options = [obj["id"] for obj in response["data"]]
+        # Some hardcoded support for OpenRouter (so that we only get models with tool calling support)...
+        # See: https://openrouter.ai/docs/requests
+        extra_params = {"supported_parameters": "tools"} if "openrouter.ai" in self.base_url else None
+        response = openai_get_model_list(self.base_url, api_key=self.api_key, extra_params=extra_params)
+
+        assert "data" in response, f"OpenAI model query response missing 'data' field: {response}"
 
         configs = []
-        for model_name in model_options:
-            context_window_size = self.get_model_context_window_size(model_name)
+        for model in response["data"]:
+            assert "id" in model, f"OpenAI model missing 'id' field: {model}"
+            model_name = model["id"]
+
+            if "context_length" in model:
+                # Context length is returned in OpenRouter as "context_length"
+                context_window_size = model["context_length"]
+            else:
+                context_window_size = self.get_model_context_window_size(model_name)
 
             if not context_window_size:
                 continue
             configs.append(
                 LLMConfig(model=model_name, model_endpoint_type="openai", model_endpoint=self.base_url, context_window=context_window_size)
             )
+
+        # for OpenAI, sort in reverse order
+        if self.base_url == "https://api.openai.com/v1":
+            # alphnumeric sort
+            configs.sort(key=lambda x: x.model, reverse=True)
+
         return configs
 
     def list_embedding_models(self) -> List[EmbeddingConfig]:
@@ -145,7 +166,7 @@ class OllamaProvider(OpenAIProvider):
             )
         return configs
 
-    def get_model_context_window(self, model_name: str):
+    def get_model_context_window(self, model_name: str) -> Optional[int]:
 
         import requests
 
@@ -293,7 +314,7 @@ class GoogleAIProvider(Provider):
             )
         return configs
 
-    def get_model_context_window(self, model_name: str):
+    def get_model_context_window(self, model_name: str) -> Optional[int]:
         from letta.llm_api.google_ai import google_ai_get_model_context_window
 
         return google_ai_get_model_context_window(self.base_url, self.api_key, model_name)
@@ -354,16 +375,75 @@ class AzureProvider(Provider):
             )
         return configs
 
-    def get_model_context_window(self, model_name: str):
+    def get_model_context_window(self, model_name: str) -> Optional[int]:
         """
         This is hardcoded for now, since there is no API endpoints to retrieve metadata for a model.
         """
         return AZURE_MODEL_TO_CONTEXT_LENGTH.get(model_name, 4096)
 
 
-class VLLMProvider(OpenAIProvider):
+class VLLMChatCompletionsProvider(Provider):
+    """vLLM provider that treats vLLM as an OpenAI /chat/completions proxy"""
+
     # NOTE: vLLM only serves one model at a time (so could configure that through env variables)
-    pass
+    name: str = "vllm"
+    base_url: str = Field(..., description="Base URL for the vLLM API.")
+
+    def list_llm_models(self) -> List[LLMConfig]:
+        # not supported with vLLM
+        from letta.llm_api.openai import openai_get_model_list
+
+        assert self.base_url, "base_url is required for vLLM provider"
+        response = openai_get_model_list(self.base_url, api_key=None)
+
+        configs = []
+        print(response)
+        for model in response["data"]:
+            configs.append(
+                LLMConfig(
+                    model=model["id"],
+                    model_endpoint_type="openai",
+                    model_endpoint=self.base_url,
+                    context_window=model["max_model_len"],
+                )
+            )
+        return configs
+
+    def list_embedding_models(self) -> List[EmbeddingConfig]:
+        # not supported with vLLM
+        return []
+
+
+class VLLMCompletionsProvider(Provider):
+    """This uses /completions API as the backend, not /chat/completions, so we need to specify a model wrapper"""
+
+    # NOTE: vLLM only serves one model at a time (so could configure that through env variables)
+    name: str = "vllm"
+    base_url: str = Field(..., description="Base URL for the vLLM API.")
+    default_prompt_formatter: str = Field(..., description="Default prompt formatter (aka model wrapper)to use on vLLM /completions API.")
+
+    def list_llm_models(self) -> List[LLMConfig]:
+        # not supported with vLLM
+        from letta.llm_api.openai import openai_get_model_list
+
+        response = openai_get_model_list(self.base_url, api_key=None)
+
+        configs = []
+        for model in response["data"]:
+            configs.append(
+                LLMConfig(
+                    model=model["id"],
+                    model_endpoint_type="vllm",
+                    model_endpoint=self.base_url,
+                    model_wrapper=self.default_prompt_formatter,
+                    context_window=model["max_model_len"],
+                )
+            )
+        return configs
+
+    def list_embedding_models(self) -> List[EmbeddingConfig]:
+        # not supported with vLLM
+        return []
 
 
 class CohereProvider(OpenAIProvider):

letta/schemas/embedding_config.py

@@ -52,3 +52,10 @@ class EmbeddingConfig(BaseModel):
             )
         else:
             raise ValueError(f"Model {model_name} not supported.")
+
+    def pretty_print(self) -> str:
+        return (
+            f"{self.embedding_model}"
+            + (f" [type={self.embedding_endpoint_type}]" if self.embedding_endpoint_type else "")
+            + (f" [ip={self.embedding_endpoint}]" if self.embedding_endpoint else "")
+        )

letta/schemas/llm_config.py

@@ -68,3 +68,10 @@ class LLMConfig(BaseModel):
             )
         else:
             raise ValueError(f"Model {model_name} not supported.")
+
+    def pretty_print(self) -> str:
+        return (
+            f"{self.model}"
+            + (f" [type={self.model_endpoint_type}]" if self.model_endpoint_type else "")
+            + (f" [ip={self.model_endpoint}]" if self.model_endpoint else "")
+        )

letta/server/server.py

@@ -14,8 +14,8 @@ import letta.constants as constants
 import letta.server.utils as server_utils
 import letta.system as system
 from letta.agent import Agent, save_agent
+from letta.agent_store.db import attach_base
 from letta.agent_store.storage import StorageConnector, TableType
-from letta.config import LettaConfig
 from letta.credentials import LettaCredentials
 from letta.data_sources.connectors import DataConnector, load_data
 
@@ -41,7 +41,7 @@ from letta.interface import AgentInterface  # abstract
 from letta.interface import CLIInterface  # for printing to terminal
 from letta.log import get_logger
 from letta.memory import get_memory_functions
-from letta.metadata import MetadataStore
+from letta.metadata import Base, MetadataStore
 from letta.prompts import gpt_system
 from letta.providers import (
     AnthropicProvider,
@@ -50,7 +50,9 @@ from letta.providers import (
     LettaProvider,
     OllamaProvider,
     OpenAIProvider,
-    VLLMProvider,
+    Provider,
+    VLLMChatCompletionsProvider,
+    VLLMCompletionsProvider,
 )
 from letta.schemas.agent import AgentState, AgentType, CreateAgent, UpdateAgentState
 from letta.schemas.api_key import APIKey, APIKeyCreate
@@ -149,23 +151,11 @@ class Server(object):
 
 
 from sqlalchemy import create_engine
-from sqlalchemy.orm import declarative_base, sessionmaker
+from sqlalchemy.orm import sessionmaker
 
-from letta.agent_store.db import MessageModel, PassageModel
 from letta.config import LettaConfig
 
 # NOTE: hack to see if single session management works
-from letta.metadata import (
-    AgentModel,
-    AgentSourceMappingModel,
-    APIKeyModel,
-    BlockModel,
-    JobModel,
-    OrganizationModel,
-    SourceModel,
-    ToolModel,
-    UserModel,
-)
 from letta.settings import model_settings, settings
 
 config = LettaConfig.load()
@@ -182,24 +172,12 @@ else:
     # TODO: don't rely on config storage
     engine = create_engine("sqlite:///" + os.path.join(config.recall_storage_path, "sqlite.db"))
 
-Base = declarative_base()
+
 SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
-Base.metadata.create_all(
-    engine,
-    tables=[
-        UserModel.__table__,
-        AgentModel.__table__,
-        SourceModel.__table__,
-        AgentSourceMappingModel.__table__,
-        APIKeyModel.__table__,
-        BlockModel.__table__,
-        ToolModel.__table__,
-        JobModel.__table__,
-        PassageModel.__table__,
-        MessageModel.__table__,
-        OrganizationModel.__table__,
-    ],
-)
+
+attach_base()
+
+Base.metadata.create_all(bind=engine)
 
 
 # Dependency
@@ -261,18 +239,17 @@ class SyncServer(Server):
         self.add_default_tools(module_name="base")
 
         # collect providers (always has Letta as a default)
-        self._enabled_providers = [LettaProvider()]
+        self._enabled_providers: List[Provider] = [LettaProvider()]
         if model_settings.openai_api_key:
-            self._enabled_providers.append(OpenAIProvider(api_key=model_settings.openai_api_key))
+            self._enabled_providers.append(OpenAIProvider(api_key=model_settings.openai_api_key, base_url=model_settings.openai_api_base))
         if model_settings.anthropic_api_key:
             self._enabled_providers.append(AnthropicProvider(api_key=model_settings.anthropic_api_key))
         if model_settings.ollama_base_url:
-            self._enabled_providers.append(OllamaProvider(base_url=model_settings.ollama_base_url))
-        if model_settings.vllm_base_url:
-            self._enabled_providers.append(VLLMProvider(base_url=model_settings.vllm_base_url))
+            self._enabled_providers.append(OllamaProvider(base_url=model_settings.ollama_base_url, api_key=None))
         if model_settings.gemini_api_key:
             self._enabled_providers.append(GoogleAIProvider(api_key=model_settings.gemini_api_key))
         if model_settings.azure_api_key and model_settings.azure_base_url:
+            assert model_settings.azure_api_version, "AZURE_API_VERSION is required"
             self._enabled_providers.append(
                 AzureProvider(
                     api_key=model_settings.azure_api_key,
@@ -280,6 +257,18 @@ class SyncServer(Server):
                     api_version=model_settings.azure_api_version,
                 )
             )
+        if model_settings.vllm_api_base:
+            # vLLM exposes both a /chat/completions and a /completions endpoint
+            self._enabled_providers.append(
+                VLLMCompletionsProvider(
+                    base_url=model_settings.vllm_api_base,
+                    default_prompt_formatter=model_settings.default_prompt_formatter,
+                )
+            )
+            # NOTE: to use the /chat/completions endpoint, you need to specify extra flags on vLLM startup
+            # see: https://docs.vllm.ai/en/latest/getting_started/examples/openai_chat_completion_client_with_tools.html
+            # e.g. "... --enable-auto-tool-choice --tool-call-parser hermes"
+            self._enabled_providers.append(VLLMChatCompletionsProvider(base_url=model_settings.vllm_api_base))
 
     def save_agents(self):
         """Saves all the agents that are in the in-memory object store"""

letta/settings.py

@@ -4,14 +4,20 @@ from typing import Optional
 from pydantic import Field
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
+from letta.local_llm.constants import DEFAULT_WRAPPER_NAME
+
 
 class ModelSettings(BaseSettings):
 
     # env_prefix='my_prefix_'
 
+    # when we use /completions APIs (instead of /chat/completions), we need to specify a model wrapper
+    # the "model wrapper" is responsible for prompt formatting and function calling parsing
+    default_prompt_formatter: str = DEFAULT_WRAPPER_NAME
+
     # openai
     openai_api_key: Optional[str] = None
-    # TODO: provide overriding BASE_URL?
+    openai_api_base: str = "https://api.openai.com/v1"
 
     # groq
     groq_api_key: Optional[str] = None
@@ -25,13 +31,16 @@ class ModelSettings(BaseSettings):
     # azure
     azure_api_key: Optional[str] = None
     azure_base_url: Optional[str] = None
-    azure_api_version: Optional[str] = None
+    # We provide a default here, since usually people will want to be on the latest API version.
+    azure_api_version: Optional[str] = (
+        "2024-09-01-preview"  # https://learn.microsoft.com/en-us/azure/ai-services/openai/api-version-deprecation
+    )
 
     # google ai
     gemini_api_key: Optional[str] = None
 
     # vLLM
-    vllm_base_url: Optional[str] = None
+    vllm_api_base: Optional[str] = None
 
     # openllm
     openllm_auth_type: Optional[str] = None

{letta_nightly-0.4.1.dev20241010104112.dist-info → letta_nightly-0.4.1.dev20241012104008.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: letta-nightly
-Version: 0.4.1.dev20241010104112
+Version: 0.4.1.dev20241012104008
 Summary: Create LLM agents with long-term memory and custom tools
 License: Apache License
 Author: Letta Team
@@ -20,6 +20,7 @@ Provides-Extra: postgres
 Provides-Extra: qdrant
 Provides-Extra: server
 Provides-Extra: tests
+Requires-Dist: alembic (>=1.13.3,<2.0.0)
 Requires-Dist: autoflake (>=2.3.0,<3.0.0) ; extra == "dev"
 Requires-Dist: black[jupyter] (>=24.2.0,<25.0.0) ; extra == "dev"
 Requires-Dist: chromadb (>=0.4.24,<0.5.0)

{letta_nightly-0.4.1.dev20241010104112.dist-info → letta_nightly-0.4.1.dev20241012104008.dist-info}/RECORD

@@ -1,15 +1,16 @@
 letta/__init__.py,sha256=btKRPdyhkpIyHlCPRLwwz-SCSVcEGNBeheYA-XNesiI,996
 letta/__main__.py,sha256=6Hs2PV7EYc5Tid4g4OtcLXhqVHiNYTGzSBdoOnW2HXA,29
-letta/agent.py,sha256=2yYk8H76EYF8-EF5eFqv8dp8avaBcV_hfXOQcwYuAwU,66559
+letta/agent.py,sha256=pdsd1M_AF3uF-AgJrMmxj8uB0Rkt5AXKBxHNw9mMaP4,66559
 letta/agent_store/chroma.py,sha256=upR5zGnGs6I6btulEYbiZdGG87BgKjxUJOQZ4Y-RQ_M,12492
-letta/agent_store/db.py,sha256=Hbw4HsxPZSQL9h_3I-SiJRisaYdzk1XSdYJFarjrUtM,22575
+letta/agent_store/db.py,sha256=TAQ1rcqS8zuC1hT_BaiXn6tSz5aKHAb-YQxr4QLKgz4,22724
 letta/agent_store/lancedb.py,sha256=8RWmqVjowm5g0cc6DNRcb6f1FHGEqFnccnuekhWY39U,5101
 letta/agent_store/milvus.py,sha256=VxEKz9XR7_3QTY59K_38NtJCCQvi41rhHoFibfzW7yw,8469
 letta/agent_store/qdrant.py,sha256=qIEJhXJb6GzcT4wp8iV5Ox5W1CFMvcPViTI4HLSh59E,7879
 letta/agent_store/storage.py,sha256=QWrPdIEJCnsPg1xnPrG1xbOXmbjpz37ZNhvuH52M7A8,6642
+letta/base.py,sha256=Ba-wt8p59bLmeUONkYSo5MhrkH-_HdT4zE1Y9MVGrSQ,83
 letta/benchmark/benchmark.py,sha256=ebvnwfp3yezaXOQyGXkYCDYpsmre-b9hvNtnyx4xkG0,3701
 letta/benchmark/constants.py,sha256=aXc5gdpMGJT327VuxsT5FngbCK2J41PQYeICBO7g_RE,536
-letta/cli/cli.py,sha256=-gJtASac1OJyboFzXXZtY2MH5DPylYYk3ZO8zV519Tc,15870
+letta/cli/cli.py,sha256=XZDF7EfBYqdYEyDbENVp-tTyGcdvd9Wa1PNk70D-MGs,16195
 letta/cli/cli_config.py,sha256=G7QqPNTtlQ4TdrXZrrFFGblZEhnkyrqN1Cl5z415C-g,8689
 letta/cli/cli_load.py,sha256=aVlGWiNEUs_eG793HLl7cES-dEIuA1CJfZpT1Cm8Uo4,4591
 letta/client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -18,7 +19,7 @@ letta/client/client.py,sha256=bPvSQrbym4xXZu9EfEbX02fpkNVxFBpKoyzK9PFwykE,84515
 letta/client/streaming.py,sha256=bfWlUu7z7EoPfKxBqIarYxGKyrL7Pj79BlliToqcCgI,4592
 letta/client/utils.py,sha256=AQWl2q11AzSjd1Y7slIENoZ6fO1YW1JSv1qw0fPt57k,2419
 letta/config.py,sha256=j2I90fOh9d9__kOYObwTDLbvVwYR50rIql5nzrvREKg,19161
-letta/constants.py,sha256=VV6T8O4w4ju8q5CrPCbvPwHlUTltkeFji-r7hz8LTIw,5930
+letta/constants.py,sha256=PAvnF4l9hDuWSrPSKVoXhwjWb6V282dI43EOOpKZaFE,6572
 letta/credentials.py,sha256=D9mlcPsdDWlIIXQQD8wSPE9M_QvsRrb0p3LB5i9OF5Q,5806
 letta/data_sources/connectors.py,sha256=E2rJNqVT4WEvxBqOQl0YgNKa_JQXkG0h1luw_XLcTis,10232
 letta/embeddings.py,sha256=ayAMxW6RUK1RUpLsDiJCG1oY2H6fgricaoqMa4GBjRE,8170
@@ -35,13 +36,13 @@ letta/humans/examples/cs_phd.txt,sha256=9C9ZAV_VuG7GB31ksy3-_NAyk8rjE6YtVOkhp08k
 letta/interface.py,sha256=QI4hFP0WrNsgM5qX6TbnhH1ZZxsLYr5DaccuxpEQ8S4,12768
 letta/llm_api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 letta/llm_api/anthropic.py,sha256=bAb9PVrpYjo2QN51_SJbW7Vry2_Sf55B05UoruHXb7A,12932
-letta/llm_api/azure_openai.py,sha256=8uBQIYa3WpRcxLXgJTYZrl-GhDGflqxXraRLZlpkdh4,4566
+letta/llm_api/azure_openai.py,sha256=C-fuuholudcLJDWjqnXJwpXsfmGWfNugEVWyj6YCrpg,4572
 letta/llm_api/azure_openai_constants.py,sha256=oXtKrgBFHf744gyt5l1thILXgyi8NDNUrKEa2GGGpjw,278
 letta/llm_api/cohere.py,sha256=vDRd-SUGp1t_JUIdwC3RkIhwMl0OY7n-tAU9uPORYkY,14826
 letta/llm_api/google_ai.py,sha256=3xZ074nSOCC22c15yerA5ngWzh0ex4wxeI-6faNbHPE,17708
-letta/llm_api/helpers.py,sha256=Qe1YC36QjjOKE-Xh1Ss3dhMNcWergOK_MpG9xdtN9CM,9519
-letta/llm_api/llm_api_tools.py,sha256=8HndYHAH6ENL5vFEYn2px6CjZdx3ttCGxbEtRfK2RAY,15237
-letta/llm_api/openai.py,sha256=3C_1PjyBcLDraQur6jBzEONqANRYmL78Ym1SaE6LgqE,21469
+letta/llm_api/helpers.py,sha256=LjUtCjvPzSP3-3Ak0J--2RqUXOO6Of8287mm1L1LAMU,9549
+letta/llm_api/llm_api_tools.py,sha256=Z3eiYUtvZKBVBcmKI2l4qWkKM4hgvLN9Y1aSxXc7y-k,15344
+letta/llm_api/openai.py,sha256=EXpktSI_TYjsCDEXBxdNXsY5uE9Rzb7BPF1F6cz8bkg,21689
 letta/local_llm/README.md,sha256=hFJyw5B0TU2jrh9nb0zGZMgdH-Ei1dSRfhvPQG_NSoU,168
 letta/local_llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 letta/local_llm/chat_completion_proxy.py,sha256=PXgNveahts5DbZ7GVcPShxmrDKropL81PY2JHc31yAA,13091
@@ -74,7 +75,7 @@ letta/local_llm/settings/deterministic_mirostat.py,sha256=kgRikcxYHfIbPFydHW6W7I
 letta/local_llm/settings/settings.py,sha256=ZAbzDpu2WsBXjVGXJ-TKUpS99VTI__3EoZml9KqYef0,2971
 letta/local_llm/settings/simple.py,sha256=HAO2jBJ_hJCEsXWIJcD0sckR0tI0zs3x2CPdf6ORQLs,719
 letta/local_llm/utils.py,sha256=DTv8S0lIpFBZlY6fjprtAU3n06eOPDKT6EXwnOgexLo,11363
-letta/local_llm/vllm/api.py,sha256=KhIB-cSYGKKsq5ZVarxPDAL9fTL8HT2-WmWS5gnY6QQ,2593
+letta/local_llm/vllm/api.py,sha256=2kAGZjc_GH9ILJnVRq-45yfsfKELVfbC9VEl_cIC6vg,2590
 letta/local_llm/webui/api.py,sha256=kkxncdCFq1vjgvaHOoQ__j7rcDPgC1F64KcEm94Y6Rs,2639
 letta/local_llm/webui/legacy_api.py,sha256=k3H3y4qp2Fs-XmP24iSIEyvq6wjWFWBzklY3-wRAJNI,2335
 letta/local_llm/webui/legacy_settings.py,sha256=BLmd3TSx5StnY3ibjwaxYATPt_Lvq-o1rlcc_-Q1JcU,538
@@ -82,7 +83,7 @@ letta/local_llm/webui/settings.py,sha256=gmLHfiOl1u4JmlAZU2d2O8YKF9lafdakyjwR_ft
 letta/log.py,sha256=QHquDnL7oUAvdKlAwUlCK9zXKDMUjrU9WA0bxnMsP0Y,2101
 letta/main.py,sha256=7uuiFv8TeOd3K92WwfYkls7l7KXJ5P0l2zgK9AfmD08,18502
 letta/memory.py,sha256=6q1x3-PY-PeXzAt6hvP-UF1ajvroPZ7XW-5nLy-JhMo,17657
-letta/metadata.py,sha256=-LUv7w5yaCn10wh7vnFCrhH4s5M5F0K3Ui7pClXN3tE,33225
+letta/metadata.py,sha256=50pEJ5BuNKT8dPv34-H_Id9an3ql45Ze7BKqn7DGQ6s,33256
 letta/openai_backcompat/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 letta/openai_backcompat/openai_object.py,sha256=Y1ZS1sATP60qxJiOsjOP3NbwSzuzvkNAvb3DeuhM5Uk,13490
 letta/persistence_manager.py,sha256=LlLgEDpSafCPAiyKmuq0NvVAnfBkZo6TWbGIKYQjQBs,5200
@@ -106,13 +107,13 @@ letta/prompts/system/memgpt_doc.txt,sha256=AsT55NOORoH-K-p0fxklrDRZ3qHs4MIKMuR-M
 letta/prompts/system/memgpt_gpt35_extralong.txt,sha256=FheNhYoIzNz6qnJKhVquZVSMj3HduC48reFaX7Pf7ig,5046
 letta/prompts/system/memgpt_intuitive_knowledge.txt,sha256=sA7c3urYqREVnSBI81nTGImXAekqC0Fxc7RojFqud1g,2966
 letta/prompts/system/memgpt_modified_chat.txt,sha256=HOaPVurEftD8KsuwsclDgE2afIfklMjxhuSO96q1-6I,4656
-letta/providers.py,sha256=nC0xGWEDIpnZz_tiFMswrko9HFSUQQJO8o9jRxGStb0,13579
+letta/providers.py,sha256=1vxUxMpLHyZ_XDVTUcpB98HOqszRwnt2sX9xU8zts1U,16949
 letta/pytest.ini,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 letta/schemas/agent.py,sha256=ztnUqdhY9V3g0jsbTjF1ypKPC1tZx4QVFaRuLAOXNSA,6230
 letta/schemas/api_key.py,sha256=u07yzzMn-hBAHZIIKbWY16KsgiFjSNR8lAghpMUo3_4,682
 letta/schemas/block.py,sha256=1_GwFtfykloYU4Mp2DV3-DqkvsKo79Mu3LAzVYOgMzk,3998
 letta/schemas/document.py,sha256=JpvU0YkvOVLvHaDNcg-ihFzpeHC2zqsWBgyJ6zHnfNw,745
-letta/schemas/embedding_config.py,sha256=KhEOStOD8VbHFyLI9pkQVbTG1Av2g-Ql0yf9M868LME,2570
+letta/schemas/embedding_config.py,sha256=1kD6NpiXeH4roVumxqDAKk7xt8SpXGWNhZs_XXUSlEU,2855
 letta/schemas/enums.py,sha256=WfRYpLh_pD-VyhEnp3Y6pPfx063zq2o4jky6PulqO8w,629
 letta/schemas/health.py,sha256=zT6mYovvD17iJRuu2rcaQQzbEEYrkwvAE9TB7iU824c,139
 letta/schemas/job.py,sha256=bYDrjbJm2B4LUTSkLUdQR_HDhL2E23g0EHf7E23ezYU,1547
@@ -120,7 +121,7 @@ letta/schemas/letta_base.py,sha256=4QXFgyjCHqIagi8B6_4nmqb9eoJ52Y6aCxBxQpGX48M,2
 letta/schemas/letta_message.py,sha256=Slgxa59qZfdvqXuCVHOt03u-7JL456ZY-WLaK5UYYKU,6234
 letta/schemas/letta_request.py,sha256=Fv4hZKLMefCISwxuElXB0vsGDjqnzZuCbCbHPUPTirQ,1852
 letta/schemas/letta_response.py,sha256=_UJoO3UtC3F5DtQCHzdiGM1SHNPYPKvopIWqg8t5YZw,1564
-letta/schemas/llm_config.py,sha256=oC-RTimpLhNmn80AyCLnx8YUwA0sIR1UqshYxHVbamU,2671
+letta/schemas/llm_config.py,sha256=xwHBO_sY-Kahtwxt_0Qz4j2CRSeJ0pGAIxpqyK_2Exk,2930
 letta/schemas/memory.py,sha256=mDJbe9mzTw0HSn6tiouC3z32r-8Fc_EaeqxDy_hXf0U,9327
 letta/schemas/message.py,sha256=1C6lz4yvMTusE8zORKbi_BKnb_X_-RuUvdpM5YeZ21o,33409
 letta/schemas/openai/chat_completion_request.py,sha256=Fa7xnSnG7WUQounJhnDu0fTSxoR6xOAh2bODuqmfypI,3345
@@ -166,7 +167,7 @@ letta/server/rest_api/routers/v1/tools.py,sha256=MEhxu-zMS2ff_wwcRpMuQyWA71w_3BJ
 letta/server/rest_api/routers/v1/users.py,sha256=Y2rDvHOG1B5FLSOjutY3R22vt48IngbZ-9h8CohG5rc,3378
 letta/server/rest_api/static_files.py,sha256=NG8sN4Z5EJ8JVQdj19tkFa9iQ1kBPTab9f_CUxd_u4Q,3143
 letta/server/rest_api/utils.py,sha256=Fc2ZGKzLaBa2sEtSTVjJ8D5M0xIwsWC0CVAOIJaD3rY,2176
-letta/server/server.py,sha256=Vac_o-C1l5wOybARMhmuU4Q02rl7YJPB51rjLA0cL9I,81970
+letta/server/server.py,sha256=XWnXt7ViLwihQVpwykxdr-fcAOGhhyGjKSM4Z-nvxw4,82206
 letta/server/startup.sh,sha256=jeGV7B_PS0hS-tT6o6GpACrUbV9WV1NI2L9aLoUDDtc,311
 letta/server/static_files/assets/index-3ab03d5b.css,sha256=OrA9W4iKJ5h2Wlr7GwdAT4wow0CM8hVit1yOxEL49Qw,54295
 letta/server/static_files/assets/index-9a9c449b.js,sha256=qoWUq6_kuLhE9NFkNeCBptgq-oERW46r0tB3JlWe_qc,1818951
@@ -179,12 +180,12 @@ letta/server/ws_api/example_client.py,sha256=95AA5UFgTlNJ0FUQkLxli8dKNx48MNm3eWG
 letta/server/ws_api/interface.py,sha256=TWl9vkcMCnLsUtgsuENZ-ku2oMDA-OUTzLh_yNRoMa4,4120
 letta/server/ws_api/protocol.py,sha256=M_-gM5iuDBwa1cuN2IGNCG5GxMJwU2d3XW93XALv9s8,1821
 letta/server/ws_api/server.py,sha256=C2Kv48PCwl46DQFb0ZP30s86KJLQ6dZk2AhWQEZn9pY,6004
-letta/settings.py,sha256=a1dN-ntNXM46IuF-ITG9u881aLdESfNGWl8_uBYSH20,2677
+letta/settings.py,sha256=6VWC3vtTa8vqj6dqos4p_xHTMJNJS_8LRGJmqvaU1-o,3219
 letta/streaming_interface.py,sha256=_FPUWy58j50evHcpXyd7zB1wWqeCc71NCFeWh_TBvnw,15736
 letta/system.py,sha256=buKYPqG5n2x41hVmWpu6JUpyd7vTWED9Km2_M7dLrvk,6960
 letta/utils.py,sha256=neUs7mxNfndzRL5XUxerr8Lic6w7qnyyvf8FBwMnyWw,30852
-letta_nightly-0.4.1.dev20241010104112.dist-info/LICENSE,sha256=mExtuZ_GYJgDEI38GWdiEYZizZS4KkVt2SF1g_GPNhI,10759
-letta_nightly-0.4.1.dev20241010104112.dist-info/METADATA,sha256=9yfZIXzwV7Roc1ajgsVQT6PkEN1q2TBKODtjJrMyVeo,5967
-letta_nightly-0.4.1.dev20241010104112.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-letta_nightly-0.4.1.dev20241010104112.dist-info/entry_points.txt,sha256=2zdiyGNEZGV5oYBuS-y2nAAgjDgcC9yM_mHJBFSRt5U,40
-letta_nightly-0.4.1.dev20241010104112.dist-info/RECORD,,
+letta_nightly-0.4.1.dev20241012104008.dist-info/LICENSE,sha256=mExtuZ_GYJgDEI38GWdiEYZizZS4KkVt2SF1g_GPNhI,10759
+letta_nightly-0.4.1.dev20241012104008.dist-info/METADATA,sha256=bzoVG8Kbu7WPOswZWQ4HGRQoNqmrA2EvT8XP1sbuQ6U,6008
+letta_nightly-0.4.1.dev20241012104008.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+letta_nightly-0.4.1.dev20241012104008.dist-info/entry_points.txt,sha256=2zdiyGNEZGV5oYBuS-y2nAAgjDgcC9yM_mHJBFSRt5U,40
+letta_nightly-0.4.1.dev20241012104008.dist-info/RECORD,,