letta-nightly 0.4.1.dev20241011104054__py3-none-any.whl → 0.4.1.dev20241013104006__py3-none-any.whl

letta/agent.py

@@ -481,7 +481,7 @@ class Agent(BaseAgent):
                 first_message=first_message,
                 # streaming
                 stream=stream,
-                stream_inferface=self.interface,
+                stream_interface=self.interface,
                 # putting inner thoughts in func args or not
                 inner_thoughts_in_kwargs_option=inner_thoughts_in_kwargs_option,
             )

letta/agent_store/db.py

@@ -18,13 +18,14 @@ from sqlalchemy import (
     select,
     text,
 )
-from sqlalchemy.orm import declarative_base, mapped_column
+from sqlalchemy.orm import mapped_column
 from sqlalchemy.orm.session import close_all_sessions
 from sqlalchemy.sql import func
 from sqlalchemy_json import MutableJson
 from tqdm import tqdm
 
 from letta.agent_store.storage import StorageConnector, TableType
+from letta.base import Base
 from letta.config import LettaConfig
 from letta.constants import MAX_EMBEDDING_DIM
 from letta.metadata import EmbeddingConfigColumn, ToolCallColumn
@@ -35,7 +36,6 @@ from letta.schemas.openai.chat_completions import ToolCall
 from letta.schemas.passage import Passage
 from letta.settings import settings
 
-Base = declarative_base()
 config = LettaConfig()
 
 
@@ -560,3 +560,9 @@ class SQLLiteStorageConnector(SQLStorageConnector):
 
             # Commit the changes to the database
             session.commit()
+
+
+def attach_base():
+    # This should be invoked in server.py to make sure Base gets initialized properly
+    # DO NOT REMOVE
+    print("Initializing database...")

letta/base.py

@@ -0,0 +1,3 @@
+from sqlalchemy.ext.declarative import declarative_base
+
+Base = declarative_base()

letta/cli/cli.py

@@ -14,9 +14,7 @@ from letta.constants import CLI_WARNING_PREFIX, LETTA_DIR
 from letta.local_llm.constants import ASSISTANT_MESSAGE_CLI_SYMBOL
 from letta.log import get_logger
 from letta.metadata import MetadataStore
-from letta.schemas.embedding_config import EmbeddingConfig
 from letta.schemas.enums import OptionState
-from letta.schemas.llm_config import LLMConfig
 from letta.schemas.memory import ChatMemory, Memory
 from letta.server.server import logger as server_logger
 
@@ -235,12 +233,7 @@ def run(
         # choose from list of llm_configs
         llm_configs = client.list_llm_configs()
         llm_options = [llm_config.model for llm_config in llm_configs]
-
-        # TODO move into LLMConfig as a class method?
-        def prettify_llm_config(llm_config: LLMConfig) -> str:
-            return f"{llm_config.model}" + f" ({llm_config.model_endpoint})" if llm_config.model_endpoint else ""
-
-        llm_choices = [questionary.Choice(title=prettify_llm_config(llm_config), value=llm_config) for llm_config in llm_configs]
+        llm_choices = [questionary.Choice(title=llm_config.pretty_print(), value=llm_config) for llm_config in llm_configs]
 
         # select model
         if len(llm_options) == 0:
@@ -255,17 +248,8 @@ def run(
         embedding_configs = client.list_embedding_configs()
         embedding_options = [embedding_config.embedding_model for embedding_config in embedding_configs]
 
-        # TODO move into EmbeddingConfig as a class method?
-        def prettify_embed_config(embedding_config: EmbeddingConfig) -> str:
-            return (
-                f"{embedding_config.embedding_model}" + f" ({embedding_config.embedding_endpoint})"
-                if embedding_config.embedding_endpoint
-                else ""
-            )
-
         embedding_choices = [
-            questionary.Choice(title=prettify_embed_config(embedding_config), value=embedding_config)
-            for embedding_config in embedding_configs
+            questionary.Choice(title=embedding_config.pretty_print(), value=embedding_config) for embedding_config in embedding_configs
         ]
 
         # select model

letta/llm_api/azure_openai.py

@@ -79,7 +79,7 @@ def azure_openai_chat_completions_request(
         data.pop("tools")
         data.pop("tool_choice", None)  # extra safe,  should exist always (default="auto")
 
-    url = get_azure_chat_completions_endpoint(model_settings.azure_base_url, llm_config.model, model_settings.api_version)
+    url = get_azure_chat_completions_endpoint(model_settings.azure_base_url, llm_config.model, model_settings.azure_api_version)
     response_json = make_post_request(url, headers, data)
     # NOTE: azure openai does not include "content" in the response when it is None, so we need to add it
     if "content" not in response_json["choices"][0].get("message"):

letta/llm_api/helpers.py

@@ -153,6 +153,7 @@ def unpack_inner_thoughts_from_kwargs(choice: Choice, inner_thoughts_key: str) -
                 return new_choice
             else:
                 warnings.warn(f"Did not find inner thoughts in tool call: {str(tool_call)}")
+                return choice
 
         except json.JSONDecodeError as e:
             warnings.warn(f"Failed to strip inner thoughts from kwargs: {e}")

letta/llm_api/llm_api_tools.py

@@ -70,6 +70,10 @@ def retry_with_exponential_backoff(
                 return func(*args, **kwargs)
 
             except requests.exceptions.HTTPError as http_err:
+
+                if not hasattr(http_err, "response") or not http_err.response:
+                    raise
+
                 # Retry on specified errors
                 if http_err.response.status_code in error_codes:
                     # Increment retries
@@ -115,7 +119,7 @@ def create(
     use_tool_naming: bool = True,
     # streaming?
     stream: bool = False,
-    stream_inferface: Optional[Union[AgentRefreshStreamingInterface, AgentChunkStreamingInterface]] = None,
+    stream_interface: Optional[Union[AgentRefreshStreamingInterface, AgentChunkStreamingInterface]] = None,
     # TODO move to llm_config?
     # if unspecified (None), default to something we've tested
     inner_thoughts_in_kwargs_option: OptionState = OptionState.DEFAULT,
@@ -149,19 +153,19 @@ def create(
 
         if stream:  # Client requested token streaming
             data.stream = True
-            assert isinstance(stream_inferface, AgentChunkStreamingInterface) or isinstance(
-                stream_inferface, AgentRefreshStreamingInterface
-            ), type(stream_inferface)
+            assert isinstance(stream_interface, AgentChunkStreamingInterface) or isinstance(
+                stream_interface, AgentRefreshStreamingInterface
+            ), type(stream_interface)
             response = openai_chat_completions_process_stream(
                 url=llm_config.model_endpoint,  # https://api.openai.com/v1 -> https://api.openai.com/v1/chat/completions
                 api_key=model_settings.openai_api_key,
                 chat_completion_request=data,
-                stream_inferface=stream_inferface,
+                stream_interface=stream_interface,
             )
         else:  # Client did not request token streaming (expect a blocking backend response)
             data.stream = False
-            if isinstance(stream_inferface, AgentChunkStreamingInterface):
-                stream_inferface.stream_start()
+            if isinstance(stream_interface, AgentChunkStreamingInterface):
+                stream_interface.stream_start()
             try:
                 response = openai_chat_completions_request(
                     url=llm_config.model_endpoint,  # https://api.openai.com/v1 -> https://api.openai.com/v1/chat/completions
@@ -169,8 +173,8 @@ def create(
                     chat_completion_request=data,
                 )
             finally:
-                if isinstance(stream_inferface, AgentChunkStreamingInterface):
-                    stream_inferface.stream_end()
+                if isinstance(stream_interface, AgentChunkStreamingInterface):
+                    stream_interface.stream_end()
 
         if inner_thoughts_in_kwargs:
             response = unpack_all_inner_thoughts_from_kwargs(response=response, inner_thoughts_key=INNER_THOUGHTS_KWARG)
@@ -317,8 +321,8 @@ def create(
         # They mention that none of the messages can have names, but it seems to not error out (for now)
 
         data.stream = False
-        if isinstance(stream_inferface, AgentChunkStreamingInterface):
-            stream_inferface.stream_start()
+        if isinstance(stream_interface, AgentChunkStreamingInterface):
+            stream_interface.stream_start()
         try:
             # groq uses the openai chat completions API, so this component should be reusable
             assert model_settings.groq_api_key is not None, "Groq key is missing"
@@ -328,8 +332,8 @@ def create(
                 chat_completion_request=data,
             )
         finally:
-            if isinstance(stream_inferface, AgentChunkStreamingInterface):
-                stream_inferface.stream_end()
+            if isinstance(stream_interface, AgentChunkStreamingInterface):
+                stream_interface.stream_end()
 
         if inner_thoughts_in_kwargs:
             response = unpack_all_inner_thoughts_from_kwargs(response=response, inner_thoughts_key=INNER_THOUGHTS_KWARG)

letta/llm_api/openai.py

@@ -61,6 +61,7 @@ def openai_get_model_list(
         headers["Authorization"] = f"Bearer {api_key}"
 
     printd(f"Sending request to {url}")
+    response = None
     try:
         # TODO add query param "tool" to be true
         response = requests.get(url, headers=headers, params=extra_params)
@@ -71,7 +72,8 @@ def openai_get_model_list(
     except requests.exceptions.HTTPError as http_err:
         # Handle HTTP errors (e.g., response 4XX, 5XX)
         try:
-            response = response.json()
+            if response:
+                response = response.json()
         except:
             pass
         printd(f"Got HTTPError, exception={http_err}, response={response}")
@@ -79,7 +81,8 @@ def openai_get_model_list(
     except requests.exceptions.RequestException as req_err:
         # Handle other requests-related errors (e.g., connection error)
         try:
-            response = response.json()
+            if response:
+                response = response.json()
         except:
             pass
         printd(f"Got RequestException, exception={req_err}, response={response}")
@@ -87,7 +90,8 @@ def openai_get_model_list(
     except Exception as e:
         # Handle other potential errors
         try:
-            response = response.json()
+            if response:
+                response = response.json()
         except:
             pass
         printd(f"Got unknown Exception, exception={e}, response={response}")
@@ -157,7 +161,7 @@ def openai_chat_completions_process_stream(
     url: str,
     api_key: str,
     chat_completion_request: ChatCompletionRequest,
-    stream_inferface: Optional[Union[AgentChunkStreamingInterface, AgentRefreshStreamingInterface]] = None,
+    stream_interface: Optional[Union[AgentChunkStreamingInterface, AgentRefreshStreamingInterface]] = None,
     create_message_id: bool = True,
     create_message_datetime: bool = True,
 ) -> ChatCompletionResponse:
@@ -167,7 +171,7 @@ def openai_chat_completions_process_stream(
     on the chunks received from the OpenAI-compatible server POST SSE response.
     """
     assert chat_completion_request.stream == True
-    assert stream_inferface is not None, "Required"
+    assert stream_interface is not None, "Required"
 
     # Count the prompt tokens
     # TODO move to post-request?
@@ -220,8 +224,8 @@ def openai_chat_completions_process_stream(
         ),
     )
 
-    if stream_inferface:
-        stream_inferface.stream_start()
+    if stream_interface:
+        stream_interface.stream_start()
 
     n_chunks = 0  # approx == n_tokens
     try:
@@ -230,17 +234,17 @@ def openai_chat_completions_process_stream(
         ):
             assert isinstance(chat_completion_chunk, ChatCompletionChunkResponse), type(chat_completion_chunk)
 
-            if stream_inferface:
-                if isinstance(stream_inferface, AgentChunkStreamingInterface):
-                    stream_inferface.process_chunk(
+            if stream_interface:
+                if isinstance(stream_interface, AgentChunkStreamingInterface):
+                    stream_interface.process_chunk(
                         chat_completion_chunk,
                         message_id=chat_completion_response.id if create_message_id else chat_completion_chunk.id,
                         message_date=chat_completion_response.created if create_message_datetime else chat_completion_chunk.created,
                     )
-                elif isinstance(stream_inferface, AgentRefreshStreamingInterface):
-                    stream_inferface.process_refresh(chat_completion_response)
+                elif isinstance(stream_interface, AgentRefreshStreamingInterface):
+                    stream_interface.process_refresh(chat_completion_response)
                 else:
-                    raise TypeError(stream_inferface)
+                    raise TypeError(stream_interface)
 
             if chunk_idx == 0:
                 # initialize the choice objects which we will increment with the deltas
@@ -314,13 +318,13 @@ def openai_chat_completions_process_stream(
             n_chunks += 1
 
     except Exception as e:
-        if stream_inferface:
-            stream_inferface.stream_end()
+        if stream_interface:
+            stream_interface.stream_end()
         print(f"Parsing ChatCompletion stream failed with error:\n{str(e)}")
         raise e
     finally:
-        if stream_inferface:
-            stream_inferface.stream_end()
+        if stream_interface:
+            stream_interface.stream_end()
 
     # make sure we didn't leave temp stuff in
     assert all([c.finish_reason != TEMP_STREAM_FINISH_REASON for c in chat_completion_response.choices])

letta/local_llm/chat_completion_proxy.py

@@ -85,9 +85,7 @@ def get_chat_completion(
     elif wrapper is None:
         # Warn the user that we're using the fallback
         if not has_shown_warning:
-            print(
-                f"{CLI_WARNING_PREFIX}no wrapper specified for local LLM, using the default wrapper (you can remove this warning by specifying the wrapper with --model-wrapper)"
-            )
+            print(f"{CLI_WARNING_PREFIX}no prompt formatter specified for local LLM, using the default formatter")
             has_shown_warning = True
 
         llm_wrapper = DEFAULT_WRAPPER()

letta/local_llm/vllm/api.py

@@ -3,7 +3,7 @@ from urllib.parse import urljoin
 from letta.local_llm.settings.settings import get_completions_settings
 from letta.local_llm.utils import count_tokens, post_json_auth_request
 
-WEBUI_API_SUFFIX = "/v1/completions"
+WEBUI_API_SUFFIX = "/completions"
 
 
 def get_vllm_completion(endpoint, auth_type, auth_key, model, prompt, context_window, user, grammar=None):

letta/metadata.py

@@ -14,11 +14,10 @@ from sqlalchemy import (
     String,
     TypeDecorator,
     desc,
-    func,
 )
-from sqlalchemy.orm import declarative_base
 from sqlalchemy.sql import func
 
+from letta.base import Base
 from letta.config import LettaConfig
 from letta.schemas.agent import AgentState
 from letta.schemas.api_key import APIKey
@@ -28,6 +27,8 @@ from letta.schemas.enums import JobStatus
 from letta.schemas.job import Job
 from letta.schemas.llm_config import LLMConfig
 from letta.schemas.memory import Memory
+
+# from letta.schemas.message import Message, Passage, Record, RecordType, ToolCall
 from letta.schemas.openai.chat_completions import ToolCall, ToolCallFunction
 from letta.schemas.organization import Organization
 from letta.schemas.source import Source
@@ -36,8 +37,6 @@ from letta.schemas.user import User
 from letta.settings import settings
 from letta.utils import enforce_types, get_utc_time, printd
 
-Base = declarative_base()
-
 
 class LLMConfigColumn(TypeDecorator):
     """Custom type for storing LLMConfig as JSON"""

letta/providers.py

@@ -14,14 +14,18 @@ from letta.schemas.llm_config import LLMConfig
 
 class Provider(BaseModel):
 
-    def list_llm_models(self):
+    def list_llm_models(self) -> List[LLMConfig]:
         return []
 
-    def list_embedding_models(self):
+    def list_embedding_models(self) -> List[EmbeddingConfig]:
         return []
 
-    def get_model_context_window(self, model_name: str):
-        pass
+    def get_model_context_window(self, model_name: str) -> Optional[int]:
+        raise NotImplementedError
+
+    def provider_tag(self) -> str:
+        """String representation of the provider for display purposes"""
+        raise NotImplementedError
 
 
 class LettaProvider(Provider):
@@ -136,9 +140,17 @@ class AnthropicProvider(Provider):
 
 
 class OllamaProvider(OpenAIProvider):
+    """Ollama provider that uses the native /api/generate endpoint
+
+    See: https://github.com/ollama/ollama/blob/main/docs/api.md#generate-a-completion
+    """
+
     name: str = "ollama"
     base_url: str = Field(..., description="Base URL for the Ollama API.")
     api_key: Optional[str] = Field(None, description="API key for the Ollama API (default: `None`).")
+    default_prompt_formatter: str = Field(
+        ..., description="Default prompt formatter (aka model wrapper) to use on a /completions style API."
+    )
 
     def list_llm_models(self) -> List[LLMConfig]:
         # https://github.com/ollama/ollama/blob/main/docs/api.md#list-local-models
@@ -152,17 +164,21 @@ class OllamaProvider(OpenAIProvider):
         configs = []
         for model in response_json["models"]:
             context_window = self.get_model_context_window(model["name"])
+            if context_window is None:
+                print(f"Ollama model {model['name']} has no context window")
+                continue
             configs.append(
                 LLMConfig(
                     model=model["name"],
                     model_endpoint_type="ollama",
                     model_endpoint=self.base_url,
+                    model_wrapper=self.default_prompt_formatter,
                     context_window=context_window,
                 )
             )
         return configs
 
-    def get_model_context_window(self, model_name: str):
+    def get_model_context_window(self, model_name: str) -> Optional[int]:
 
         import requests
 
@@ -188,6 +204,10 @@ class OllamaProvider(OpenAIProvider):
         # ]
         # max_position_embeddings
         # parse model cards: nous, dolphon, llama
+        if "model_info" not in response_json:
+            if "error" in response_json:
+                print(f"Ollama fetch model info error for {model_name}: {response_json['error']}")
+            return None
         for key, value in response_json["model_info"].items():
             if "context_length" in key:
                 return value
@@ -198,6 +218,10 @@ class OllamaProvider(OpenAIProvider):
 
         response = requests.post(f"{self.base_url}/api/show", json={"name": model_name, "verbose": True})
         response_json = response.json()
+        if "model_info" not in response_json:
+            if "error" in response_json:
+                print(f"Ollama fetch model info error for {model_name}: {response_json['error']}")
+            return None
         for key, value in response_json["model_info"].items():
             if "embedding_length" in key:
                 return value
@@ -216,6 +240,7 @@ class OllamaProvider(OpenAIProvider):
         for model in response_json["models"]:
             embedding_dim = self.get_model_embedding_dim(model["name"])
             if not embedding_dim:
+                print(f"Ollama model {model['name']} has no embedding dimension")
                 continue
             configs.append(
                 EmbeddingConfig(
@@ -310,7 +335,7 @@ class GoogleAIProvider(Provider):
             )
         return configs
 
-    def get_model_context_window(self, model_name: str):
+    def get_model_context_window(self, model_name: str) -> Optional[int]:
         from letta.llm_api.google_ai import google_ai_get_model_context_window
 
         return google_ai_get_model_context_window(self.base_url, self.api_key, model_name)
@@ -371,16 +396,75 @@ class AzureProvider(Provider):
             )
         return configs
 
-    def get_model_context_window(self, model_name: str):
+    def get_model_context_window(self, model_name: str) -> Optional[int]:
         """
         This is hardcoded for now, since there is no API endpoints to retrieve metadata for a model.
         """
         return AZURE_MODEL_TO_CONTEXT_LENGTH.get(model_name, 4096)
 
 
-class VLLMProvider(OpenAIProvider):
+class VLLMChatCompletionsProvider(Provider):
+    """vLLM provider that treats vLLM as an OpenAI /chat/completions proxy"""
+
     # NOTE: vLLM only serves one model at a time (so could configure that through env variables)
-    pass
+    name: str = "vllm"
+    base_url: str = Field(..., description="Base URL for the vLLM API.")
+
+    def list_llm_models(self) -> List[LLMConfig]:
+        # not supported with vLLM
+        from letta.llm_api.openai import openai_get_model_list
+
+        assert self.base_url, "base_url is required for vLLM provider"
+        response = openai_get_model_list(self.base_url, api_key=None)
+
+        configs = []
+        print(response)
+        for model in response["data"]:
+            configs.append(
+                LLMConfig(
+                    model=model["id"],
+                    model_endpoint_type="openai",
+                    model_endpoint=self.base_url,
+                    context_window=model["max_model_len"],
+                )
+            )
+        return configs
+
+    def list_embedding_models(self) -> List[EmbeddingConfig]:
+        # not supported with vLLM
+        return []
+
+
+class VLLMCompletionsProvider(Provider):
+    """This uses /completions API as the backend, not /chat/completions, so we need to specify a model wrapper"""
+
+    # NOTE: vLLM only serves one model at a time (so could configure that through env variables)
+    name: str = "vllm"
+    base_url: str = Field(..., description="Base URL for the vLLM API.")
+    default_prompt_formatter: str = Field(..., description="Default prompt formatter (aka model wrapper) to use on vLLM /completions API.")
+
+    def list_llm_models(self) -> List[LLMConfig]:
+        # not supported with vLLM
+        from letta.llm_api.openai import openai_get_model_list
+
+        response = openai_get_model_list(self.base_url, api_key=None)
+
+        configs = []
+        for model in response["data"]:
+            configs.append(
+                LLMConfig(
+                    model=model["id"],
+                    model_endpoint_type="vllm",
+                    model_endpoint=self.base_url,
+                    model_wrapper=self.default_prompt_formatter,
+                    context_window=model["max_model_len"],
+                )
+            )
+        return configs
+
+    def list_embedding_models(self) -> List[EmbeddingConfig]:
+        # not supported with vLLM
+        return []
 
 
 class CohereProvider(OpenAIProvider):

letta/schemas/embedding_config.py

@@ -52,3 +52,10 @@ class EmbeddingConfig(BaseModel):
             )
         else:
             raise ValueError(f"Model {model_name} not supported.")
+
+    def pretty_print(self) -> str:
+        return (
+            f"{self.embedding_model}"
+            + (f" [type={self.embedding_endpoint_type}]" if self.embedding_endpoint_type else "")
+            + (f" [ip={self.embedding_endpoint}]" if self.embedding_endpoint else "")
+        )

letta/schemas/llm_config.py

@@ -68,3 +68,10 @@ class LLMConfig(BaseModel):
             )
         else:
             raise ValueError(f"Model {model_name} not supported.")
+
+    def pretty_print(self) -> str:
+        return (
+            f"{self.model}"
+            + (f" [type={self.model_endpoint_type}]" if self.model_endpoint_type else "")
+            + (f" [ip={self.model_endpoint}]" if self.model_endpoint else "")
+        )

letta/server/server.py

@@ -14,8 +14,8 @@ import letta.constants as constants
 import letta.server.utils as server_utils
 import letta.system as system
 from letta.agent import Agent, save_agent
+from letta.agent_store.db import attach_base
 from letta.agent_store.storage import StorageConnector, TableType
-from letta.config import LettaConfig
 from letta.credentials import LettaCredentials
 from letta.data_sources.connectors import DataConnector, load_data
 
@@ -41,7 +41,7 @@ from letta.interface import AgentInterface  # abstract
 from letta.interface import CLIInterface  # for printing to terminal
 from letta.log import get_logger
 from letta.memory import get_memory_functions
-from letta.metadata import MetadataStore
+from letta.metadata import Base, MetadataStore
 from letta.prompts import gpt_system
 from letta.providers import (
     AnthropicProvider,
@@ -51,7 +51,8 @@ from letta.providers import (
     OllamaProvider,
     OpenAIProvider,
     Provider,
-    VLLMProvider,
+    VLLMChatCompletionsProvider,
+    VLLMCompletionsProvider,
 )
 from letta.schemas.agent import AgentState, AgentType, CreateAgent, UpdateAgentState
 from letta.schemas.api_key import APIKey, APIKeyCreate
@@ -150,23 +151,11 @@ class Server(object):
 
 
 from sqlalchemy import create_engine
-from sqlalchemy.orm import declarative_base, sessionmaker
+from sqlalchemy.orm import sessionmaker
 
-from letta.agent_store.db import MessageModel, PassageModel
 from letta.config import LettaConfig
 
 # NOTE: hack to see if single session management works
-from letta.metadata import (
-    AgentModel,
-    AgentSourceMappingModel,
-    APIKeyModel,
-    BlockModel,
-    JobModel,
-    OrganizationModel,
-    SourceModel,
-    ToolModel,
-    UserModel,
-)
 from letta.settings import model_settings, settings
 
 config = LettaConfig.load()
@@ -183,24 +172,12 @@ else:
     # TODO: don't rely on config storage
     engine = create_engine("sqlite:///" + os.path.join(config.recall_storage_path, "sqlite.db"))
 
-Base = declarative_base()
+
 SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
-Base.metadata.create_all(
-    engine,
-    tables=[
-        UserModel.__table__,
-        AgentModel.__table__,
-        SourceModel.__table__,
-        AgentSourceMappingModel.__table__,
-        APIKeyModel.__table__,
-        BlockModel.__table__,
-        ToolModel.__table__,
-        JobModel.__table__,
-        PassageModel.__table__,
-        MessageModel.__table__,
-        OrganizationModel.__table__,
-    ],
-)
+
+attach_base()
+
+Base.metadata.create_all(bind=engine)
 
 
 # Dependency
@@ -223,7 +200,7 @@ class SyncServer(Server):
     def __init__(
         self,
         chaining: bool = True,
-        max_chaining_steps: bool = None,
+        max_chaining_steps: Optional[bool] = None,
         default_interface_factory: Callable[[], AgentInterface] = lambda: CLIInterface(),
         # default_interface: AgentInterface = CLIInterface(),
         # default_persistence_manager_cls: PersistenceManager = LocalStateManager,
@@ -264,16 +241,34 @@ class SyncServer(Server):
         # collect providers (always has Letta as a default)
         self._enabled_providers: List[Provider] = [LettaProvider()]
         if model_settings.openai_api_key:
-            self._enabled_providers.append(OpenAIProvider(api_key=model_settings.openai_api_key, base_url=model_settings.openai_api_base))
+            self._enabled_providers.append(
+                OpenAIProvider(
+                    api_key=model_settings.openai_api_key,
+                    base_url=model_settings.openai_api_base,
+                )
+            )
         if model_settings.anthropic_api_key:
-            self._enabled_providers.append(AnthropicProvider(api_key=model_settings.anthropic_api_key))
+            self._enabled_providers.append(
+                AnthropicProvider(
+                    api_key=model_settings.anthropic_api_key,
+                )
+            )
         if model_settings.ollama_base_url:
-            self._enabled_providers.append(OllamaProvider(base_url=model_settings.ollama_base_url))
-        if model_settings.vllm_base_url:
-            self._enabled_providers.append(VLLMProvider(base_url=model_settings.vllm_base_url))
+            self._enabled_providers.append(
+                OllamaProvider(
+                    base_url=model_settings.ollama_base_url,
+                    api_key=None,
+                    default_prompt_formatter=model_settings.default_prompt_formatter,
+                )
+            )
         if model_settings.gemini_api_key:
-            self._enabled_providers.append(GoogleAIProvider(api_key=model_settings.gemini_api_key))
+            self._enabled_providers.append(
+                GoogleAIProvider(
+                    api_key=model_settings.gemini_api_key,
+                )
+            )
         if model_settings.azure_api_key and model_settings.azure_base_url:
+            assert model_settings.azure_api_version, "AZURE_API_VERSION is required"
             self._enabled_providers.append(
                 AzureProvider(
                     api_key=model_settings.azure_api_key,
@@ -281,6 +276,22 @@ class SyncServer(Server):
                     api_version=model_settings.azure_api_version,
                 )
             )
+        if model_settings.vllm_api_base:
+            # vLLM exposes both a /chat/completions and a /completions endpoint
+            self._enabled_providers.append(
+                VLLMCompletionsProvider(
+                    base_url=model_settings.vllm_api_base,
+                    default_prompt_formatter=model_settings.default_prompt_formatter,
+                )
+            )
+            # NOTE: to use the /chat/completions endpoint, you need to specify extra flags on vLLM startup
+            # see: https://docs.vllm.ai/en/latest/getting_started/examples/openai_chat_completion_client_with_tools.html
+            # e.g. "... --enable-auto-tool-choice --tool-call-parser hermes"
+            self._enabled_providers.append(
+                VLLMChatCompletionsProvider(
+                    base_url=model_settings.vllm_api_base,
+                )
+            )
 
     def save_agents(self):
         """Saves all the agents that are in the in-memory object store"""

letta/settings.py

@@ -4,14 +4,20 @@ from typing import Optional
 from pydantic import Field
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
+from letta.local_llm.constants import DEFAULT_WRAPPER_NAME
+
 
 class ModelSettings(BaseSettings):
 
     # env_prefix='my_prefix_'
 
+    # when we use /completions APIs (instead of /chat/completions), we need to specify a model wrapper
+    # the "model wrapper" is responsible for prompt formatting and function calling parsing
+    default_prompt_formatter: str = DEFAULT_WRAPPER_NAME
+
     # openai
     openai_api_key: Optional[str] = None
-    openai_api_base: Optional[str] = "https://api.openai.com/v1"
+    openai_api_base: str = "https://api.openai.com/v1"
 
     # groq
     groq_api_key: Optional[str] = None
@@ -25,13 +31,16 @@ class ModelSettings(BaseSettings):
     # azure
     azure_api_key: Optional[str] = None
     azure_base_url: Optional[str] = None
-    azure_api_version: Optional[str] = None
+    # We provide a default here, since usually people will want to be on the latest API version.
+    azure_api_version: Optional[str] = (
+        "2024-09-01-preview"  # https://learn.microsoft.com/en-us/azure/ai-services/openai/api-version-deprecation
+    )
 
     # google ai
     gemini_api_key: Optional[str] = None
 
     # vLLM
-    vllm_base_url: Optional[str] = None
+    vllm_api_base: Optional[str] = None
 
     # openllm
     openllm_auth_type: Optional[str] = None

{letta_nightly-0.4.1.dev20241011104054.dist-info → letta_nightly-0.4.1.dev20241013104006.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: letta-nightly
-Version: 0.4.1.dev20241011104054
+Version: 0.4.1.dev20241013104006
 Summary: Create LLM agents with long-term memory and custom tools
 License: Apache License
 Author: Letta Team
@@ -20,6 +20,7 @@ Provides-Extra: postgres
 Provides-Extra: qdrant
 Provides-Extra: server
 Provides-Extra: tests
+Requires-Dist: alembic (>=1.13.3,<2.0.0)
 Requires-Dist: autoflake (>=2.3.0,<3.0.0) ; extra == "dev"
 Requires-Dist: black[jupyter] (>=24.2.0,<25.0.0) ; extra == "dev"
 Requires-Dist: chromadb (>=0.4.24,<0.5.0)

{letta_nightly-0.4.1.dev20241011104054.dist-info → letta_nightly-0.4.1.dev20241013104006.dist-info}/RECORD

@@ -1,15 +1,16 @@
 letta/__init__.py,sha256=btKRPdyhkpIyHlCPRLwwz-SCSVcEGNBeheYA-XNesiI,996
 letta/__main__.py,sha256=6Hs2PV7EYc5Tid4g4OtcLXhqVHiNYTGzSBdoOnW2HXA,29
-letta/agent.py,sha256=2yYk8H76EYF8-EF5eFqv8dp8avaBcV_hfXOQcwYuAwU,66559
+letta/agent.py,sha256=pdsd1M_AF3uF-AgJrMmxj8uB0Rkt5AXKBxHNw9mMaP4,66559
 letta/agent_store/chroma.py,sha256=upR5zGnGs6I6btulEYbiZdGG87BgKjxUJOQZ4Y-RQ_M,12492
-letta/agent_store/db.py,sha256=Hbw4HsxPZSQL9h_3I-SiJRisaYdzk1XSdYJFarjrUtM,22575
+letta/agent_store/db.py,sha256=TAQ1rcqS8zuC1hT_BaiXn6tSz5aKHAb-YQxr4QLKgz4,22724
 letta/agent_store/lancedb.py,sha256=8RWmqVjowm5g0cc6DNRcb6f1FHGEqFnccnuekhWY39U,5101
 letta/agent_store/milvus.py,sha256=VxEKz9XR7_3QTY59K_38NtJCCQvi41rhHoFibfzW7yw,8469
 letta/agent_store/qdrant.py,sha256=qIEJhXJb6GzcT4wp8iV5Ox5W1CFMvcPViTI4HLSh59E,7879
 letta/agent_store/storage.py,sha256=QWrPdIEJCnsPg1xnPrG1xbOXmbjpz37ZNhvuH52M7A8,6642
+letta/base.py,sha256=Ba-wt8p59bLmeUONkYSo5MhrkH-_HdT4zE1Y9MVGrSQ,83
 letta/benchmark/benchmark.py,sha256=ebvnwfp3yezaXOQyGXkYCDYpsmre-b9hvNtnyx4xkG0,3701
 letta/benchmark/constants.py,sha256=aXc5gdpMGJT327VuxsT5FngbCK2J41PQYeICBO7g_RE,536
-letta/cli/cli.py,sha256=1r_UlOonkrBXd6SW3KMob6uy4iUnxVs1jQmfKK-ylZw,16912
+letta/cli/cli.py,sha256=XZDF7EfBYqdYEyDbENVp-tTyGcdvd9Wa1PNk70D-MGs,16195
 letta/cli/cli_config.py,sha256=G7QqPNTtlQ4TdrXZrrFFGblZEhnkyrqN1Cl5z415C-g,8689
 letta/cli/cli_load.py,sha256=aVlGWiNEUs_eG793HLl7cES-dEIuA1CJfZpT1Cm8Uo4,4591
 letta/client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -35,16 +36,16 @@ letta/humans/examples/cs_phd.txt,sha256=9C9ZAV_VuG7GB31ksy3-_NAyk8rjE6YtVOkhp08k
 letta/interface.py,sha256=QI4hFP0WrNsgM5qX6TbnhH1ZZxsLYr5DaccuxpEQ8S4,12768
 letta/llm_api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 letta/llm_api/anthropic.py,sha256=bAb9PVrpYjo2QN51_SJbW7Vry2_Sf55B05UoruHXb7A,12932
-letta/llm_api/azure_openai.py,sha256=8uBQIYa3WpRcxLXgJTYZrl-GhDGflqxXraRLZlpkdh4,4566
+letta/llm_api/azure_openai.py,sha256=C-fuuholudcLJDWjqnXJwpXsfmGWfNugEVWyj6YCrpg,4572
 letta/llm_api/azure_openai_constants.py,sha256=oXtKrgBFHf744gyt5l1thILXgyi8NDNUrKEa2GGGpjw,278
 letta/llm_api/cohere.py,sha256=vDRd-SUGp1t_JUIdwC3RkIhwMl0OY7n-tAU9uPORYkY,14826
 letta/llm_api/google_ai.py,sha256=3xZ074nSOCC22c15yerA5ngWzh0ex4wxeI-6faNbHPE,17708
-letta/llm_api/helpers.py,sha256=Qe1YC36QjjOKE-Xh1Ss3dhMNcWergOK_MpG9xdtN9CM,9519
-letta/llm_api/llm_api_tools.py,sha256=8HndYHAH6ENL5vFEYn2px6CjZdx3ttCGxbEtRfK2RAY,15237
-letta/llm_api/openai.py,sha256=gqFIyy7nc254RJddbTs6CXfB_bmABote18KZOePuOdI,21582
+letta/llm_api/helpers.py,sha256=LjUtCjvPzSP3-3Ak0J--2RqUXOO6Of8287mm1L1LAMU,9549
+letta/llm_api/llm_api_tools.py,sha256=Z3eiYUtvZKBVBcmKI2l4qWkKM4hgvLN9Y1aSxXc7y-k,15344
+letta/llm_api/openai.py,sha256=EXpktSI_TYjsCDEXBxdNXsY5uE9Rzb7BPF1F6cz8bkg,21689
 letta/local_llm/README.md,sha256=hFJyw5B0TU2jrh9nb0zGZMgdH-Ei1dSRfhvPQG_NSoU,168
 letta/local_llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-letta/local_llm/chat_completion_proxy.py,sha256=PXgNveahts5DbZ7GVcPShxmrDKropL81PY2JHc31yAA,13091
+letta/local_llm/chat_completion_proxy.py,sha256=SiohxsjGTku4vOryOZx7I0t0xoO_sUuhXgoe62fKq3c,12995
 letta/local_llm/constants.py,sha256=GIu0184EIiOLEqGeupLUYQvkgT_imIjLg3T-KM9TcFM,1125
 letta/local_llm/function_parser.py,sha256=BlNsGo1VzyfY5KdF_RwjRQNOWIsaudo7o37u1W5eg0s,2626
 letta/local_llm/grammars/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -74,7 +75,7 @@ letta/local_llm/settings/deterministic_mirostat.py,sha256=kgRikcxYHfIbPFydHW6W7I
 letta/local_llm/settings/settings.py,sha256=ZAbzDpu2WsBXjVGXJ-TKUpS99VTI__3EoZml9KqYef0,2971
 letta/local_llm/settings/simple.py,sha256=HAO2jBJ_hJCEsXWIJcD0sckR0tI0zs3x2CPdf6ORQLs,719
 letta/local_llm/utils.py,sha256=DTv8S0lIpFBZlY6fjprtAU3n06eOPDKT6EXwnOgexLo,11363
-letta/local_llm/vllm/api.py,sha256=KhIB-cSYGKKsq5ZVarxPDAL9fTL8HT2-WmWS5gnY6QQ,2593
+letta/local_llm/vllm/api.py,sha256=2kAGZjc_GH9ILJnVRq-45yfsfKELVfbC9VEl_cIC6vg,2590
 letta/local_llm/webui/api.py,sha256=kkxncdCFq1vjgvaHOoQ__j7rcDPgC1F64KcEm94Y6Rs,2639
 letta/local_llm/webui/legacy_api.py,sha256=k3H3y4qp2Fs-XmP24iSIEyvq6wjWFWBzklY3-wRAJNI,2335
 letta/local_llm/webui/legacy_settings.py,sha256=BLmd3TSx5StnY3ibjwaxYATPt_Lvq-o1rlcc_-Q1JcU,538
@@ -82,7 +83,7 @@ letta/local_llm/webui/settings.py,sha256=gmLHfiOl1u4JmlAZU2d2O8YKF9lafdakyjwR_ft
 letta/log.py,sha256=QHquDnL7oUAvdKlAwUlCK9zXKDMUjrU9WA0bxnMsP0Y,2101
 letta/main.py,sha256=7uuiFv8TeOd3K92WwfYkls7l7KXJ5P0l2zgK9AfmD08,18502
 letta/memory.py,sha256=6q1x3-PY-PeXzAt6hvP-UF1ajvroPZ7XW-5nLy-JhMo,17657
-letta/metadata.py,sha256=-LUv7w5yaCn10wh7vnFCrhH4s5M5F0K3Ui7pClXN3tE,33225
+letta/metadata.py,sha256=50pEJ5BuNKT8dPv34-H_Id9an3ql45Ze7BKqn7DGQ6s,33256
 letta/openai_backcompat/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 letta/openai_backcompat/openai_object.py,sha256=Y1ZS1sATP60qxJiOsjOP3NbwSzuzvkNAvb3DeuhM5Uk,13490
 letta/persistence_manager.py,sha256=LlLgEDpSafCPAiyKmuq0NvVAnfBkZo6TWbGIKYQjQBs,5200
@@ -106,13 +107,13 @@ letta/prompts/system/memgpt_doc.txt,sha256=AsT55NOORoH-K-p0fxklrDRZ3qHs4MIKMuR-M
 letta/prompts/system/memgpt_gpt35_extralong.txt,sha256=FheNhYoIzNz6qnJKhVquZVSMj3HduC48reFaX7Pf7ig,5046
 letta/prompts/system/memgpt_intuitive_knowledge.txt,sha256=sA7c3urYqREVnSBI81nTGImXAekqC0Fxc7RojFqud1g,2966
 letta/prompts/system/memgpt_modified_chat.txt,sha256=HOaPVurEftD8KsuwsclDgE2afIfklMjxhuSO96q1-6I,4656
-letta/providers.py,sha256=UTgD9VhH0CEszW7sTMvfTnHOpDCd1m1RmgoI6Pr3Vy0,14444
+letta/providers.py,sha256=e4jWshGMu4UQ9B0yEzcHP1bDxNHEo_mgpO5M0txuOIo,17978
 letta/pytest.ini,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 letta/schemas/agent.py,sha256=ztnUqdhY9V3g0jsbTjF1ypKPC1tZx4QVFaRuLAOXNSA,6230
 letta/schemas/api_key.py,sha256=u07yzzMn-hBAHZIIKbWY16KsgiFjSNR8lAghpMUo3_4,682
 letta/schemas/block.py,sha256=1_GwFtfykloYU4Mp2DV3-DqkvsKo79Mu3LAzVYOgMzk,3998
 letta/schemas/document.py,sha256=JpvU0YkvOVLvHaDNcg-ihFzpeHC2zqsWBgyJ6zHnfNw,745
-letta/schemas/embedding_config.py,sha256=KhEOStOD8VbHFyLI9pkQVbTG1Av2g-Ql0yf9M868LME,2570
+letta/schemas/embedding_config.py,sha256=1kD6NpiXeH4roVumxqDAKk7xt8SpXGWNhZs_XXUSlEU,2855
 letta/schemas/enums.py,sha256=WfRYpLh_pD-VyhEnp3Y6pPfx063zq2o4jky6PulqO8w,629
 letta/schemas/health.py,sha256=zT6mYovvD17iJRuu2rcaQQzbEEYrkwvAE9TB7iU824c,139
 letta/schemas/job.py,sha256=bYDrjbJm2B4LUTSkLUdQR_HDhL2E23g0EHf7E23ezYU,1547
@@ -120,7 +121,7 @@ letta/schemas/letta_base.py,sha256=4QXFgyjCHqIagi8B6_4nmqb9eoJ52Y6aCxBxQpGX48M,2
 letta/schemas/letta_message.py,sha256=Slgxa59qZfdvqXuCVHOt03u-7JL456ZY-WLaK5UYYKU,6234
 letta/schemas/letta_request.py,sha256=Fv4hZKLMefCISwxuElXB0vsGDjqnzZuCbCbHPUPTirQ,1852
 letta/schemas/letta_response.py,sha256=_UJoO3UtC3F5DtQCHzdiGM1SHNPYPKvopIWqg8t5YZw,1564
-letta/schemas/llm_config.py,sha256=oC-RTimpLhNmn80AyCLnx8YUwA0sIR1UqshYxHVbamU,2671
+letta/schemas/llm_config.py,sha256=xwHBO_sY-Kahtwxt_0Qz4j2CRSeJ0pGAIxpqyK_2Exk,2930
 letta/schemas/memory.py,sha256=mDJbe9mzTw0HSn6tiouC3z32r-8Fc_EaeqxDy_hXf0U,9327
 letta/schemas/message.py,sha256=1C6lz4yvMTusE8zORKbi_BKnb_X_-RuUvdpM5YeZ21o,33409
 letta/schemas/openai/chat_completion_request.py,sha256=Fa7xnSnG7WUQounJhnDu0fTSxoR6xOAh2bODuqmfypI,3345
@@ -166,7 +167,7 @@ letta/server/rest_api/routers/v1/tools.py,sha256=MEhxu-zMS2ff_wwcRpMuQyWA71w_3BJ
 letta/server/rest_api/routers/v1/users.py,sha256=Y2rDvHOG1B5FLSOjutY3R22vt48IngbZ-9h8CohG5rc,3378
 letta/server/rest_api/static_files.py,sha256=NG8sN4Z5EJ8JVQdj19tkFa9iQ1kBPTab9f_CUxd_u4Q,3143
 letta/server/rest_api/utils.py,sha256=Fc2ZGKzLaBa2sEtSTVjJ8D5M0xIwsWC0CVAOIJaD3rY,2176
-letta/server/server.py,sha256=Vjc6-GdvU5n-VmvFm9rxvD_ahBsz1BYFSbDxOQ9g1JI,82041
+letta/server/server.py,sha256=txNgf3AIraK-kTV4PLwzROfgfBTIul_Y74hPygFtOHw,82687
 letta/server/startup.sh,sha256=jeGV7B_PS0hS-tT6o6GpACrUbV9WV1NI2L9aLoUDDtc,311
 letta/server/static_files/assets/index-3ab03d5b.css,sha256=OrA9W4iKJ5h2Wlr7GwdAT4wow0CM8hVit1yOxEL49Qw,54295
 letta/server/static_files/assets/index-9a9c449b.js,sha256=qoWUq6_kuLhE9NFkNeCBptgq-oERW46r0tB3JlWe_qc,1818951
@@ -179,12 +180,12 @@ letta/server/ws_api/example_client.py,sha256=95AA5UFgTlNJ0FUQkLxli8dKNx48MNm3eWG
 letta/server/ws_api/interface.py,sha256=TWl9vkcMCnLsUtgsuENZ-ku2oMDA-OUTzLh_yNRoMa4,4120
 letta/server/ws_api/protocol.py,sha256=M_-gM5iuDBwa1cuN2IGNCG5GxMJwU2d3XW93XALv9s8,1821
 letta/server/ws_api/server.py,sha256=C2Kv48PCwl46DQFb0ZP30s86KJLQ6dZk2AhWQEZn9pY,6004
-letta/settings.py,sha256=nRtdDiCV-ffIPobpTQMiM50fHRWy0DoX3D1VveaSo_0,2701
+letta/settings.py,sha256=6VWC3vtTa8vqj6dqos4p_xHTMJNJS_8LRGJmqvaU1-o,3219
 letta/streaming_interface.py,sha256=_FPUWy58j50evHcpXyd7zB1wWqeCc71NCFeWh_TBvnw,15736
 letta/system.py,sha256=buKYPqG5n2x41hVmWpu6JUpyd7vTWED9Km2_M7dLrvk,6960
 letta/utils.py,sha256=neUs7mxNfndzRL5XUxerr8Lic6w7qnyyvf8FBwMnyWw,30852
-letta_nightly-0.4.1.dev20241011104054.dist-info/LICENSE,sha256=mExtuZ_GYJgDEI38GWdiEYZizZS4KkVt2SF1g_GPNhI,10759
-letta_nightly-0.4.1.dev20241011104054.dist-info/METADATA,sha256=fqSS3tY0N8sjslAt2u0MFQv8DPLhVKymVktdjs__DL0,5967
-letta_nightly-0.4.1.dev20241011104054.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-letta_nightly-0.4.1.dev20241011104054.dist-info/entry_points.txt,sha256=2zdiyGNEZGV5oYBuS-y2nAAgjDgcC9yM_mHJBFSRt5U,40
-letta_nightly-0.4.1.dev20241011104054.dist-info/RECORD,,
+letta_nightly-0.4.1.dev20241013104006.dist-info/LICENSE,sha256=mExtuZ_GYJgDEI38GWdiEYZizZS4KkVt2SF1g_GPNhI,10759
+letta_nightly-0.4.1.dev20241013104006.dist-info/METADATA,sha256=L5hOlFIaGT5BnE945CxaUdQEoKsOLrysrrL_bD9MjVU,6008
+letta_nightly-0.4.1.dev20241013104006.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+letta_nightly-0.4.1.dev20241013104006.dist-info/entry_points.txt,sha256=2zdiyGNEZGV5oYBuS-y2nAAgjDgcC9yM_mHJBFSRt5U,40
+letta_nightly-0.4.1.dev20241013104006.dist-info/RECORD,,