letta-nightly 0.7.20.dev20250521104258__py3-none-any.whl → 0.7.21.dev20250522104246__py3-none-any.whl

letta/schemas/providers.py

@@ -47,12 +47,21 @@ class Provider(ProviderBase):
     def list_llm_models(self) -> List[LLMConfig]:
         return []
 
+    async def list_llm_models_async(self) -> List[LLMConfig]:
+        return []
+
     def list_embedding_models(self) -> List[EmbeddingConfig]:
         return []
 
+    async def list_embedding_models_async(self) -> List[EmbeddingConfig]:
+        return []
+
     def get_model_context_window(self, model_name: str) -> Optional[int]:
         raise NotImplementedError
 
+    async def get_model_context_window_async(self, model_name: str) -> Optional[int]:
+        raise NotImplementedError
+
     def provider_tag(self) -> str:
         """String representation of the provider for display purposes"""
         raise NotImplementedError
@@ -140,6 +149,19 @@ class LettaProvider(Provider):
             )
         ]
 
+    async def list_llm_models_async(self) -> List[LLMConfig]:
+        return [
+            LLMConfig(
+                model="letta-free",  # NOTE: renamed
+                model_endpoint_type="openai",
+                model_endpoint=LETTA_MODEL_ENDPOINT,
+                context_window=8192,
+                handle=self.get_handle("letta-free"),
+                provider_name=self.name,
+                provider_category=self.provider_category,
+            )
+        ]
+
     def list_embedding_models(self):
         return [
             EmbeddingConfig(
@@ -189,9 +211,40 @@ class OpenAIProvider(Provider):
 
         return data
 
+    async def _get_models_async(self) -> List[dict]:
+        from letta.llm_api.openai import openai_get_model_list_async
+
+        # Some hardcoded support for OpenRouter (so that we only get models with tool calling support)...
+        # See: https://openrouter.ai/docs/requests
+        extra_params = {"supported_parameters": "tools"} if "openrouter.ai" in self.base_url else None
+
+        # Similar to Nebius
+        extra_params = {"verbose": True} if "nebius.com" in self.base_url else None
+
+        response = await openai_get_model_list_async(
+            self.base_url,
+            api_key=self.api_key,
+            extra_params=extra_params,
+            # fix_url=True,  # NOTE: make sure together ends with /v1
+        )
+
+        if "data" in response:
+            data = response["data"]
+        else:
+            # TogetherAI's response is missing the 'data' field
+            data = response
+
+        return data
+
     def list_llm_models(self) -> List[LLMConfig]:
         data = self._get_models()
+        return self._list_llm_models(data)
 
+    async def list_llm_models_async(self) -> List[LLMConfig]:
+        data = await self._get_models_async()
+        return self._list_llm_models(data)
+
+    def _list_llm_models(self, data) -> List[LLMConfig]:
         configs = []
         for model in data:
             assert "id" in model, f"OpenAI model missing 'id' field: {model}"
@@ -279,7 +332,6 @@ class OpenAIProvider(Provider):
         return configs
 
     def list_embedding_models(self) -> List[EmbeddingConfig]:
-
         if self.base_url == "https://api.openai.com/v1":
             # TODO: actually automatically list models for OpenAI
             return [
@@ -312,55 +364,92 @@ class OpenAIProvider(Provider):
         else:
             # Actually attempt to list
             data = self._get_models()
+            return self._list_embedding_models(data)
 
-            configs = []
-            for model in data:
-                assert "id" in model, f"Model missing 'id' field: {model}"
-                model_name = model["id"]
+    async def list_embedding_models_async(self) -> List[EmbeddingConfig]:
+        if self.base_url == "https://api.openai.com/v1":
+            # TODO: actually automatically list models for OpenAI
+            return [
+                EmbeddingConfig(
+                    embedding_model="text-embedding-ada-002",
+                    embedding_endpoint_type="openai",
+                    embedding_endpoint=self.base_url,
+                    embedding_dim=1536,
+                    embedding_chunk_size=300,
+                    handle=self.get_handle("text-embedding-ada-002", is_embedding=True),
+                ),
+                EmbeddingConfig(
+                    embedding_model="text-embedding-3-small",
+                    embedding_endpoint_type="openai",
+                    embedding_endpoint=self.base_url,
+                    embedding_dim=2000,
+                    embedding_chunk_size=300,
+                    handle=self.get_handle("text-embedding-3-small", is_embedding=True),
+                ),
+                EmbeddingConfig(
+                    embedding_model="text-embedding-3-large",
+                    embedding_endpoint_type="openai",
+                    embedding_endpoint=self.base_url,
+                    embedding_dim=2000,
+                    embedding_chunk_size=300,
+                    handle=self.get_handle("text-embedding-3-large", is_embedding=True),
+                ),
+            ]
 
-                if "context_length" in model:
-                    # Context length is returned in Nebius as "context_length"
-                    context_window_size = model["context_length"]
-                else:
-                    context_window_size = self.get_model_context_window_size(model_name)
+        else:
+            # Actually attempt to list
+            data = await self._get_models_async()
+            return self._list_embedding_models(data)
 
-                # We need the context length for embeddings too
-                if not context_window_size:
-                    continue
+    def _list_embedding_models(self, data) -> List[EmbeddingConfig]:
+        configs = []
+        for model in data:
+            assert "id" in model, f"Model missing 'id' field: {model}"
+            model_name = model["id"]
 
-                if "nebius.com" in self.base_url:
-                    # Nebius includes the type, which we can use to filter for embedidng models
-                    try:
-                        model_type = model["architecture"]["modality"]
-                        if model_type not in ["text->embedding"]:
-                            # print(f"Skipping model w/ modality {model_type}:\n{model}")
-                            continue
-                    except KeyError:
-                        print(f"Couldn't access architecture type field, skipping model:\n{model}")
-                        continue
+            if "context_length" in model:
+                # Context length is returned in Nebius as "context_length"
+                context_window_size = model["context_length"]
+            else:
+                context_window_size = self.get_model_context_window_size(model_name)
+
+            # We need the context length for embeddings too
+            if not context_window_size:
+                continue
 
-                elif "together.ai" in self.base_url or "together.xyz" in self.base_url:
-                    # TogetherAI includes the type, which we can use to filter for embedding models
-                    if "type" in model and model["type"] not in ["embedding"]:
+            if "nebius.com" in self.base_url:
+                # Nebius includes the type, which we can use to filter for embedidng models
+                try:
+                    model_type = model["architecture"]["modality"]
+                    if model_type not in ["text->embedding"]:
                         # print(f"Skipping model w/ modality {model_type}:\n{model}")
                         continue
+                except KeyError:
+                    print(f"Couldn't access architecture type field, skipping model:\n{model}")
+                    continue
 
-                else:
-                    # For other providers we should skip by default, since we don't want to assume embeddings are supported
+            elif "together.ai" in self.base_url or "together.xyz" in self.base_url:
+                # TogetherAI includes the type, which we can use to filter for embedding models
+                if "type" in model and model["type"] not in ["embedding"]:
+                    # print(f"Skipping model w/ modality {model_type}:\n{model}")
                     continue
 
-                configs.append(
-                    EmbeddingConfig(
-                        embedding_model=model_name,
-                        embedding_endpoint_type=self.provider_type,
-                        embedding_endpoint=self.base_url,
-                        embedding_dim=context_window_size,
-                        embedding_chunk_size=DEFAULT_EMBEDDING_CHUNK_SIZE,
-                        handle=self.get_handle(model, is_embedding=True),
-                    )
+            else:
+                # For other providers we should skip by default, since we don't want to assume embeddings are supported
+                continue
+
+            configs.append(
+                EmbeddingConfig(
+                    embedding_model=model_name,
+                    embedding_endpoint_type=self.provider_type,
+                    embedding_endpoint=self.base_url,
+                    embedding_dim=context_window_size,
+                    embedding_chunk_size=DEFAULT_EMBEDDING_CHUNK_SIZE,
+                    handle=self.get_handle(model, is_embedding=True),
                 )
+            )
 
-            return configs
+        return configs
 
     def get_model_context_window_size(self, model_name: str):
         if model_name in LLM_MAX_TOKENS:
@@ -647,26 +736,19 @@ class AnthropicProvider(Provider):
         anthropic_check_valid_api_key(self.api_key)
 
     def list_llm_models(self) -> List[LLMConfig]:
-        from letta.llm_api.anthropic import MODEL_LIST, anthropic_get_model_list
+        from letta.llm_api.anthropic import anthropic_get_model_list
 
-        models = anthropic_get_model_list(self.base_url, api_key=self.api_key)
+        models = anthropic_get_model_list(api_key=self.api_key)
+        return self._list_llm_models(models)
 
-        """
-        Example response:
-        {
-          "data": [
-            {
-              "type": "model",
-              "id": "claude-3-5-sonnet-20241022",
-              "display_name": "Claude 3.5 Sonnet (New)",
-              "created_at": "2024-10-22T00:00:00Z"
-            }
-          ],
-          "has_more": true,
-          "first_id": "<string>",
-          "last_id": "<string>"
-        }
-        """
+    async def list_llm_models_async(self) -> List[LLMConfig]:
+        from letta.llm_api.anthropic import anthropic_get_model_list_async
+
+        models = await anthropic_get_model_list_async(api_key=self.api_key)
+        return self._list_llm_models(models)
+
+    def _list_llm_models(self, models) -> List[LLMConfig]:
+        from letta.llm_api.anthropic import MODEL_LIST
 
         configs = []
         for model in models:
@@ -724,9 +806,6 @@ class AnthropicProvider(Provider):
             )
         return configs
 
-    def list_embedding_models(self) -> List[EmbeddingConfig]:
-        return []
-
 
 class MistralProvider(Provider):
     provider_type: Literal[ProviderType.mistral] = Field(ProviderType.mistral, description="The type of the provider.")
@@ -948,14 +1027,24 @@ class TogetherProvider(OpenAIProvider):
     def list_llm_models(self) -> List[LLMConfig]:
         from letta.llm_api.openai import openai_get_model_list
 
-        response = openai_get_model_list(self.base_url, api_key=self.api_key)
+        models = openai_get_model_list(self.base_url, api_key=self.api_key)
+        return self._list_llm_models(models)
+
+    async def list_llm_models_async(self) -> List[LLMConfig]:
+        from letta.llm_api.openai import openai_get_model_list_async
+
+        models = await openai_get_model_list_async(self.base_url, api_key=self.api_key)
+        return self._list_llm_models(models)
+
+    def _list_llm_models(self, models) -> List[LLMConfig]:
+        pass
 
         # TogetherAI's response is missing the 'data' field
         # assert "data" in response, f"OpenAI model query response missing 'data' field: {response}"
-        if "data" in response:
-            data = response["data"]
+        if "data" in models:
+            data = models["data"]
         else:
-            data = response
+            data = models
 
         configs = []
         for model in data:
@@ -1057,7 +1146,6 @@ class GoogleAIProvider(Provider):
         from letta.llm_api.google_ai_client import google_ai_get_model_list
 
         model_options = google_ai_get_model_list(base_url=self.base_url, api_key=self.api_key)
-        # filter by 'generateContent' models
         model_options = [mo for mo in model_options if "generateContent" in mo["supportedGenerationMethods"]]
         model_options = [str(m["name"]) for m in model_options]
 
@@ -1081,6 +1169,42 @@ class GoogleAIProvider(Provider):
                     provider_category=self.provider_category,
                 )
             )
+
+        return configs
+
+    async def list_llm_models_async(self):
+        import asyncio
+
+        from letta.llm_api.google_ai_client import google_ai_get_model_list_async
+
+        # Get and filter the model list
+        model_options = await google_ai_get_model_list_async(base_url=self.base_url, api_key=self.api_key)
+        model_options = [mo for mo in model_options if "generateContent" in mo["supportedGenerationMethods"]]
+        model_options = [str(m["name"]) for m in model_options]
+
+        # filter by model names
+        model_options = [mo[len("models/") :] if mo.startswith("models/") else mo for mo in model_options]
+
+        # Add support for all gemini models
+        model_options = [mo for mo in model_options if str(mo).startswith("gemini-")]
+
+        # Prepare tasks for context window lookups in parallel
+        async def create_config(model):
+            context_window = await self.get_model_context_window_async(model)
+            return LLMConfig(
+                model=model,
+                model_endpoint_type="google_ai",
+                model_endpoint=self.base_url,
+                context_window=context_window,
+                handle=self.get_handle(model),
+                max_tokens=8192,
+                provider_name=self.name,
+                provider_category=self.provider_category,
+            )
+
+        # Execute all config creation tasks concurrently
+        configs = await asyncio.gather(*[create_config(model) for model in model_options])
+
         return configs
 
     def list_embedding_models(self):
@@ -1088,6 +1212,16 @@ class GoogleAIProvider(Provider):
 
         # TODO: use base_url instead
         model_options = google_ai_get_model_list(base_url=self.base_url, api_key=self.api_key)
+        return self._list_embedding_models(model_options)
+
+    async def list_embedding_models_async(self):
+        from letta.llm_api.google_ai_client import google_ai_get_model_list_async
+
+        # TODO: use base_url instead
+        model_options = await google_ai_get_model_list_async(base_url=self.base_url, api_key=self.api_key)
+        return self._list_embedding_models(model_options)
+
+    def _list_embedding_models(self, model_options):
         # filter by 'generateContent' models
         model_options = [mo for mo in model_options if "embedContent" in mo["supportedGenerationMethods"]]
         model_options = [str(m["name"]) for m in model_options]
@@ -1110,7 +1244,18 @@ class GoogleAIProvider(Provider):
     def get_model_context_window(self, model_name: str) -> Optional[int]:
         from letta.llm_api.google_ai_client import google_ai_get_model_context_window
 
-        return google_ai_get_model_context_window(self.base_url, self.api_key, model_name)
+        if model_name in LLM_MAX_TOKENS:
+            return LLM_MAX_TOKENS[model_name]
+        else:
+            return google_ai_get_model_context_window(self.base_url, self.api_key, model_name)
+
+    async def get_model_context_window_async(self, model_name: str) -> Optional[int]:
+        from letta.llm_api.google_ai_client import google_ai_get_model_context_window_async
+
+        if model_name in LLM_MAX_TOKENS:
+            return LLM_MAX_TOKENS[model_name]
+        else:
+            return await google_ai_get_model_context_window_async(self.base_url, self.api_key, model_name)
 
 
 class GoogleVertexProvider(Provider):

letta/schemas/step.py

@@ -20,6 +20,7 @@ class Step(StepBase):
     )
     agent_id: Optional[str] = Field(None, description="The ID of the agent that performed the step.")
     provider_name: Optional[str] = Field(None, description="The name of the provider used for this step.")
+    provider_category: Optional[str] = Field(None, description="The category of the provider used for this step.")
     model: Optional[str] = Field(None, description="The name of the model used for this step.")
     model_endpoint: Optional[str] = Field(None, description="The model endpoint url used for this step.")
     context_window_limit: Optional[int] = Field(None, description="The context window limit configured for this step.")

letta/schemas/tool.py

@@ -5,6 +5,7 @@ from pydantic import Field, model_validator
 from letta.constants import (
     COMPOSIO_TOOL_TAG_NAME,
     FUNCTION_RETURN_CHAR_LIMIT,
+    LETTA_BUILTIN_TOOL_MODULE_NAME,
     LETTA_CORE_TOOL_MODULE_NAME,
     LETTA_MULTI_AGENT_TOOL_MODULE_NAME,
     LETTA_VOICE_TOOL_MODULE_NAME,
@@ -104,6 +105,9 @@ class Tool(BaseTool):
         elif self.tool_type in {ToolType.LETTA_VOICE_SLEEPTIME_CORE}:
             # If it's letta voice tool, we generate the json_schema on the fly here
             self.json_schema = get_json_schema_from_module(module_name=LETTA_VOICE_TOOL_MODULE_NAME, function_name=self.name)
+        elif self.tool_type in {ToolType.LETTA_BUILTIN}:
+            # If it's letta voice tool, we generate the json_schema on the fly here
+            self.json_schema = get_json_schema_from_module(module_name=LETTA_BUILTIN_TOOL_MODULE_NAME, function_name=self.name)
 
         # At this point, we need to validate that at least json_schema is populated
         if not self.json_schema:

letta/server/db.py

@@ -6,7 +6,7 @@ from typing import Any, AsyncGenerator, Generator
 from rich.console import Console
 from rich.panel import Panel
 from rich.text import Text
-from sqlalchemy import Engine, create_engine
+from sqlalchemy import Engine, NullPool, QueuePool, create_engine
 from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
 from sqlalchemy.orm import sessionmaker
 
@@ -14,6 +14,8 @@ from letta.config import LettaConfig
 from letta.log import get_logger
 from letta.settings import settings
 
+logger = get_logger(__name__)
+
 
 def print_sqlite_schema_error():
     """Print a formatted error message for SQLite schema issues"""
@@ -76,16 +78,7 @@ class DatabaseRegistry:
                 self.config.archival_storage_type = "postgres"
                 self.config.archival_storage_uri = settings.letta_pg_uri_no_default
 
-                engine = create_engine(
-                    settings.letta_pg_uri,
-                    # f"{settings.letta_pg_uri}?options=-c%20client_encoding=UTF8",
-                    pool_size=settings.pg_pool_size,
-                    max_overflow=settings.pg_max_overflow,
-                    pool_timeout=settings.pg_pool_timeout,
-                    pool_recycle=settings.pg_pool_recycle,
-                    echo=settings.pg_echo,
-                    # connect_args={"client_encoding": "utf8"},
-                )
+                engine = create_engine(settings.letta_pg_uri, **self._build_sqlalchemy_engine_args(is_async=False))
 
                 self._engines["default"] = engine
             # SQLite engine
@@ -125,14 +118,7 @@ class DatabaseRegistry:
                     async_pg_uri = f"postgresql+asyncpg://{pg_uri.split('://', 1)[1]}" if "://" in pg_uri else pg_uri
                 async_pg_uri = async_pg_uri.replace("sslmode=", "ssl=")
 
-                async_engine = create_async_engine(
-                    async_pg_uri,
-                    pool_size=settings.pg_pool_size,
-                    max_overflow=settings.pg_max_overflow,
-                    pool_timeout=settings.pg_pool_timeout,
-                    pool_recycle=settings.pg_pool_recycle,
-                    echo=settings.pg_echo,
-                )
+                async_engine = create_async_engine(async_pg_uri, **self._build_sqlalchemy_engine_args(is_async=True))
 
                 self._async_engines["default"] = async_engine
 
@@ -146,6 +132,38 @@ class DatabaseRegistry:
                 # TODO (cliandy): unclear around async sqlite support in sqlalchemy, we will not currently support this
                 self._initialized["async"] = False
 
+    def _build_sqlalchemy_engine_args(self, *, is_async: bool) -> dict:
+        """Prepare keyword arguments for create_engine / create_async_engine."""
+        use_null_pool = settings.disable_sqlalchemy_pooling
+
+        if use_null_pool:
+            logger.info("Disabling pooling on SqlAlchemy")
+            pool_cls = NullPool
+        else:
+            logger.info("Enabling pooling on SqlAlchemy")
+            pool_cls = QueuePool if not is_async else None
+
+        base_args = {
+            "echo": settings.pg_echo,
+            "pool_pre_ping": settings.pool_pre_ping,
+        }
+
+        if pool_cls:
+            base_args["poolclass"] = pool_cls
+
+        if not use_null_pool and not is_async:
+            base_args.update(
+                {
+                    "pool_size": settings.pg_pool_size,
+                    "max_overflow": settings.pg_max_overflow,
+                    "pool_timeout": settings.pg_pool_timeout,
+                    "pool_recycle": settings.pg_pool_recycle,
+                    "pool_use_lifo": settings.pool_use_lifo,
+                }
+            )
+
+        return base_args
+
     def _wrap_sqlite_engine(self, engine: Engine) -> None:
         """Wrap SQLite engine with error handling."""
         original_connect = engine.connect

letta/server/rest_api/routers/v1/__init__.py

@@ -13,6 +13,7 @@ from letta.server.rest_api.routers.v1.sandbox_configs import router as sandbox_c
 from letta.server.rest_api.routers.v1.sources import router as sources_router
 from letta.server.rest_api.routers.v1.steps import router as steps_router
 from letta.server.rest_api.routers.v1.tags import router as tags_router
+from letta.server.rest_api.routers.v1.telemetry import router as telemetry_router
 from letta.server.rest_api.routers.v1.tools import router as tools_router
 from letta.server.rest_api.routers.v1.voice import router as voice_router
 
@@ -31,6 +32,7 @@ ROUTERS = [
     runs_router,
     steps_router,
     tags_router,
+    telemetry_router,
     messages_router,
     voice_router,
     embeddings_router,