letta-nightly 0.8.17.dev20250722104501__py3-none-any.whl → 0.9.0.dev20250724081419__py3-none-any.whl

letta/schemas/providers/google_gemini.py

@@ -0,0 +1,102 @@
+import asyncio
+from typing import Literal
+
+from pydantic import Field
+
+from letta.constants import DEFAULT_EMBEDDING_CHUNK_SIZE, LLM_MAX_TOKENS
+from letta.schemas.embedding_config import EmbeddingConfig
+from letta.schemas.enums import ProviderCategory, ProviderType
+from letta.schemas.llm_config import LLMConfig
+from letta.schemas.providers.base import Provider
+
+
+class GoogleAIProvider(Provider):
+    provider_type: Literal[ProviderType.google_ai] = Field(ProviderType.google_ai, description="The type of the provider.")
+    provider_category: ProviderCategory = Field(ProviderCategory.base, description="The category of the provider (base or byok)")
+    api_key: str = Field(..., description="API key for the Google AI API.")
+    base_url: str = "https://generativelanguage.googleapis.com"
+
+    async def check_api_key(self):
+        from letta.llm_api.google_ai_client import google_ai_check_valid_api_key
+
+        google_ai_check_valid_api_key(self.api_key)
+
+    async def list_llm_models_async(self):
+        from letta.llm_api.google_ai_client import google_ai_get_model_list_async
+
+        # Get and filter the model list
+        model_options = await google_ai_get_model_list_async(base_url=self.base_url, api_key=self.api_key)
+        model_options = [mo for mo in model_options if "generateContent" in mo["supportedGenerationMethods"]]
+        model_options = [str(m["name"]) for m in model_options]
+
+        # filter by model names
+        model_options = [mo[len("models/") :] if mo.startswith("models/") else mo for mo in model_options]
+
+        # Add support for all gemini models
+        model_options = [mo for mo in model_options if str(mo).startswith("gemini-")]
+
+        # Prepare tasks for context window lookups in parallel
+        async def create_config(model):
+            context_window = await self.get_model_context_window_async(model)
+            return LLMConfig(
+                model=model,
+                model_endpoint_type="google_ai",
+                model_endpoint=self.base_url,
+                context_window=context_window,
+                handle=self.get_handle(model),
+                max_tokens=8192,
+                provider_name=self.name,
+                provider_category=self.provider_category,
+            )
+
+        # Execute all config creation tasks concurrently
+        configs = await asyncio.gather(*[create_config(model) for model in model_options])
+
+        return configs
+
+    async def list_embedding_models_async(self):
+        from letta.llm_api.google_ai_client import google_ai_get_model_list_async
+
+        # TODO: use base_url instead
+        model_options = await google_ai_get_model_list_async(base_url=self.base_url, api_key=self.api_key)
+        return self._list_embedding_models(model_options)
+
+    def _list_embedding_models(self, model_options):
+        # filter by 'generateContent' models
+        model_options = [mo for mo in model_options if "embedContent" in mo["supportedGenerationMethods"]]
+        model_options = [str(m["name"]) for m in model_options]
+        model_options = [mo[len("models/") :] if mo.startswith("models/") else mo for mo in model_options]
+
+        configs = []
+        for model in model_options:
+            configs.append(
+                EmbeddingConfig(
+                    embedding_model=model,
+                    embedding_endpoint_type="google_ai",
+                    embedding_endpoint=self.base_url,
+                    embedding_dim=768,
+                    embedding_chunk_size=DEFAULT_EMBEDDING_CHUNK_SIZE,  # NOTE: max is 2048
+                    handle=self.get_handle(model, is_embedding=True),
+                    batch_size=1024,
+                )
+            )
+        return configs
+
+    def get_model_context_window(self, model_name: str) -> int | None:
+        import warnings
+
+        warnings.warn("This is deprecated, use get_model_context_window_async when possible.", DeprecationWarning)
+        from letta.llm_api.google_ai_client import google_ai_get_model_context_window
+
+        if model_name in LLM_MAX_TOKENS:
+            return LLM_MAX_TOKENS[model_name]
+        else:
+            return google_ai_get_model_context_window(self.base_url, self.api_key, model_name)
+
+    async def get_model_context_window_async(self, model_name: str) -> int | None:
+        from letta.llm_api.google_ai_client import google_ai_get_model_context_window_async
+
+        if model_name in LLM_MAX_TOKENS:
+            return LLM_MAX_TOKENS[model_name]
+        else:
+            return await google_ai_get_model_context_window_async(self.base_url, self.api_key, model_name)

letta/schemas/providers/google_vertex.py

@@ -0,0 +1,54 @@
+from typing import Literal
+
+from pydantic import Field
+
+from letta.constants import DEFAULT_EMBEDDING_CHUNK_SIZE
+from letta.schemas.embedding_config import EmbeddingConfig
+from letta.schemas.enums import ProviderCategory, ProviderType
+from letta.schemas.llm_config import LLMConfig
+from letta.schemas.providers.base import Provider
+
+
+# TODO (cliandy): GoogleVertexProvider uses hardcoded models vs Gemini fetches from API
+class GoogleVertexProvider(Provider):
+    provider_type: Literal[ProviderType.google_vertex] = Field(ProviderType.google_vertex, description="The type of the provider.")
+    provider_category: ProviderCategory = Field(ProviderCategory.base, description="The category of the provider (base or byok)")
+    google_cloud_project: str = Field(..., description="GCP project ID for the Google Vertex API.")
+    google_cloud_location: str = Field(..., description="GCP region for the Google Vertex API.")
+
+    async def list_llm_models_async(self) -> list[LLMConfig]:
+        from letta.llm_api.google_constants import GOOGLE_MODEL_TO_CONTEXT_LENGTH
+
+        configs = []
+        for model, context_length in GOOGLE_MODEL_TO_CONTEXT_LENGTH.items():
+            configs.append(
+                LLMConfig(
+                    model=model,
+                    model_endpoint_type="google_vertex",
+                    model_endpoint=f"https://{self.google_cloud_location}-aiplatform.googleapis.com/v1/projects/{self.google_cloud_project}/locations/{self.google_cloud_location}",
+                    context_window=context_length,
+                    handle=self.get_handle(model),
+                    max_tokens=8192,
+                    provider_name=self.name,
+                    provider_category=self.provider_category,
+                )
+            )
+        return configs
+
+    async def list_embedding_models_async(self) -> list[EmbeddingConfig]:
+        from letta.llm_api.google_constants import GOOGLE_EMBEDING_MODEL_TO_DIM
+
+        configs = []
+        for model, dim in GOOGLE_EMBEDING_MODEL_TO_DIM.items():
+            configs.append(
+                EmbeddingConfig(
+                    embedding_model=model,
+                    embedding_endpoint_type="google_vertex",
+                    embedding_endpoint=f"https://{self.google_cloud_location}-aiplatform.googleapis.com/v1/projects/{self.google_cloud_project}/locations/{self.google_cloud_location}",
+                    embedding_dim=dim,
+                    embedding_chunk_size=DEFAULT_EMBEDDING_CHUNK_SIZE,  # NOTE: max is 2048
+                    handle=self.get_handle(model, is_embedding=True),
+                    batch_size=1024,
+                )
+            )
+        return configs

letta/schemas/providers/groq.py

@@ -0,0 +1,35 @@
+from typing import Literal
+
+from pydantic import Field
+
+from letta.schemas.enums import ProviderCategory, ProviderType
+from letta.schemas.llm_config import LLMConfig
+from letta.schemas.providers.openai import OpenAIProvider
+
+
+class GroqProvider(OpenAIProvider):
+    provider_type: Literal[ProviderType.groq] = Field(ProviderType.groq, description="The type of the provider.")
+    provider_category: ProviderCategory = Field(ProviderCategory.base, description="The category of the provider (base or byok)")
+    base_url: str = "https://api.groq.com/openai/v1"
+    api_key: str = Field(..., description="API key for the Groq API.")
+
+    async def list_llm_models_async(self) -> list[LLMConfig]:
+        from letta.llm_api.openai import openai_get_model_list_async
+
+        response = await openai_get_model_list_async(self.base_url, api_key=self.api_key)
+        configs = []
+        for model in response["data"]:
+            if "context_window" not in model:
+                continue
+            configs.append(
+                LLMConfig(
+                    model=model["id"],
+                    model_endpoint_type="groq",
+                    model_endpoint=self.base_url,
+                    context_window=model["context_window"],
+                    handle=self.get_handle(model["id"]),
+                    provider_name=self.name,
+                    provider_category=self.provider_category,
+                )
+            )
+        return configs

letta/schemas/providers/letta.py

@@ -0,0 +1,39 @@
+from typing import Literal
+
+from pydantic import Field
+
+from letta.constants import DEFAULT_EMBEDDING_CHUNK_SIZE, LETTA_MODEL_ENDPOINT
+from letta.schemas.embedding_config import EmbeddingConfig
+from letta.schemas.enums import ProviderCategory, ProviderType
+from letta.schemas.llm_config import LLMConfig
+from letta.schemas.providers.base import Provider
+
+
+class LettaProvider(Provider):
+    provider_type: Literal[ProviderType.letta] = Field(ProviderType.letta, description="The type of the provider.")
+    provider_category: ProviderCategory = Field(ProviderCategory.base, description="The category of the provider (base or byok)")
+
+    async def list_llm_models_async(self) -> list[LLMConfig]:
+        return [
+            LLMConfig(
+                model="letta-free",  # NOTE: renamed
+                model_endpoint_type="openai",
+                model_endpoint=LETTA_MODEL_ENDPOINT,
+                context_window=30000,
+                handle=self.get_handle("letta-free"),
+                provider_name=self.name,
+                provider_category=self.provider_category,
+            )
+        ]
+
+    async def list_embedding_models_async(self):
+        return [
+            EmbeddingConfig(
+                embedding_model="letta-free",  # NOTE: renamed
+                embedding_endpoint_type="hugging-face",
+                embedding_endpoint="https://embeddings.memgpt.ai",
+                embedding_dim=1024,
+                embedding_chunk_size=DEFAULT_EMBEDDING_CHUNK_SIZE,
+                handle=self.get_handle("letta-free", is_embedding=True),
+            )
+        ]

letta/schemas/providers/lmstudio.py

@@ -0,0 +1,97 @@
+import warnings
+from typing import Literal
+
+from pydantic import Field
+
+from letta.constants import DEFAULT_EMBEDDING_CHUNK_SIZE
+from letta.schemas.embedding_config import EmbeddingConfig
+from letta.schemas.enums import ProviderCategory, ProviderType
+from letta.schemas.llm_config import LLMConfig
+from letta.schemas.providers.openai import OpenAIProvider
+
+
+class LMStudioOpenAIProvider(OpenAIProvider):
+    provider_type: Literal[ProviderType.lmstudio_openai] = Field(ProviderType.lmstudio_openai, description="The type of the provider.")
+    provider_category: ProviderCategory = Field(ProviderCategory.base, description="The category of the provider (base or byok)")
+    base_url: str = Field(..., description="Base URL for the LMStudio OpenAI API.")
+    api_key: str | None = Field(None, description="API key for the LMStudio API.")
+
+    @property
+    def model_endpoint_url(self):
+        # For LMStudio, we want to hit 'GET /api/v0/models' instead of 'GET /v1/models'
+        return f"{self.base_url.strip('/v1')}/api/v0"
+
+    async def list_llm_models_async(self) -> list[LLMConfig]:
+        from letta.llm_api.openai import openai_get_model_list_async
+
+        response = await openai_get_model_list_async(self.model_endpoint_url)
+
+        if "data" not in response:
+            warnings.warn(f"LMStudio OpenAI model query response missing 'data' field: {response}")
+            return []
+
+        configs = []
+        for model in response["data"]:
+            model_type = model.get("type")
+            if not model_type:
+                warnings.warn(f"LMStudio OpenAI model missing 'type' field: {model}")
+                continue
+            if model_type not in ("vlm", "llm"):
+                continue
+
+            # TODO (cliandy): previously we didn't get the backup context size, is this valid?
+            check = self._do_model_checks_for_name_and_context_size(model)
+            if check is None:
+                continue
+            model_name, context_window_size = check
+
+            configs.append(
+                LLMConfig(
+                    model=model_name,
+                    model_endpoint_type="openai",
+                    model_endpoint=self.base_url,
+                    context_window=context_window_size,
+                    handle=self.get_handle(model_name),
+                    provider_name=self.name,
+                    provider_category=self.provider_category,
+                )
+            )
+
+        return configs
+
+    async def list_embedding_models_async(self) -> list[EmbeddingConfig]:
+        from letta.llm_api.openai import openai_get_model_list_async
+
+        response = await openai_get_model_list_async(self.model_endpoint_url)
+
+        if "data" not in response:
+            warnings.warn(f"LMStudio OpenAI model query response missing 'data' field: {response}")
+            return []
+
+        configs = []
+        for model in response["data"]:
+            model_type = model.get("type")
+            if not model_type:
+                warnings.warn(f"LMStudio OpenAI model missing 'type' field: {model}")
+                continue
+            if model_type not in ("embeddings"):
+                continue
+
+            # TODO (cliandy): previously we didn't get the backup context size, is this valid?
+            check = self._do_model_checks_for_name_and_context_size(model, length_key="max_context_length")
+            if check is None:
+                continue
+            model_name, context_window_size = check
+
+            configs.append(
+                EmbeddingConfig(
+                    embedding_model=model_name,
+                    embedding_endpoint_type="openai",
+                    embedding_endpoint=self.base_url,
+                    embedding_dim=768,  # Default embedding dimension, not context window
+                    embedding_chunk_size=DEFAULT_EMBEDDING_CHUNK_SIZE,  # NOTE: max is 2048
+                    handle=self.get_handle(model_name),
+                ),
+            )
+
+        return configs

letta/schemas/providers/mistral.py

@@ -0,0 +1,41 @@
+from typing import Literal
+
+from pydantic import Field
+
+from letta.schemas.enums import ProviderCategory, ProviderType
+from letta.schemas.llm_config import LLMConfig
+from letta.schemas.providers.base import Provider
+
+
+class MistralProvider(Provider):
+    provider_type: Literal[ProviderType.mistral] = Field(ProviderType.mistral, description="The type of the provider.")
+    provider_category: ProviderCategory = Field(ProviderCategory.base, description="The category of the provider (base or byok)")
+    api_key: str = Field(..., description="API key for the Mistral API.")
+    base_url: str = "https://api.mistral.ai/v1"
+
+    async def list_llm_models_async(self) -> list[LLMConfig]:
+        from letta.llm_api.mistral import mistral_get_model_list_async
+
+        # Some hardcoded support for OpenRouter (so that we only get models with tool calling support)...
+        # See: https://openrouter.ai/docs/requests
+        response = await mistral_get_model_list_async(self.base_url, api_key=self.api_key)
+
+        assert "data" in response, f"Mistral model query response missing 'data' field: {response}"
+
+        configs = []
+        for model in response["data"]:
+            # If model has chat completions and function calling enabled
+            if model["capabilities"]["completion_chat"] and model["capabilities"]["function_calling"]:
+                configs.append(
+                    LLMConfig(
+                        model=model["id"],
+                        model_endpoint_type="openai",
+                        model_endpoint=self.base_url,
+                        context_window=model["max_context_length"],
+                        handle=self.get_handle(model["id"]),
+                        provider_name=self.name,
+                        provider_category=self.provider_category,
+                    )
+                )
+
+        return configs

letta/schemas/providers/ollama.py

@@ -0,0 +1,151 @@
+from typing import Literal
+
+import aiohttp
+import requests
+from pydantic import Field
+
+from letta.constants import DEFAULT_EMBEDDING_CHUNK_SIZE
+from letta.log import get_logger
+from letta.schemas.embedding_config import EmbeddingConfig
+from letta.schemas.enums import ProviderCategory, ProviderType
+from letta.schemas.llm_config import LLMConfig
+from letta.schemas.providers.openai import OpenAIProvider
+
+logger = get_logger(__name__)
+
+
+class OllamaProvider(OpenAIProvider):
+    """Ollama provider that uses the native /api/generate endpoint
+
+    See: https://github.com/ollama/ollama/blob/main/docs/api.md#generate-a-completion
+    """
+
+    provider_type: Literal[ProviderType.ollama] = Field(ProviderType.ollama, description="The type of the provider.")
+    provider_category: ProviderCategory = Field(ProviderCategory.base, description="The category of the provider (base or byok)")
+    base_url: str = Field(..., description="Base URL for the Ollama API.")
+    api_key: str | None = Field(None, description="API key for the Ollama API (default: `None`).")
+    default_prompt_formatter: str = Field(
+        ..., description="Default prompt formatter (aka model wrapper) to use on a /completions style API."
+    )
+
+    async def list_llm_models_async(self) -> list[LLMConfig]:
+        """List available LLM Models from Ollama
+
+        https://github.com/ollama/ollama/blob/main/docs/api.md#list-local-models"""
+        endpoint = f"{self.base_url}/api/tags"
+        async with aiohttp.ClientSession() as session:
+            async with session.get(endpoint) as response:
+                if response.status != 200:
+                    raise Exception(f"Failed to list Ollama models: {response.text}")
+                response_json = await response.json()
+
+        configs = []
+        for model in response_json["models"]:
+            context_window = self.get_model_context_window(model["name"])
+            if context_window is None:
+                print(f"Ollama model {model['name']} has no context window")
+                continue
+            configs.append(
+                LLMConfig(
+                    model=model["name"],
+                    model_endpoint_type="ollama",
+                    model_endpoint=self.base_url,
+                    model_wrapper=self.default_prompt_formatter,
+                    context_window=context_window,
+                    handle=self.get_handle(model["name"]),
+                    provider_name=self.name,
+                    provider_category=self.provider_category,
+                )
+            )
+        return configs
+
+    async def list_embedding_models_async(self) -> list[EmbeddingConfig]:
+        """List available embedding models from Ollama
+
+        https://github.com/ollama/ollama/blob/main/docs/api.md#list-local-models
+        """
+        endpoint = f"{self.base_url}/api/tags"
+        async with aiohttp.ClientSession() as session:
+            async with session.get(endpoint) as response:
+                if response.status != 200:
+                    raise Exception(f"Failed to list Ollama models: {response.text}")
+                response_json = await response.json()
+
+        configs = []
+        for model in response_json["models"]:
+            embedding_dim = await self._get_model_embedding_dim_async(model["name"])
+            if not embedding_dim:
+                print(f"Ollama model {model['name']} has no embedding dimension")
+                continue
+            configs.append(
+                EmbeddingConfig(
+                    embedding_model=model["name"],
+                    embedding_endpoint_type="ollama",
+                    embedding_endpoint=self.base_url,
+                    embedding_dim=embedding_dim,
+                    embedding_chunk_size=DEFAULT_EMBEDDING_CHUNK_SIZE,
+                    handle=self.get_handle(model["name"], is_embedding=True),
+                )
+            )
+        return configs
+
+    def get_model_context_window(self, model_name: str) -> int | None:
+        """Gets model context window for Ollama. As this can look different based on models,
+        we use the following for guidance:
+
+        "llama.context_length": 8192,
+        "llama.embedding_length": 4096,
+        source: https://github.com/ollama/ollama/blob/main/docs/api.md#show-model-information
+
+        FROM 2024-10-08
+        Notes from vLLM around keys
+        source: https://github.com/vllm-project/vllm/blob/72ad2735823e23b4e1cc79b7c73c3a5f3c093ab0/vllm/config.py#L3488
+
+        possible_keys = [
+            # OPT
+            "max_position_embeddings",
+            # GPT-2
+            "n_positions",
+            # MPT
+            "max_seq_len",
+            # ChatGLM2
+            "seq_length",
+            # Command-R
+            "model_max_length",
+            # Whisper
+            "max_target_positions",
+            # Others
+            "max_sequence_length",
+            "max_seq_length",
+            "seq_len",
+        ]
+        max_position_embeddings
+        parse model cards: nous, dolphon, llama
+        """
+        endpoint = f"{self.base_url}/api/show"
+        payload = {"name": model_name, "verbose": True}
+        response = requests.post(endpoint, json=payload)
+        if response.status_code != 200:
+            return None
+
+        try:
+            model_info = response.json()
+            # Try to extract context window from model parameters
+            if "model_info" in model_info and "llama.context_length" in model_info["model_info"]:
+                return int(model_info["model_info"]["llama.context_length"])
+        except Exception:
+            pass
+        logger.warning(f"Failed to get model context window for {model_name}")
+        return None
+
+    async def _get_model_embedding_dim_async(self, model_name: str):
+        async with aiohttp.ClientSession() as session:
+            async with session.post(f"{self.base_url}/api/show", json={"name": model_name, "verbose": True}) as response:
+                response_json = await response.json()
+
+        if "model_info" not in response_json:
+            if "error" in response_json:
+                logger.warning("Ollama fetch model info error for %s: %s", model_name, response_json["error"])
+            return None
+
+        return response_json["model_info"].get("embedding_length")