letta-nightly 0.4.1.dev20241012104008__py3-none-any.whl → 0.4.1.dev20241013104006__py3-none-any.whl

letta/local_llm/chat_completion_proxy.py

@@ -85,9 +85,7 @@ def get_chat_completion(
     elif wrapper is None:
         # Warn the user that we're using the fallback
         if not has_shown_warning:
-            print(
-                f"{CLI_WARNING_PREFIX}no wrapper specified for local LLM, using the default wrapper (you can remove this warning by specifying the wrapper with --model-wrapper)"
-            )
+            print(f"{CLI_WARNING_PREFIX}no prompt formatter specified for local LLM, using the default formatter")
             has_shown_warning = True
 
         llm_wrapper = DEFAULT_WRAPPER()

letta/providers.py

@@ -140,9 +140,17 @@ class AnthropicProvider(Provider):
 
 
 class OllamaProvider(OpenAIProvider):
+    """Ollama provider that uses the native /api/generate endpoint
+
+    See: https://github.com/ollama/ollama/blob/main/docs/api.md#generate-a-completion
+    """
+
     name: str = "ollama"
     base_url: str = Field(..., description="Base URL for the Ollama API.")
     api_key: Optional[str] = Field(None, description="API key for the Ollama API (default: `None`).")
+    default_prompt_formatter: str = Field(
+        ..., description="Default prompt formatter (aka model wrapper) to use on a /completions style API."
+    )
 
     def list_llm_models(self) -> List[LLMConfig]:
         # https://github.com/ollama/ollama/blob/main/docs/api.md#list-local-models
@@ -156,11 +164,15 @@ class OllamaProvider(OpenAIProvider):
         configs = []
         for model in response_json["models"]:
             context_window = self.get_model_context_window(model["name"])
+            if context_window is None:
+                print(f"Ollama model {model['name']} has no context window")
+                continue
             configs.append(
                 LLMConfig(
                     model=model["name"],
                     model_endpoint_type="ollama",
                     model_endpoint=self.base_url,
+                    model_wrapper=self.default_prompt_formatter,
                     context_window=context_window,
                 )
             )
@@ -192,6 +204,10 @@ class OllamaProvider(OpenAIProvider):
         # ]
         # max_position_embeddings
         # parse model cards: nous, dolphon, llama
+        if "model_info" not in response_json:
+            if "error" in response_json:
+                print(f"Ollama fetch model info error for {model_name}: {response_json['error']}")
+            return None
         for key, value in response_json["model_info"].items():
             if "context_length" in key:
                 return value
@@ -202,6 +218,10 @@ class OllamaProvider(OpenAIProvider):
 
         response = requests.post(f"{self.base_url}/api/show", json={"name": model_name, "verbose": True})
         response_json = response.json()
+        if "model_info" not in response_json:
+            if "error" in response_json:
+                print(f"Ollama fetch model info error for {model_name}: {response_json['error']}")
+            return None
         for key, value in response_json["model_info"].items():
             if "embedding_length" in key:
                 return value
@@ -220,6 +240,7 @@ class OllamaProvider(OpenAIProvider):
         for model in response_json["models"]:
             embedding_dim = self.get_model_embedding_dim(model["name"])
             if not embedding_dim:
+                print(f"Ollama model {model['name']} has no embedding dimension")
                 continue
             configs.append(
                 EmbeddingConfig(
@@ -420,7 +441,7 @@ class VLLMCompletionsProvider(Provider):
     # NOTE: vLLM only serves one model at a time (so could configure that through env variables)
     name: str = "vllm"
     base_url: str = Field(..., description="Base URL for the vLLM API.")
-    default_prompt_formatter: str = Field(..., description="Default prompt formatter (aka model wrapper)to use on vLLM /completions API.")
+    default_prompt_formatter: str = Field(..., description="Default prompt formatter (aka model wrapper) to use on vLLM /completions API.")
 
     def list_llm_models(self) -> List[LLMConfig]:
         # not supported with vLLM

letta/server/server.py

@@ -200,7 +200,7 @@ class SyncServer(Server):
     def __init__(
         self,
         chaining: bool = True,
-        max_chaining_steps: bool = None,
+        max_chaining_steps: Optional[bool] = None,
         default_interface_factory: Callable[[], AgentInterface] = lambda: CLIInterface(),
         # default_interface: AgentInterface = CLIInterface(),
         # default_persistence_manager_cls: PersistenceManager = LocalStateManager,
@@ -241,13 +241,32 @@ class SyncServer(Server):
         # collect providers (always has Letta as a default)
         self._enabled_providers: List[Provider] = [LettaProvider()]
         if model_settings.openai_api_key:
-            self._enabled_providers.append(OpenAIProvider(api_key=model_settings.openai_api_key, base_url=model_settings.openai_api_base))
+            self._enabled_providers.append(
+                OpenAIProvider(
+                    api_key=model_settings.openai_api_key,
+                    base_url=model_settings.openai_api_base,
+                )
+            )
         if model_settings.anthropic_api_key:
-            self._enabled_providers.append(AnthropicProvider(api_key=model_settings.anthropic_api_key))
+            self._enabled_providers.append(
+                AnthropicProvider(
+                    api_key=model_settings.anthropic_api_key,
+                )
+            )
         if model_settings.ollama_base_url:
-            self._enabled_providers.append(OllamaProvider(base_url=model_settings.ollama_base_url, api_key=None))
+            self._enabled_providers.append(
+                OllamaProvider(
+                    base_url=model_settings.ollama_base_url,
+                    api_key=None,
+                    default_prompt_formatter=model_settings.default_prompt_formatter,
+                )
+            )
         if model_settings.gemini_api_key:
-            self._enabled_providers.append(GoogleAIProvider(api_key=model_settings.gemini_api_key))
+            self._enabled_providers.append(
+                GoogleAIProvider(
+                    api_key=model_settings.gemini_api_key,
+                )
+            )
         if model_settings.azure_api_key and model_settings.azure_base_url:
             assert model_settings.azure_api_version, "AZURE_API_VERSION is required"
             self._enabled_providers.append(
@@ -268,7 +287,11 @@ class SyncServer(Server):
             # NOTE: to use the /chat/completions endpoint, you need to specify extra flags on vLLM startup
             # see: https://docs.vllm.ai/en/latest/getting_started/examples/openai_chat_completion_client_with_tools.html
             # e.g. "... --enable-auto-tool-choice --tool-call-parser hermes"
-            self._enabled_providers.append(VLLMChatCompletionsProvider(base_url=model_settings.vllm_api_base))
+            self._enabled_providers.append(
+                VLLMChatCompletionsProvider(
+                    base_url=model_settings.vllm_api_base,
+                )
+            )
 
     def save_agents(self):
         """Saves all the agents that are in the in-memory object store"""

{letta_nightly-0.4.1.dev20241012104008.dist-info → letta_nightly-0.4.1.dev20241013104006.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: letta-nightly
-Version: 0.4.1.dev20241012104008
+Version: 0.4.1.dev20241013104006
 Summary: Create LLM agents with long-term memory and custom tools
 License: Apache License
 Author: Letta Team

{letta_nightly-0.4.1.dev20241012104008.dist-info → letta_nightly-0.4.1.dev20241013104006.dist-info}/RECORD

@@ -45,7 +45,7 @@ letta/llm_api/llm_api_tools.py,sha256=Z3eiYUtvZKBVBcmKI2l4qWkKM4hgvLN9Y1aSxXc7y-
 letta/llm_api/openai.py,sha256=EXpktSI_TYjsCDEXBxdNXsY5uE9Rzb7BPF1F6cz8bkg,21689
 letta/local_llm/README.md,sha256=hFJyw5B0TU2jrh9nb0zGZMgdH-Ei1dSRfhvPQG_NSoU,168
 letta/local_llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-letta/local_llm/chat_completion_proxy.py,sha256=PXgNveahts5DbZ7GVcPShxmrDKropL81PY2JHc31yAA,13091
+letta/local_llm/chat_completion_proxy.py,sha256=SiohxsjGTku4vOryOZx7I0t0xoO_sUuhXgoe62fKq3c,12995
 letta/local_llm/constants.py,sha256=GIu0184EIiOLEqGeupLUYQvkgT_imIjLg3T-KM9TcFM,1125
 letta/local_llm/function_parser.py,sha256=BlNsGo1VzyfY5KdF_RwjRQNOWIsaudo7o37u1W5eg0s,2626
 letta/local_llm/grammars/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -107,7 +107,7 @@ letta/prompts/system/memgpt_doc.txt,sha256=AsT55NOORoH-K-p0fxklrDRZ3qHs4MIKMuR-M
 letta/prompts/system/memgpt_gpt35_extralong.txt,sha256=FheNhYoIzNz6qnJKhVquZVSMj3HduC48reFaX7Pf7ig,5046
 letta/prompts/system/memgpt_intuitive_knowledge.txt,sha256=sA7c3urYqREVnSBI81nTGImXAekqC0Fxc7RojFqud1g,2966
 letta/prompts/system/memgpt_modified_chat.txt,sha256=HOaPVurEftD8KsuwsclDgE2afIfklMjxhuSO96q1-6I,4656
-letta/providers.py,sha256=1vxUxMpLHyZ_XDVTUcpB98HOqszRwnt2sX9xU8zts1U,16949
+letta/providers.py,sha256=e4jWshGMu4UQ9B0yEzcHP1bDxNHEo_mgpO5M0txuOIo,17978
 letta/pytest.ini,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 letta/schemas/agent.py,sha256=ztnUqdhY9V3g0jsbTjF1ypKPC1tZx4QVFaRuLAOXNSA,6230
 letta/schemas/api_key.py,sha256=u07yzzMn-hBAHZIIKbWY16KsgiFjSNR8lAghpMUo3_4,682
@@ -167,7 +167,7 @@ letta/server/rest_api/routers/v1/tools.py,sha256=MEhxu-zMS2ff_wwcRpMuQyWA71w_3BJ
 letta/server/rest_api/routers/v1/users.py,sha256=Y2rDvHOG1B5FLSOjutY3R22vt48IngbZ-9h8CohG5rc,3378
 letta/server/rest_api/static_files.py,sha256=NG8sN4Z5EJ8JVQdj19tkFa9iQ1kBPTab9f_CUxd_u4Q,3143
 letta/server/rest_api/utils.py,sha256=Fc2ZGKzLaBa2sEtSTVjJ8D5M0xIwsWC0CVAOIJaD3rY,2176
-letta/server/server.py,sha256=XWnXt7ViLwihQVpwykxdr-fcAOGhhyGjKSM4Z-nvxw4,82206
+letta/server/server.py,sha256=txNgf3AIraK-kTV4PLwzROfgfBTIul_Y74hPygFtOHw,82687
 letta/server/startup.sh,sha256=jeGV7B_PS0hS-tT6o6GpACrUbV9WV1NI2L9aLoUDDtc,311
 letta/server/static_files/assets/index-3ab03d5b.css,sha256=OrA9W4iKJ5h2Wlr7GwdAT4wow0CM8hVit1yOxEL49Qw,54295
 letta/server/static_files/assets/index-9a9c449b.js,sha256=qoWUq6_kuLhE9NFkNeCBptgq-oERW46r0tB3JlWe_qc,1818951
@@ -184,8 +184,8 @@ letta/settings.py,sha256=6VWC3vtTa8vqj6dqos4p_xHTMJNJS_8LRGJmqvaU1-o,3219
 letta/streaming_interface.py,sha256=_FPUWy58j50evHcpXyd7zB1wWqeCc71NCFeWh_TBvnw,15736
 letta/system.py,sha256=buKYPqG5n2x41hVmWpu6JUpyd7vTWED9Km2_M7dLrvk,6960
 letta/utils.py,sha256=neUs7mxNfndzRL5XUxerr8Lic6w7qnyyvf8FBwMnyWw,30852
-letta_nightly-0.4.1.dev20241012104008.dist-info/LICENSE,sha256=mExtuZ_GYJgDEI38GWdiEYZizZS4KkVt2SF1g_GPNhI,10759
-letta_nightly-0.4.1.dev20241012104008.dist-info/METADATA,sha256=bzoVG8Kbu7WPOswZWQ4HGRQoNqmrA2EvT8XP1sbuQ6U,6008
-letta_nightly-0.4.1.dev20241012104008.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-letta_nightly-0.4.1.dev20241012104008.dist-info/entry_points.txt,sha256=2zdiyGNEZGV5oYBuS-y2nAAgjDgcC9yM_mHJBFSRt5U,40
-letta_nightly-0.4.1.dev20241012104008.dist-info/RECORD,,
+letta_nightly-0.4.1.dev20241013104006.dist-info/LICENSE,sha256=mExtuZ_GYJgDEI38GWdiEYZizZS4KkVt2SF1g_GPNhI,10759
+letta_nightly-0.4.1.dev20241013104006.dist-info/METADATA,sha256=L5hOlFIaGT5BnE945CxaUdQEoKsOLrysrrL_bD9MjVU,6008
+letta_nightly-0.4.1.dev20241013104006.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+letta_nightly-0.4.1.dev20241013104006.dist-info/entry_points.txt,sha256=2zdiyGNEZGV5oYBuS-y2nAAgjDgcC9yM_mHJBFSRt5U,40
+letta_nightly-0.4.1.dev20241013104006.dist-info/RECORD,,