letta-nightly 0.4.1.dev20241009104130__tar.gz → 0.4.1.dev20241011104054__tar.gz

{letta_nightly-0.4.1.dev20241009104130 → letta_nightly-0.4.1.dev20241011104054}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: letta-nightly
-Version: 0.4.1.dev20241009104130
+Version: 0.4.1.dev20241011104054
 Summary: Create LLM agents with long-term memory and custom tools
 License: Apache License
 Author: Letta Team

{letta_nightly-0.4.1.dev20241009104130 → letta_nightly-0.4.1.dev20241011104054}/letta/agent_store/db.py

@@ -398,8 +398,6 @@ class PostgresStorageConnector(SQLStorageConnector):
         return records
 
     def insert_many(self, records, exists_ok=True, show_progress=False):
-        pass
-
         # TODO: this is terrible, should eventually be done the same way for all types (migrate to SQLModel)
         if len(records) == 0:
             return
@@ -506,18 +504,36 @@ class SQLLiteStorageConnector(SQLStorageConnector):
         # sqlite3.register_converter("UUID", lambda b: uuid.UUID(bytes_le=b))
 
     def insert_many(self, records, exists_ok=True, show_progress=False):
-        pass
-
         # TODO: this is terrible, should eventually be done the same way for all types (migrate to SQLModel)
         if len(records) == 0:
             return
+
+        added_ids = []  # avoid adding duplicates
+        # NOTE: this has not great performance due to the excessive commits
         with self.session_maker() as session:
             iterable = tqdm(records) if show_progress else records
             for record in iterable:
                 # db_record = self.db_model(**vars(record))
-                db_record = self.db_model(**record.dict())
-                session.add(db_record)
-            session.commit()
+
+                if record.id in added_ids:
+                    continue
+
+                existing_record = session.query(self.db_model).filter_by(id=record.id).first()
+                if existing_record:
+                    if exists_ok:
+                        fields = record.model_dump()
+                        fields.pop("id")
+                        session.query(self.db_model).filter(self.db_model.id == record.id).update(fields)
+                        session.commit()
+                    else:
+                        raise ValueError(f"Record with id {record.id} already exists.")
+
+                else:
+                    db_record = self.db_model(**record.dict())
+                    session.add(db_record)
+                    session.commit()
+
+                added_ids.append(record.id)
 
     def insert(self, record, exists_ok=True):
         self.insert_many([record], exists_ok=exists_ok)

{letta_nightly-0.4.1.dev20241009104130 → letta_nightly-0.4.1.dev20241011104054}/letta/cli/cli.py

@@ -11,9 +11,12 @@ from letta import create_client
 from letta.agent import Agent, save_agent
 from letta.config import LettaConfig
 from letta.constants import CLI_WARNING_PREFIX, LETTA_DIR
+from letta.local_llm.constants import ASSISTANT_MESSAGE_CLI_SYMBOL
 from letta.log import get_logger
 from letta.metadata import MetadataStore
+from letta.schemas.embedding_config import EmbeddingConfig
 from letta.schemas.enums import OptionState
+from letta.schemas.llm_config import LLMConfig
 from letta.schemas.memory import ChatMemory, Memory
 from letta.server.server import logger as server_logger
 
@@ -232,25 +235,46 @@ def run(
         # choose from list of llm_configs
         llm_configs = client.list_llm_configs()
         llm_options = [llm_config.model for llm_config in llm_configs]
+
+        # TODO move into LLMConfig as a class method?
+        def prettify_llm_config(llm_config: LLMConfig) -> str:
+            return f"{llm_config.model}" + f" ({llm_config.model_endpoint})" if llm_config.model_endpoint else ""
+
+        llm_choices = [questionary.Choice(title=prettify_llm_config(llm_config), value=llm_config) for llm_config in llm_configs]
+
         # select model
         if len(llm_options) == 0:
             raise ValueError("No LLM models found. Please enable a provider.")
         elif len(llm_options) == 1:
             llm_model_name = llm_options[0]
         else:
-            llm_model_name = questionary.select("Select LLM model:", choices=llm_options).ask()
+            llm_model_name = questionary.select("Select LLM model:", choices=llm_choices).ask().model
         llm_config = [llm_config for llm_config in llm_configs if llm_config.model == llm_model_name][0]
 
         # choose form list of embedding configs
         embedding_configs = client.list_embedding_configs()
         embedding_options = [embedding_config.embedding_model for embedding_config in embedding_configs]
+
+        # TODO move into EmbeddingConfig as a class method?
+        def prettify_embed_config(embedding_config: EmbeddingConfig) -> str:
+            return (
+                f"{embedding_config.embedding_model}" + f" ({embedding_config.embedding_endpoint})"
+                if embedding_config.embedding_endpoint
+                else ""
+            )
+
+        embedding_choices = [
+            questionary.Choice(title=prettify_embed_config(embedding_config), value=embedding_config)
+            for embedding_config in embedding_configs
+        ]
+
         # select model
         if len(embedding_options) == 0:
             raise ValueError("No embedding models found. Please enable a provider.")
         elif len(embedding_options) == 1:
             embedding_model_name = embedding_options[0]
         else:
-            embedding_model_name = questionary.select("Select embedding model:", choices=embedding_options).ask()
+            embedding_model_name = questionary.select("Select embedding model:", choices=embedding_choices).ask().embedding_model
         embedding_config = [
             embedding_config for embedding_config in embedding_configs if embedding_config.embedding_model == embedding_model_name
         ][0]
@@ -276,7 +300,7 @@ def run(
         memory = ChatMemory(human=human_obj.value, persona=persona_obj.value, limit=core_memory_limit)
         metadata = {"human": human_obj.name, "persona": persona_obj.name}
 
-        typer.secho(f"->  🤖 Using persona profile: '{persona_obj.name}'", fg=typer.colors.WHITE)
+        typer.secho(f"->  {ASSISTANT_MESSAGE_CLI_SYMBOL} Using persona profile: '{persona_obj.name}'", fg=typer.colors.WHITE)
         typer.secho(f"->  🧑 Using human profile: '{human_obj.name}'", fg=typer.colors.WHITE)
 
         # add tools

letta_nightly-0.4.1.dev20241011104054/letta/cli/cli_config.py

@@ -0,0 +1,228 @@
+import ast
+import os
+from enum import Enum
+from typing import Annotated, List, Optional
+
+import questionary
+import typer
+from prettytable.colortable import ColorTable, Themes
+from tqdm import tqdm
+
+from letta import utils
+
+app = typer.Typer()
+
+
+@app.command()
+def configure():
+    """Updates default Letta configurations
+
+    This function and quickstart should be the ONLY place where LettaConfig.save() is called
+    """
+    print("`letta configure` has been deprecated. Please see documentation on configuration, and run `letta run` instead.")
+
+
+class ListChoice(str, Enum):
+    agents = "agents"
+    humans = "humans"
+    personas = "personas"
+    sources = "sources"
+
+
+@app.command()
+def list(arg: Annotated[ListChoice, typer.Argument]):
+    from letta.client.client import create_client
+
+    client = create_client()
+    table = ColorTable(theme=Themes.OCEAN)
+    if arg == ListChoice.agents:
+        """List all agents"""
+        table.field_names = ["Name", "LLM Model", "Embedding Model", "Embedding Dim", "Persona", "Human", "Data Source", "Create Time"]
+        for agent in tqdm(client.list_agents()):
+            # TODO: add this function
+            sources = client.list_attached_sources(agent_id=agent.id)
+            source_names = [source.name for source in sources if source is not None]
+            table.add_row(
+                [
+                    agent.name,
+                    agent.llm_config.model,
+                    agent.embedding_config.embedding_model,
+                    agent.embedding_config.embedding_dim,
+                    agent.memory.get_block("persona").value[:100] + "...",
+                    agent.memory.get_block("human").value[:100] + "...",
+                    ",".join(source_names),
+                    utils.format_datetime(agent.created_at),
+                ]
+            )
+        print(table)
+    elif arg == ListChoice.humans:
+        """List all humans"""
+        table.field_names = ["Name", "Text"]
+        for human in client.list_humans():
+            table.add_row([human.name, human.value.replace("\n", "")[:100]])
+        print(table)
+    elif arg == ListChoice.personas:
+        """List all personas"""
+        table.field_names = ["Name", "Text"]
+        for persona in client.list_personas():
+            table.add_row([persona.name, persona.value.replace("\n", "")[:100]])
+        print(table)
+    elif arg == ListChoice.sources:
+        """List all data sources"""
+
+        # create table
+        table.field_names = ["Name", "Description", "Embedding Model", "Embedding Dim", "Created At"]
+        # TODO: eventually look accross all storage connections
+        # TODO: add data source stats
+        # TODO: connect to agents
+
+        # get all sources
+        for source in client.list_sources():
+            # get attached agents
+            table.add_row(
+                [
+                    source.name,
+                    source.description,
+                    source.embedding_config.embedding_model,
+                    source.embedding_config.embedding_dim,
+                    utils.format_datetime(source.created_at),
+                ]
+            )
+
+        print(table)
+    else:
+        raise ValueError(f"Unknown argument {arg}")
+    return table
+
+
+@app.command()
+def add_tool(
+    filename: str = typer.Option(..., help="Path to the Python file containing the function"),
+    name: Optional[str] = typer.Option(None, help="Name of the tool"),
+    update: bool = typer.Option(True, help="Update the tool if it already exists"),
+    tags: Optional[List[str]] = typer.Option(None, help="Tags for the tool"),
+):
+    """Add or update a tool from a Python file."""
+    from letta.client.client import create_client
+
+    client = create_client(base_url=os.getenv("MEMGPT_BASE_URL"), token=os.getenv("MEMGPT_SERVER_PASS"))
+
+    # 1. Parse the Python file
+    with open(filename, "r", encoding="utf-8") as file:
+        source_code = file.read()
+
+    # 2. Parse the source code to extract the function
+    # Note: here we assume it is one function only in the file.
+    module = ast.parse(source_code)
+    func_def = None
+    for node in module.body:
+        if isinstance(node, ast.FunctionDef):
+            func_def = node
+            break
+
+    if not func_def:
+        raise ValueError("No function found in the provided file")
+
+    # 3. Compile the function to make it callable
+    # Explanation courtesy of GPT-4:
+    # Compile the AST (Abstract Syntax Tree) node representing the function definition into a code object
+    # ast.Module creates a module node containing the function definition (func_def)
+    # compile converts the AST into a code object that can be executed by the Python interpreter
+    # The exec function executes the compiled code object in the current context,
+    # effectively defining the function within the current namespace
+    exec(compile(ast.Module([func_def], []), filename, "exec"))
+    # Retrieve the function object by evaluating its name in the current namespace
+    # eval looks up the function name in the current scope and returns the function object
+    func = eval(func_def.name)
+
+    # 4. Add or update the tool
+    tool = client.create_tool(func=func, name=name, tags=tags, update=update)
+    print(f"Tool {tool.name} added successfully")
+
+
+@app.command()
+def list_tools():
+    """List all available tools."""
+    from letta.client.client import create_client
+
+    client = create_client(base_url=os.getenv("MEMGPT_BASE_URL"), token=os.getenv("MEMGPT_SERVER_PASS"))
+
+    tools = client.list_tools()
+    for tool in tools:
+        print(f"Tool: {tool.name}")
+
+
+@app.command()
+def add(
+    option: str,  # [human, persona]
+    name: Annotated[str, typer.Option(help="Name of human/persona")],
+    text: Annotated[Optional[str], typer.Option(help="Text of human/persona")] = None,
+    filename: Annotated[Optional[str], typer.Option("-f", help="Specify filename")] = None,
+):
+    """Add a person/human"""
+    from letta.client.client import create_client
+
+    client = create_client(base_url=os.getenv("MEMGPT_BASE_URL"), token=os.getenv("MEMGPT_SERVER_PASS"))
+    if filename:  # read from file
+        assert text is None, "Cannot specify both text and filename"
+        with open(filename, "r", encoding="utf-8") as f:
+            text = f.read()
+    else:
+        assert text is not None, "Must specify either text or filename"
+    if option == "persona":
+        persona_id = client.get_persona_id(name)
+        if persona_id:
+            client.get_persona(persona_id)
+            # config if user wants to overwrite
+            if not questionary.confirm(f"Persona {name} already exists. Overwrite?").ask():
+                return
+            client.update_persona(persona_id, text=text)
+        else:
+            client.create_persona(name=name, text=text)
+
+    elif option == "human":
+        human_id = client.get_human_id(name)
+        if human_id:
+            human = client.get_human(human_id)
+            # config if user wants to overwrite
+            if not questionary.confirm(f"Human {name} already exists. Overwrite?").ask():
+                return
+            client.update_human(human_id, text=text)
+        else:
+            human = client.create_human(name=name, text=text)
+    else:
+        raise ValueError(f"Unknown kind {option}")
+
+
+@app.command()
+def delete(option: str, name: str):
+    """Delete a source from the archival memory."""
+    from letta.client.client import create_client
+
+    client = create_client(base_url=os.getenv("MEMGPT_BASE_URL"), token=os.getenv("MEMGPT_API_KEY"))
+    try:
+        # delete from metadata
+        if option == "source":
+            # delete metadata
+            source_id = client.get_source_id(name)
+            assert source_id is not None, f"Source {name} does not exist"
+            client.delete_source(source_id)
+        elif option == "agent":
+            agent_id = client.get_agent_id(name)
+            assert agent_id is not None, f"Agent {name} does not exist"
+            client.delete_agent(agent_id=agent_id)
+        elif option == "human":
+            human_id = client.get_human_id(name)
+            assert human_id is not None, f"Human {name} does not exist"
+            client.delete_human(human_id)
+        elif option == "persona":
+            persona_id = client.get_persona_id(name)
+            assert persona_id is not None, f"Persona {name} does not exist"
+            client.delete_persona(persona_id)
+        else:
+            raise ValueError(f"Option {option} not implemented")
+
+        typer.secho(f"Deleted {option} '{name}'", fg=typer.colors.GREEN)
+
+    except Exception as e:
+        typer.secho(f"Failed to delete {option}'{name}'\n{e}", fg=typer.colors.RED)

{letta_nightly-0.4.1.dev20241009104130 → letta_nightly-0.4.1.dev20241011104054}/letta/client/utils.py

@@ -2,6 +2,11 @@ from datetime import datetime
 
 from IPython.display import HTML, display
 
+from letta.local_llm.constants import (
+    ASSISTANT_MESSAGE_CLI_SYMBOL,
+    INNER_THOUGHTS_CLI_SYMBOL,
+)
+
 
 def pprint(messages):
     """Utility function for pretty-printing the output of client.send_message in notebooks"""
@@ -47,13 +52,13 @@ def pprint(messages):
             html_content += f"<p><strong>🛠️ [{date_formatted}] Function Return ({return_status}):</strong></p>"
             html_content += f"<p class='function-return'>{return_string}</p>"
         elif "internal_monologue" in message:
-            html_content += f"<p><strong>💭 [{date_formatted}] Internal Monologue:</strong></p>"
+            html_content += f"<p><strong>{INNER_THOUGHTS_CLI_SYMBOL} [{date_formatted}] Internal Monologue:</strong></p>"
             html_content += f"<p class='internal-monologue'>{message['internal_monologue']}</p>"
         elif "function_call" in message:
             html_content += f"<p><strong>🛠️ [[{date_formatted}] Function Call:</strong></p>"
             html_content += f"<p class='function-call'>{message['function_call']}</p>"
         elif "assistant_message" in message:
-            html_content += f"<p><strong>🤖 [{date_formatted}] Assistant Message:</strong></p>"
+            html_content += f"<p><strong>{ASSISTANT_MESSAGE_CLI_SYMBOL} [{date_formatted}] Assistant Message:</strong></p>"
             html_content += f"<p class='assistant-message'>{message['assistant_message']}</p>"
         html_content += "<br>"
     html_content += "</div>"

{letta_nightly-0.4.1.dev20241009104130 → letta_nightly-0.4.1.dev20241011104054}/letta/constants.py

@@ -75,6 +75,27 @@ NON_USER_MSG_PREFIX = "[This is an automated system message hidden from the user
 LLM_MAX_TOKENS = {
     "DEFAULT": 8192,
     ## OpenAI models: https://platform.openai.com/docs/models/overview
+    # "o1-preview
+    "chatgpt-4o-latest": 128000,
+    # "o1-preview-2024-09-12
+    "gpt-4o-2024-08-06": 128000,
+    "gpt-4-turbo-preview": 128000,
+    "gpt-4o": 128000,
+    "gpt-3.5-turbo-instruct": 16385,
+    "gpt-4-0125-preview": 128000,
+    "gpt-3.5-turbo-0125": 16385,
+    # "babbage-002": 128000,
+    # "davinci-002": 128000,
+    "gpt-4-turbo-2024-04-09": 128000,
+    # "gpt-4o-realtime-preview-2024-10-01
+    "gpt-4-turbo": 8192,
+    "gpt-4o-2024-05-13": 128000,
+    # "o1-mini
+    # "o1-mini-2024-09-12
+    # "gpt-3.5-turbo-instruct-0914
+    "gpt-4o-mini": 128000,
+    # "gpt-4o-realtime-preview
+    "gpt-4o-mini-2024-07-18": 128000,
     # gpt-4
     "gpt-4-1106-preview": 128000,
     "gpt-4": 8192,

{letta_nightly-0.4.1.dev20241009104130 → letta_nightly-0.4.1.dev20241011104054}/letta/embeddings.py

@@ -91,6 +91,9 @@ class EmbeddingEndpoint:
             raise ValueError(
                 f"Embeddings endpoint was provided an invalid URL (set to: '{base_url}'). Make sure embedding_endpoint is set correctly in your Letta config."
             )
+        # TODO: find a neater solution - re-mapping for letta endpoint
+        if model == "letta-free":
+            model = "BAAI/bge-large-en-v1.5"
         self.model_name = model
         self._user = user
         self._base_url = base_url

{letta_nightly-0.4.1.dev20241009104130 → letta_nightly-0.4.1.dev20241011104054}/letta/interface.py

@@ -5,6 +5,10 @@ from typing import List, Optional
 from colorama import Fore, Style, init
 
 from letta.constants import CLI_WARNING_PREFIX
+from letta.local_llm.constants import (
+    ASSISTANT_MESSAGE_CLI_SYMBOL,
+    INNER_THOUGHTS_CLI_SYMBOL,
+)
 from letta.schemas.message import Message
 from letta.utils import json_loads, printd
 
@@ -79,14 +83,14 @@ class CLIInterface(AgentInterface):
     @staticmethod
     def internal_monologue(msg: str, msg_obj: Optional[Message] = None):
         # ANSI escape code for italic is '\x1B[3m'
-        fstr = f"\x1B[3m{Fore.LIGHTBLACK_EX}💭 {{msg}}{Style.RESET_ALL}"
+        fstr = f"\x1B[3m{Fore.LIGHTBLACK_EX}{INNER_THOUGHTS_CLI_SYMBOL} {{msg}}{Style.RESET_ALL}"
         if STRIP_UI:
             fstr = "{msg}"
         print(fstr.format(msg=msg))
 
     @staticmethod
     def assistant_message(msg: str, msg_obj: Optional[Message] = None):
-        fstr = f"{Fore.YELLOW}{Style.BRIGHT}🤖 {Fore.YELLOW}{{msg}}{Style.RESET_ALL}"
+        fstr = f"{Fore.YELLOW}{Style.BRIGHT}{ASSISTANT_MESSAGE_CLI_SYMBOL} {Fore.YELLOW}{{msg}}{Style.RESET_ALL}"
         if STRIP_UI:
             fstr = "{msg}"
         print(fstr.format(msg=msg))

{letta_nightly-0.4.1.dev20241009104130 → letta_nightly-0.4.1.dev20241011104054}/letta/llm_api/google_ai.py

@@ -436,7 +436,7 @@ def google_ai_chat_completions_request(
             response_json=response_json,
             model=data.get("model"),
             input_messages=data["contents"],
-            pull_inner_thoughts_from_args=data.get("inner_thoughts_in_kwargs", False),
+            pull_inner_thoughts_from_args=inner_thoughts_in_kwargs,
         )
     except Exception as conversion_error:
         print(f"Error during response conversion: {conversion_error}")

{letta_nightly-0.4.1.dev20241009104130 → letta_nightly-0.4.1.dev20241011104054}/letta/llm_api/helpers.py

@@ -21,10 +21,17 @@ def make_post_request(url: str, headers: dict[str, str], data: dict[str, Any]) -
         # Raise for 4XX/5XX HTTP errors
         response.raise_for_status()
 
-        # Ensure the content is JSON before parsing
-        if response.headers.get("Content-Type") == "application/json":
-            response_data = response.json()  # Convert to dict from JSON
-            printd(f"Response JSON: {response_data}")
+        # Check if the response content type indicates JSON and attempt to parse it
+        content_type = response.headers.get("Content-Type", "")
+        if "application/json" in content_type.lower():
+            try:
+                response_data = response.json()  # Attempt to parse the response as JSON
+                printd(f"Response JSON: {response_data}")
+            except ValueError as json_err:
+                # Handle the case where the content type says JSON but the body is invalid
+                error_message = f"Failed to parse JSON despite Content-Type being {content_type}: {json_err}"
+                printd(error_message)
+                raise ValueError(error_message) from json_err
         else:
             error_message = f"Unexpected content type returned: {response.headers.get('Content-Type')}"
             printd(error_message)

{letta_nightly-0.4.1.dev20241009104130 → letta_nightly-0.4.1.dev20241011104054}/letta/llm_api/llm_api_tools.py

@@ -217,19 +217,14 @@ def create(
         if not use_tool_naming:
             raise NotImplementedError("Only tool calling supported on Google AI API requests")
 
-        # NOTE: until Google AI supports CoT / text alongside function calls,
-        # we need to put it in a kwarg (unless we want to split the message into two)
-        google_ai_inner_thoughts_in_kwarg = True
-
         if functions is not None:
             tools = [{"type": "function", "function": f} for f in functions]
             tools = [Tool(**t) for t in tools]
-            tools = convert_tools_to_google_ai_format(tools, inner_thoughts_in_kwargs=google_ai_inner_thoughts_in_kwarg)
+            tools = convert_tools_to_google_ai_format(tools, inner_thoughts_in_kwargs=True)
         else:
             tools = None
 
         return google_ai_chat_completions_request(
-            inner_thoughts_in_kwargs=google_ai_inner_thoughts_in_kwarg,
             base_url=llm_config.model_endpoint,
             model=llm_config.model,
             api_key=model_settings.gemini_api_key,
@@ -238,6 +233,7 @@ def create(
                 contents=[m.to_google_ai_dict() for m in messages],
                 tools=tools,
             ),
+            inner_thoughts_in_kwargs=True,
         )
 
     elif llm_config.model_endpoint_type == "anthropic":
@@ -246,12 +242,6 @@ def create(
         if not use_tool_naming:
             raise NotImplementedError("Only tool calling supported on Anthropic API requests")
 
-        if functions is not None:
-            tools = [{"type": "function", "function": f} for f in functions]
-            tools = [Tool(**t) for t in tools]
-        else:
-            tools = None
-
         return anthropic_chat_completions_request(
             url=llm_config.model_endpoint,
             api_key=model_settings.anthropic_api_key,

{letta_nightly-0.4.1.dev20241009104130 → letta_nightly-0.4.1.dev20241011104054}/letta/llm_api/openai.py

@@ -41,7 +41,9 @@ from letta.utils import smart_urljoin
 OPENAI_SSE_DONE = "[DONE]"
 
 
-def openai_get_model_list(url: str, api_key: Union[str, None], fix_url: Optional[bool] = False) -> dict:
+def openai_get_model_list(
+    url: str, api_key: Union[str, None], fix_url: Optional[bool] = False, extra_params: Optional[dict] = None
+) -> dict:
     """https://platform.openai.com/docs/api-reference/models/list"""
     from letta.utils import printd
 
@@ -60,7 +62,8 @@ def openai_get_model_list(url: str, api_key: Union[str, None], fix_url: Optional
 
     printd(f"Sending request to {url}")
     try:
-        response = requests.get(url, headers=headers)
+        # TODO add query param "tool" to be true
+        response = requests.get(url, headers=headers, params=extra_params)
         response.raise_for_status()  # Raises HTTPError for 4XX/5XX status
         response = response.json()  # convert to dict from string
         printd(f"response = {response}")
@@ -145,6 +148,7 @@ def build_openai_chat_completions_request(
         import uuid
 
         data.user = str(uuid.UUID(int=0))
+        data.model = "memgpt-openai"
 
     return data
 

{letta_nightly-0.4.1.dev20241009104130 → letta_nightly-0.4.1.dev20241011104054}/letta/local_llm/constants.py

@@ -29,3 +29,6 @@ DEFAULT_WRAPPER_NAME = "chatml"
 
 INNER_THOUGHTS_KWARG = "inner_thoughts"
 INNER_THOUGHTS_KWARG_DESCRIPTION = "Deep inner monologue private to you only."
+INNER_THOUGHTS_CLI_SYMBOL = "💭"
+
+ASSISTANT_MESSAGE_CLI_SYMBOL = "🤖"

{letta_nightly-0.4.1.dev20241009104130 → letta_nightly-0.4.1.dev20241011104054}/letta/providers.py

@@ -13,7 +13,6 @@ from letta.schemas.llm_config import LLMConfig
 
 
 class Provider(BaseModel):
-    base_url: str
 
     def list_llm_models(self):
         return []
@@ -25,26 +24,69 @@ class Provider(BaseModel):
         pass
 
 
+class LettaProvider(Provider):
+
+    name: str = "letta"
+
+    def list_llm_models(self) -> List[LLMConfig]:
+        return [
+            LLMConfig(
+                model="letta-free",  # NOTE: renamed
+                model_endpoint_type="openai",
+                model_endpoint="https://inference.memgpt.ai",
+                context_window=16384,
+            )
+        ]
+
+    def list_embedding_models(self):
+        return [
+            EmbeddingConfig(
+                embedding_model="letta-free",  # NOTE: renamed
+                embedding_endpoint_type="hugging-face",
+                embedding_endpoint="https://embeddings.memgpt.ai",
+                embedding_dim=1024,
+                embedding_chunk_size=300,
+            )
+        ]
+
+
 class OpenAIProvider(Provider):
     name: str = "openai"
     api_key: str = Field(..., description="API key for the OpenAI API.")
-    base_url: str = "https://api.openai.com/v1"
+    base_url: str = Field(..., description="Base URL for the OpenAI API.")
 
     def list_llm_models(self) -> List[LLMConfig]:
         from letta.llm_api.openai import openai_get_model_list
 
-        response = openai_get_model_list(self.base_url, api_key=self.api_key)
-        model_options = [obj["id"] for obj in response["data"]]
+        # Some hardcoded support for OpenRouter (so that we only get models with tool calling support)...
+        # See: https://openrouter.ai/docs/requests
+        extra_params = {"supported_parameters": "tools"} if "openrouter.ai" in self.base_url else None
+        response = openai_get_model_list(self.base_url, api_key=self.api_key, extra_params=extra_params)
+
+        assert "data" in response, f"OpenAI model query response missing 'data' field: {response}"
 
         configs = []
-        for model_name in model_options:
-            context_window_size = self.get_model_context_window_size(model_name)
+        for model in response["data"]:
+            assert "id" in model, f"OpenAI model missing 'id' field: {model}"
+            model_name = model["id"]
+
+            if "context_length" in model:
+                # Context length is returned in OpenRouter as "context_length"
+                context_window_size = model["context_length"]
+            else:
+                context_window_size = self.get_model_context_window_size(model_name)
 
             if not context_window_size:
                 continue
             configs.append(
                 LLMConfig(model=model_name, model_endpoint_type="openai", model_endpoint=self.base_url, context_window=context_window_size)
             )
+
+        # for OpenAI, sort in reverse order
+        if self.base_url == "https://api.openai.com/v1":
+            # alphnumeric sort
+            configs.sort(key=lambda x: x.model, reverse=True)
+
         return configs
 
     def list_embedding_models(self) -> List[EmbeddingConfig]: