letta-nightly 0.6.14.dev20250123104106__py3-none-any.whl → 0.6.15.dev20250124104035__py3-none-any.whl

letta/llm_api/llm_api_tools.py

@@ -237,6 +237,7 @@ def create(
             data=dict(
                 contents=[m.to_google_ai_dict() for m in messages],
                 tools=tools,
+                generation_config={"temperature": llm_config.temperature},
             ),
             inner_thoughts_in_kwargs=llm_config.put_inner_thoughts_in_kwargs,
         )
@@ -261,6 +262,7 @@ def create(
                 # user=str(user_id),
                 # NOTE: max_tokens is required for Anthropic API
                 max_tokens=1024,  # TODO make dynamic
+                temperature=llm_config.temperature,
             ),
         )
 
@@ -290,7 +292,6 @@ def create(
     #             # max_tokens=1024,  # TODO make dynamic
     #         ),
     #     )
-
     elif llm_config.model_endpoint_type == "groq":
         if stream:
             raise NotImplementedError(f"Streaming not yet implemented for Groq.")
@@ -329,7 +330,6 @@ def create(
         try:
             # groq uses the openai chat completions API, so this component should be reusable
             response = openai_chat_completions_request(
-                url=llm_config.model_endpoint,
                 api_key=model_settings.groq_api_key,
                 chat_completion_request=data,
             )

letta/llm_api/openai.py

@@ -1,14 +1,9 @@
-import json
 import warnings
 from typing import Generator, List, Optional, Union
 
-import httpx
 import requests
-from httpx_sse import connect_sse
-from httpx_sse._exceptions import SSEError
+from openai import OpenAI
 
-from letta.constants import OPENAI_CONTEXT_WINDOW_ERROR_SUBSTRING
-from letta.errors import LLMError
 from letta.llm_api.helpers import add_inner_thoughts_to_functions, convert_to_structured_output, make_post_request
 from letta.local_llm.constants import INNER_THOUGHTS_KWARG, INNER_THOUGHTS_KWARG_DESCRIPTION
 from letta.local_llm.utils import num_tokens_from_functions, num_tokens_from_messages
@@ -130,7 +125,8 @@ def build_openai_chat_completions_request(
             tools=[Tool(type="function", function=f) for f in functions] if functions else None,
             tool_choice=tool_choice,
             user=str(user_id),
-            max_tokens=max_tokens,
+            max_completion_tokens=max_tokens,
+            temperature=llm_config.temperature,
         )
     else:
         data = ChatCompletionRequest(
@@ -139,7 +135,8 @@ def build_openai_chat_completions_request(
             functions=functions,
             function_call=function_call,
             user=str(user_id),
-            max_tokens=max_tokens,
+            max_completion_tokens=max_tokens,
+            temperature=llm_config.temperature,
         )
         # https://platform.openai.com/docs/guides/text-generation/json-mode
         # only supported by gpt-4o, gpt-4-turbo, or gpt-3.5-turbo
@@ -378,126 +375,21 @@ def openai_chat_completions_process_stream(
     return chat_completion_response
 
 
-def _sse_post(url: str, data: dict, headers: dict) -> Generator[ChatCompletionChunkResponse, None, None]:
-
-    with httpx.Client() as client:
-        with connect_sse(client, method="POST", url=url, json=data, headers=headers) as event_source:
-
-            # Inspect for errors before iterating (see https://github.com/florimondmanca/httpx-sse/pull/12)
-            if not event_source.response.is_success:
-                # handle errors
-                from letta.utils import printd
-
-                printd("Caught error before iterating SSE request:", vars(event_source.response))
-                printd(event_source.response.read())
-
-                try:
-                    response_bytes = event_source.response.read()
-                    response_dict = json.loads(response_bytes.decode("utf-8"))
-                    error_message = response_dict["error"]["message"]
-                    # e.g.: This model's maximum context length is 8192 tokens. However, your messages resulted in 8198 tokens (7450 in the messages, 748 in the functions). Please reduce the length of the messages or functions.
-                    if OPENAI_CONTEXT_WINDOW_ERROR_SUBSTRING in error_message:
-                        raise LLMError(error_message)
-                except LLMError:
-                    raise
-                except:
-                    print(f"Failed to parse SSE message, throwing SSE HTTP error up the stack")
-                    event_source.response.raise_for_status()
-
-            try:
-                for sse in event_source.iter_sse():
-                    # printd(sse.event, sse.data, sse.id, sse.retry)
-                    if sse.data == OPENAI_SSE_DONE:
-                        # print("finished")
-                        break
-                    else:
-                        chunk_data = json.loads(sse.data)
-                        # print("chunk_data::", chunk_data)
-                        chunk_object = ChatCompletionChunkResponse(**chunk_data)
-                        # print("chunk_object::", chunk_object)
-                        # id=chunk_data["id"],
-                        # choices=[ChunkChoice],
-                        # model=chunk_data["model"],
-                        # system_fingerprint=chunk_data["system_fingerprint"]
-                        # )
-                        yield chunk_object
-
-            except SSEError as e:
-                print("Caught an error while iterating the SSE stream:", str(e))
-                if "application/json" in str(e):  # Check if the error is because of JSON response
-                    # TODO figure out a better way to catch the error other than re-trying with a POST
-                    response = client.post(url=url, json=data, headers=headers)  # Make the request again to get the JSON response
-                    if response.headers["Content-Type"].startswith("application/json"):
-                        error_details = response.json()  # Parse the JSON to get the error message
-                        print("Request:", vars(response.request))
-                        print("POST Error:", error_details)
-                        print("Original SSE Error:", str(e))
-                    else:
-                        print("Failed to retrieve JSON error message via retry.")
-                else:
-                    print("SSEError not related to 'application/json' content type.")
-
-                # Optionally re-raise the exception if you need to propagate it
-                raise e
-
-            except Exception as e:
-                if event_source.response.request is not None:
-                    print("HTTP Request:", vars(event_source.response.request))
-                if event_source.response is not None:
-                    print("HTTP Status:", event_source.response.status_code)
-                    print("HTTP Headers:", event_source.response.headers)
-                    # print("HTTP Body:", event_source.response.text)
-                print("Exception message:", str(e))
-                raise e
-
-
 def openai_chat_completions_request_stream(
     url: str,
     api_key: str,
     chat_completion_request: ChatCompletionRequest,
 ) -> Generator[ChatCompletionChunkResponse, None, None]:
-    from letta.utils import printd
-
-    url = smart_urljoin(url, "chat/completions")
-    headers = {"Content-Type": "application/json", "Authorization": f"Bearer {api_key}"}
-    data = chat_completion_request.model_dump(exclude_none=True)
-
-    printd("Request:\n", json.dumps(data, indent=2))
-
-    # If functions == None, strip from the payload
-    if "functions" in data and data["functions"] is None:
-        data.pop("functions")
-        data.pop("function_call", None)  # extra safe,  should exist always (default="auto")
-
-    if "tools" in data and data["tools"] is None:
-        data.pop("tools")
-        data.pop("tool_choice", None)  # extra safe,  should exist always (default="auto")
-
-    if "tools" in data:
-        for tool in data["tools"]:
-            # tool["strict"] = True
-            try:
-                tool["function"] = convert_to_structured_output(tool["function"])
-            except ValueError as e:
-                warnings.warn(f"Failed to convert tool function to structured output, tool={tool}, error={e}")
-
-    # print(f"\n\n\n\nData[tools]: {json.dumps(data['tools'], indent=2)}")
-
-    printd(f"Sending request to {url}")
-    try:
-        return _sse_post(url=url, data=data, headers=headers)
-    except requests.exceptions.HTTPError as http_err:
-        # Handle HTTP errors (e.g., response 4XX, 5XX)
-        printd(f"Got HTTPError, exception={http_err}, payload={data}")
-        raise http_err
-    except requests.exceptions.RequestException as req_err:
-        # Handle other requests-related errors (e.g., connection error)
-        printd(f"Got RequestException, exception={req_err}")
-        raise req_err
-    except Exception as e:
-        # Handle other potential errors
-        printd(f"Got unknown Exception, exception={e}")
-        raise e
+    data = prepare_openai_payload(chat_completion_request)
+    data["stream"] = True
+    client = OpenAI(
+        api_key=api_key,
+        base_url=url,
+    )
+    stream = client.chat.completions.create(**data)
+    for chunk in stream:
+        # TODO: Use the native OpenAI objects here?
+        yield ChatCompletionChunkResponse(**chunk.model_dump(exclude_none=True))
 
 
 def openai_chat_completions_request(
@@ -512,18 +404,28 @@ def openai_chat_completions_request(
 
     https://platform.openai.com/docs/guides/text-generation?lang=curl
     """
-    from letta.utils import printd
+    data = prepare_openai_payload(chat_completion_request)
+    client = OpenAI(api_key=api_key, base_url=url)
+    chat_completion = client.chat.completions.create(**data)
+    return ChatCompletionResponse(**chat_completion.model_dump())
+
+
+def openai_embeddings_request(url: str, api_key: str, data: dict) -> EmbeddingResponse:
+    """https://platform.openai.com/docs/api-reference/embeddings/create"""
 
-    url = smart_urljoin(url, "chat/completions")
+    url = smart_urljoin(url, "embeddings")
     headers = {"Content-Type": "application/json", "Authorization": f"Bearer {api_key}"}
+    response_json = make_post_request(url, headers, data)
+    return EmbeddingResponse(**response_json)
+
+
+def prepare_openai_payload(chat_completion_request: ChatCompletionRequest):
     data = chat_completion_request.model_dump(exclude_none=True)
 
     # add check otherwise will cause error: "Invalid value for 'parallel_tool_calls': 'parallel_tool_calls' is only allowed when 'tools' are specified."
     if chat_completion_request.tools is not None:
         data["parallel_tool_calls"] = False
 
-    printd("Request:\n", json.dumps(data, indent=2))
-
     # If functions == None, strip from the payload
     if "functions" in data and data["functions"] is None:
         data.pop("functions")
@@ -540,14 +442,4 @@ def openai_chat_completions_request(
             except ValueError as e:
                 warnings.warn(f"Failed to convert tool function to structured output, tool={tool}, error={e}")
 
-    response_json = make_post_request(url, headers, data)
-    return ChatCompletionResponse(**response_json)
-
-
-def openai_embeddings_request(url: str, api_key: str, data: dict) -> EmbeddingResponse:
-    """https://platform.openai.com/docs/api-reference/embeddings/create"""
-
-    url = smart_urljoin(url, "embeddings")
-    headers = {"Content-Type": "application/json", "Authorization": f"Bearer {api_key}"}
-    response_json = make_post_request(url, headers, data)
-    return EmbeddingResponse(**response_json)
+    return data

letta/memory.py

@@ -6,7 +6,7 @@ from letta.prompts.gpt_summarize import SYSTEM as SUMMARY_PROMPT_SYSTEM
 from letta.schemas.agent import AgentState
 from letta.schemas.enums import MessageRole
 from letta.schemas.memory import Memory
-from letta.schemas.message import Message
+from letta.schemas.message import Message, TextContent
 from letta.settings import summarizer_settings
 from letta.utils import count_tokens, printd
 
@@ -60,9 +60,9 @@ def summarize_messages(
 
     dummy_agent_id = agent_state.id
     message_sequence = [
-        Message(agent_id=dummy_agent_id, role=MessageRole.system, text=summary_prompt),
-        Message(agent_id=dummy_agent_id, role=MessageRole.assistant, text=MESSAGE_SUMMARY_REQUEST_ACK),
-        Message(agent_id=dummy_agent_id, role=MessageRole.user, text=summary_input),
+        Message(agent_id=dummy_agent_id, role=MessageRole.system, content=[TextContent(text=summary_prompt)]),
+        Message(agent_id=dummy_agent_id, role=MessageRole.assistant, content=[TextContent(text=MESSAGE_SUMMARY_REQUEST_ACK)]),
+        Message(agent_id=dummy_agent_id, role=MessageRole.user, content=[TextContent(text=summary_input)]),
     ]
 
     # TODO: We need to eventually have a separate LLM config for the summarizer LLM

letta/offline_memory_agent.py

@@ -8,12 +8,12 @@ from letta.schemas.openai.chat_completion_response import UsageStatistics
 from letta.schemas.usage import LettaUsageStatistics
 
 
-def trigger_rethink_memory(agent_state: "AgentState", message: Optional[str]) -> Optional[str]:  # type: ignore
+def trigger_rethink_memory(agent_state: "AgentState", message: str) -> None:  # type: ignore
     """
     Called if and only when user says the word trigger_rethink_memory". It will trigger the re-evaluation of the memory.
 
     Args:
-        message (Optional[str]): Description of what aspect of the memory should be re-evaluated.
+        message (str): Description of what aspect of the memory should be re-evaluated.
 
     """
     from letta import create_client
@@ -25,12 +25,12 @@ def trigger_rethink_memory(agent_state: "AgentState", message: Optional[str]) ->
             client.user_message(agent_id=agent.id, message=message)
 
 
-def trigger_rethink_memory_convo(agent_state: "AgentState", message: Optional[str]) -> Optional[str]:  # type: ignore
+def trigger_rethink_memory_convo(agent_state: "AgentState", message: str) -> None:  # type: ignore
     """
     Called if and only when user says the word "trigger_rethink_memory". It will trigger the re-evaluation of the memory.
 
     Args:
-        message (Optional[str]): Description of what aspect of the memory should be re-evaluated.
+        message (str): Description of what aspect of the memory should be re-evaluated.
 
     """
     from letta import create_client
@@ -48,7 +48,7 @@ def trigger_rethink_memory_convo(agent_state: "AgentState", message: Optional[st
             client.user_message(agent_id=agent.id, message=message)
 
 
-def rethink_memory_convo(agent_state: "AgentState", new_memory: str, target_block_label: Optional[str], source_block_label: Optional[str]) -> Optional[str]:  # type: ignore
+def rethink_memory_convo(agent_state: "AgentState", new_memory: str, target_block_label: str, source_block_label: str) -> None:  # type: ignore
     """
     Re-evaluate the memory in block_name, integrating new and updated facts. Replace outdated information with the most likely truths, avoiding redundancy with original memories. Ensure consistency with other memory blocks.
 
@@ -58,7 +58,7 @@ def rethink_memory_convo(agent_state: "AgentState", new_memory: str, target_bloc
         target_block_label (str): The name of the block to write to. This should be chat_agent_human_new or chat_agent_persona_new.
 
     Returns:
-        Optional[str]: None is always returned as this function does not produce a response.
+        None: None is always returned as this function does not produce a response.
     """
     if target_block_label is not None:
         if agent_state.memory.get_block(target_block_label) is None:
@@ -67,7 +67,7 @@ def rethink_memory_convo(agent_state: "AgentState", new_memory: str, target_bloc
     return None
 
 
-def rethink_memory(agent_state: "AgentState", new_memory: str, target_block_label: Optional[str], source_block_label: Optional[str]) -> Optional[str]:  # type: ignore
+def rethink_memory(agent_state: "AgentState", new_memory: str, target_block_label: str, source_block_label: str) -> None:  # type: ignore
     """
     Re-evaluate the memory in block_name, integrating new and updated facts.
     Replace outdated information with the most likely truths, avoiding redundancy with original memories.
@@ -78,7 +78,7 @@ def rethink_memory(agent_state: "AgentState", new_memory: str, target_block_labe
         source_block_label (str): The name of the block to integrate information from. None if all the information has been integrated to terminate the loop.
         target_block_label (str): The name of the block to write to.
     Returns:
-        Optional[str]: None is always returned as this function does not produce a response.
+        None: None is always returned as this function does not produce a response.
     """
 
     if target_block_label is not None:
@@ -88,7 +88,7 @@ def rethink_memory(agent_state: "AgentState", new_memory: str, target_block_labe
     return None
 
 
-def finish_rethinking_memory(agent_state: "AgentState") -> Optional[str]:  # type: ignore
+def finish_rethinking_memory(agent_state: "AgentState") -> None:  # type: ignore
     """
     This function is called when the agent is done rethinking the memory.
 
@@ -98,7 +98,7 @@ def finish_rethinking_memory(agent_state: "AgentState") -> Optional[str]:  # typ
     return None
 
 
-def finish_rethinking_memory_convo(agent_state: "AgentState") -> Optional[str]:  # type: ignore
+def finish_rethinking_memory_convo(agent_state: "AgentState") -> None:  # type: ignore
     """
     This function is called when the agent is done rethinking the memory.
 

letta/orm/agent.py

@@ -1,7 +1,7 @@
 import uuid
 from typing import TYPE_CHECKING, List, Optional
 
-from sqlalchemy import JSON, String
+from sqlalchemy import JSON, Index, String
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 
 from letta.orm.block import Block
@@ -27,6 +27,7 @@ if TYPE_CHECKING:
 class Agent(SqlalchemyBase, OrganizationMixin):
     __tablename__ = "agents"
     __pydantic_model__ = PydanticAgentState
+    __table_args__ = (Index("ix_agents_created_at", "created_at", "id"),)
 
     # agent generates its own id
     # TODO: We want to migrate all the ORM models to do this, so we will need to move this to the SqlalchemyBase
@@ -69,7 +70,14 @@ class Agent(SqlalchemyBase, OrganizationMixin):
     )
     tools: Mapped[List["Tool"]] = relationship("Tool", secondary="tools_agents", lazy="selectin", passive_deletes=True)
     sources: Mapped[List["Source"]] = relationship("Source", secondary="sources_agents", lazy="selectin")
-    core_memory: Mapped[List["Block"]] = relationship("Block", secondary="blocks_agents", lazy="selectin")
+    core_memory: Mapped[List["Block"]] = relationship(
+        "Block",
+        secondary="blocks_agents",
+        lazy="selectin",
+        passive_deletes=True,  # Ensures SQLAlchemy doesn't fetch blocks_agents rows before deleting
+        back_populates="agents",
+        doc="Blocks forming the core memory of the agent.",
+    )
     messages: Mapped[List["Message"]] = relationship(
         "Message",
         back_populates="agent",

letta/orm/block.py

@@ -1,6 +1,6 @@
-from typing import TYPE_CHECKING, Optional, Type
+from typing import TYPE_CHECKING, List, Optional, Type
 
-from sqlalchemy import JSON, BigInteger, Integer, UniqueConstraint, event
+from sqlalchemy import JSON, BigInteger, Index, Integer, UniqueConstraint, event
 from sqlalchemy.orm import Mapped, attributes, mapped_column, relationship
 
 from letta.constants import CORE_MEMORY_BLOCK_CHAR_LIMIT
@@ -20,7 +20,10 @@ class Block(OrganizationMixin, SqlalchemyBase):
     __tablename__ = "block"
     __pydantic_model__ = PydanticBlock
     # This may seem redundant, but is necessary for the BlocksAgents composite FK relationship
-    __table_args__ = (UniqueConstraint("id", "label", name="unique_block_id_label"),)
+    __table_args__ = (
+        UniqueConstraint("id", "label", name="unique_block_id_label"),
+        Index("created_at_label_idx", "created_at", "label"),
+    )
 
     template_name: Mapped[Optional[str]] = mapped_column(
         nullable=True, doc="the unique name that identifies a block in a human-readable way"
@@ -36,6 +39,14 @@ class Block(OrganizationMixin, SqlalchemyBase):
 
     # relationships
     organization: Mapped[Optional["Organization"]] = relationship("Organization")
+    agents: Mapped[List["Agent"]] = relationship(
+        "Agent",
+        secondary="blocks_agents",
+        lazy="selectin",
+        passive_deletes=True,  # Ensures SQLAlchemy doesn't fetch blocks_agents rows before deleting
+        back_populates="core_memory",
+        doc="Agents associated with this block.",
+    )
 
     def to_pydantic(self) -> Type:
         match self.label:

letta/orm/job.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 from typing import TYPE_CHECKING, List, Optional
 
-from sqlalchemy import JSON, String
+from sqlalchemy import JSON, Index, String
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 
 from letta.orm.enums import JobType
@@ -25,6 +25,7 @@ class Job(SqlalchemyBase, UserMixin):
 
     __tablename__ = "jobs"
     __pydantic_model__ = PydanticJob
+    __table_args__ = (Index("ix_jobs_created_at", "created_at", "id"),)
 
     status: Mapped[JobStatus] = mapped_column(String, default=JobStatus.created, doc="The current status of the job.")
     completed_at: Mapped[Optional[datetime]] = mapped_column(nullable=True, doc="The unix timestamp of when the job was completed.")

letta/orm/message.py

@@ -8,13 +8,17 @@ from letta.orm.custom_columns import ToolCallColumn
 from letta.orm.mixins import AgentMixin, OrganizationMixin
 from letta.orm.sqlalchemy_base import SqlalchemyBase
 from letta.schemas.message import Message as PydanticMessage
+from letta.schemas.message import TextContent as PydanticTextContent
 
 
 class Message(SqlalchemyBase, OrganizationMixin, AgentMixin):
     """Defines data model for storing Message objects"""
 
     __tablename__ = "messages"
-    __table_args__ = (Index("ix_messages_agent_created_at", "agent_id", "created_at"),)
+    __table_args__ = (
+        Index("ix_messages_agent_created_at", "agent_id", "created_at"),
+        Index("ix_messages_created_at", "created_at", "id"),
+    )
     __pydantic_model__ = PydanticMessage
 
     id: Mapped[str] = mapped_column(primary_key=True, doc="Unique message identifier")
@@ -42,3 +46,10 @@ class Message(SqlalchemyBase, OrganizationMixin, AgentMixin):
     def job(self) -> Optional["Job"]:
         """Get the job associated with this message, if any."""
         return self.job_message.job if self.job_message else None
+
+    def to_pydantic(self) -> PydanticMessage:
+        """custom pydantic conversion for message content mapping"""
+        model = self.__pydantic_model__.model_validate(self)
+        if self.text:
+            model.content = [PydanticTextContent(text=self.text)]
+        return model

letta/orm/passage.py

@@ -45,8 +45,12 @@ class BasePassage(SqlalchemyBase, OrganizationMixin):
     @declared_attr
     def __table_args__(cls):
         if settings.letta_pg_uri_no_default:
-            return (Index(f"{cls.__tablename__}_org_idx", "organization_id"), {"extend_existing": True})
-        return ({"extend_existing": True},)
+            return (
+                Index(f"{cls.__tablename__}_org_idx", "organization_id"),
+                Index(f"{cls.__tablename__}_created_at_id_idx", "created_at", "id"),
+                {"extend_existing": True},
+            )
+        return (Index(f"{cls.__tablename__}_created_at_id_idx", "created_at", "id"), {"extend_existing": True})
 
 
 class SourcePassage(BasePassage, FileMixin, SourceMixin):

letta/orm/source.py

@@ -1,6 +1,6 @@
 from typing import TYPE_CHECKING, List, Optional
 
-from sqlalchemy import JSON
+from sqlalchemy import JSON, Index
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 
 from letta.orm import FileMetadata
@@ -23,6 +23,11 @@ class Source(SqlalchemyBase, OrganizationMixin):
     __tablename__ = "sources"
     __pydantic_model__ = PydanticSource
 
+    __table_args__ = (
+        Index(f"source_created_at_id_idx", "created_at", "id"),
+        {"extend_existing": True},
+    )
+
     name: Mapped[str] = mapped_column(doc="the name of the source, must be unique within the org", nullable=False)
     description: Mapped[str] = mapped_column(nullable=True, doc="a human-readable description of the source")
     embedding_config: Mapped[EmbeddingConfig] = mapped_column(EmbeddingConfigColumn, doc="Configuration settings for embedding.")