letta-nightly 0.7.0.dev20250423003112__py3-none-any.whl → 0.7.1.dev20250423104245__py3-none-any.whl

letta/groups/supervisor_multi_agent.py

@@ -9,7 +9,7 @@ from letta.interface import AgentInterface
 from letta.orm import User
 from letta.orm.enums import ToolType
 from letta.schemas.letta_message_content import TextContent
-from letta.schemas.message import Message, MessageCreate
+from letta.schemas.message import MessageCreate
 from letta.schemas.tool import Tool
 from letta.schemas.tool_rule import ChildToolRule, InitToolRule, TerminalToolRule
 from letta.schemas.usage import LettaUsageStatistics
@@ -37,17 +37,18 @@ class SupervisorMultiAgent(Agent):
 
     def step(
         self,
-        messages: List[MessageCreate],
+        input_messages: List[MessageCreate],
         chaining: bool = True,
         max_chaining_steps: Optional[int] = None,
         put_inner_thoughts_first: bool = True,
         assistant_message_tool_name: str = DEFAULT_MESSAGE_TOOL,
         **kwargs,
     ) -> LettaUsageStatistics:
+        # Load settings
         token_streaming = self.interface.streaming_mode if hasattr(self.interface, "streaming_mode") else False
         metadata = self.interface.metadata if hasattr(self.interface, "metadata") else None
 
-        # add multi agent tool
+        # Prepare supervisor agent
         if self.tool_manager.get_tool_by_name(tool_name="send_message_to_all_agents_in_group", actor=self.user) is None:
             multi_agent_tool = Tool(
                 name=send_message_to_all_agents_in_group.__name__,
@@ -64,7 +65,6 @@ class SupervisorMultiAgent(Agent):
             )
             self.agent_state = self.agent_manager.attach_tool(agent_id=self.agent_state.id, tool_id=multi_agent_tool.id, actor=self.user)
 
-        # override tool rules
         old_tool_rules = self.agent_state.tool_rules
         self.agent_state.tool_rules = [
             InitToolRule(
@@ -79,24 +79,25 @@ class SupervisorMultiAgent(Agent):
             ),
         ]
 
-        supervisor_messages = [
-            Message(
-                agent_id=self.agent_state.id,
-                role="user",
-                content=[TextContent(text=message.content)],
-                name=None,
-                model=None,
-                tool_calls=None,
-                tool_call_id=None,
-                group_id=self.group_id,
-                otid=message.otid,
-            )
-            for message in messages
-        ]
+        # Prepare new messages
+        new_messages = []
+        for message in input_messages:
+            if isinstance(message.content, str):
+                message.content = [TextContent(text=message.content)]
+            message.group_id = self.group_id
+            new_messages.append(message)
+
         try:
-            supervisor_agent = Agent(agent_state=self.agent_state, interface=self.interface, user=self.user)
+            # Load supervisor agent
+            supervisor_agent = Agent(
+                agent_state=self.agent_state,
+                interface=self.interface,
+                user=self.user,
+            )
+
+            # Perform supervisor step
             usage_stats = supervisor_agent.step(
-                messages=supervisor_messages,
+                input_messages=new_messages,
                 chaining=chaining,
                 max_chaining_steps=max_chaining_steps,
                 stream=token_streaming,

letta/helpers/converters.py

@@ -22,6 +22,13 @@ from letta.schemas.letta_message_content import (
 )
 from letta.schemas.llm_config import LLMConfig
 from letta.schemas.message import ToolReturn
+from letta.schemas.response_format import (
+    JsonObjectResponseFormat,
+    JsonSchemaResponseFormat,
+    ResponseFormatType,
+    ResponseFormatUnion,
+    TextResponseFormat,
+)
 from letta.schemas.tool_rule import (
     ChildToolRule,
     ConditionalToolRule,
@@ -371,3 +378,25 @@ def deserialize_agent_step_state(data: Optional[Dict]) -> Optional[AgentStepStat
         return None
 
     return AgentStepState(**data)
+
+
+# --------------------------
+# Response Format Serialization
+# --------------------------
+
+
+def serialize_response_format(response_format: Optional[ResponseFormatUnion]) -> Optional[Dict[str, Any]]:
+    if not response_format:
+        return None
+    return response_format.model_dump(mode="json")
+
+
+def deserialize_response_format(data: Optional[Dict]) -> Optional[ResponseFormatUnion]:
+    if not data:
+        return None
+    if data["type"] == ResponseFormatType.text:
+        return TextResponseFormat(**data)
+    if data["type"] == ResponseFormatType.json_schema:
+        return JsonSchemaResponseFormat(**data)
+    if data["type"] == ResponseFormatType.json_object:
+        return JsonObjectResponseFormat(**data)

letta/helpers/message_helper.py

@@ -40,4 +40,5 @@ def prepare_input_message_create(
         tool_call_id=None,
         otid=message.otid,
         sender_id=message.sender_id,
+        group_id=message.group_id,
     )

letta/helpers/tool_execution_helper.py

@@ -160,12 +160,12 @@ def execute_external_tool(
             else:
                 agent_state_copy = None
 
-            sandbox_run_result = ToolExecutionSandbox(function_name, function_args, actor).run(agent_state=agent_state_copy)
-            function_response, updated_agent_state = sandbox_run_result.func_return, sandbox_run_result.agent_state
+            tool_execution_result = ToolExecutionSandbox(function_name, function_args, actor).run(agent_state=agent_state_copy)
+            function_response, updated_agent_state = tool_execution_result.func_return, tool_execution_result.agent_state
             # TODO: Bring this back
             # if allow_agent_state_modifications and updated_agent_state is not None:
             #     self.update_memory_if_changed(updated_agent_state.memory)
-            return function_response, sandbox_run_result
+            return function_response, tool_execution_result
     except Exception as e:
         # Need to catch error here, or else trunction wont happen
         # TODO: modify to function execution error

letta/orm/agent.py

@@ -5,7 +5,7 @@ from sqlalchemy import JSON, Boolean, Index, String
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 
 from letta.orm.block import Block
-from letta.orm.custom_columns import EmbeddingConfigColumn, LLMConfigColumn, ToolRulesColumn
+from letta.orm.custom_columns import EmbeddingConfigColumn, LLMConfigColumn, ResponseFormatColumn, ToolRulesColumn
 from letta.orm.identity import Identity
 from letta.orm.mixins import OrganizationMixin
 from letta.orm.organization import Organization
@@ -15,6 +15,7 @@ from letta.schemas.agent import AgentType, get_prompt_template_for_agent_type
 from letta.schemas.embedding_config import EmbeddingConfig
 from letta.schemas.llm_config import LLMConfig
 from letta.schemas.memory import Memory
+from letta.schemas.response_format import ResponseFormatUnion
 from letta.schemas.tool_rule import ToolRule
 
 if TYPE_CHECKING:
@@ -48,6 +49,11 @@ class Agent(SqlalchemyBase, OrganizationMixin):
     # This is dangerously flexible with the JSON type
     message_ids: Mapped[Optional[List[str]]] = mapped_column(JSON, nullable=True, doc="List of message IDs in in-context memory.")
 
+    # Response Format
+    response_format: Mapped[Optional[ResponseFormatUnion]] = mapped_column(
+        ResponseFormatColumn, nullable=True, doc="The response format for the agent."
+    )
+
     # Metadata and configs
     metadata_: Mapped[Optional[dict]] = mapped_column(JSON, nullable=True, doc="metadata for the agent.")
     llm_config: Mapped[Optional[LLMConfig]] = mapped_column(
@@ -168,6 +174,7 @@ class Agent(SqlalchemyBase, OrganizationMixin):
             "multi_agent_group": None,
             "tool_exec_environment_variables": [],
             "enable_sleeptime": None,
+            "response_format": self.response_format,
         }
 
         # Optional fields: only included if requested

letta/orm/custom_columns.py

@@ -9,6 +9,7 @@ from letta.helpers.converters import (
     deserialize_llm_config,
     deserialize_message_content,
     deserialize_poll_batch_response,
+    deserialize_response_format,
     deserialize_tool_calls,
     deserialize_tool_returns,
     deserialize_tool_rules,
@@ -20,6 +21,7 @@ from letta.helpers.converters import (
     serialize_llm_config,
     serialize_message_content,
     serialize_poll_batch_response,
+    serialize_response_format,
     serialize_tool_calls,
     serialize_tool_returns,
     serialize_tool_rules,
@@ -168,3 +170,16 @@ class AgentStepStateColumn(TypeDecorator):
 
     def process_result_value(self, value, dialect):
         return deserialize_agent_step_state(value)
+
+
+class ResponseFormatColumn(TypeDecorator):
+    """Custom SQLAlchemy column type for storing a list of ToolRules as JSON."""
+
+    impl = JSON
+    cache_ok = True
+
+    def process_bind_param(self, value, dialect):
+        return serialize_response_format(value)
+
+    def process_result_value(self, value, dialect):
+        return deserialize_response_format(value)

letta/schemas/agent.py

@@ -14,6 +14,7 @@ from letta.schemas.llm_config import LLMConfig
 from letta.schemas.memory import Memory
 from letta.schemas.message import Message, MessageCreate
 from letta.schemas.openai.chat_completion_response import UsageStatistics
+from letta.schemas.response_format import ResponseFormatUnion
 from letta.schemas.source import Source
 from letta.schemas.tool import Tool
 from letta.schemas.tool_rule import ToolRule
@@ -66,6 +67,9 @@ class AgentState(OrmMetadataBase, validate_assignment=True):
     # llm information
     llm_config: LLMConfig = Field(..., description="The LLM configuration used by the agent.")
     embedding_config: EmbeddingConfig = Field(..., description="The embedding configuration used by the agent.")
+    response_format: Optional[ResponseFormatUnion] = Field(
+        None, description="The response format used by the agent when returning from `send_message`."
+    )
 
     # This is an object representing the in-process state of a running `Agent`
     # Field in this object can be theoretically edited by tools, and will be persisted by the ORM
@@ -180,6 +184,7 @@ class CreateAgent(BaseModel, validate_assignment=True):  #
         description="If set to True, the agent will not remember previous messages (though the agent will still retain state via core memory blocks and archival/recall memory). Not recommended unless you have an advanced use case.",
     )
     enable_sleeptime: Optional[bool] = Field(None, description="If set to True, memory management will move to a background agent thread.")
+    response_format: Optional[ResponseFormatUnion] = Field(None, description="The response format for the agent.")
 
     @field_validator("name")
     @classmethod
@@ -259,6 +264,7 @@ class UpdateAgent(BaseModel):
         None, description="The embedding configuration handle used by the agent, specified in the format provider/model-name."
     )
     enable_sleeptime: Optional[bool] = Field(None, description="If set to True, memory management will move to a background agent thread.")
+    response_format: Optional[ResponseFormatUnion] = Field(None, description="The response format for the agent.")
 
     class Config:
         extra = "ignore"  # Ignores extra fields

letta/schemas/message.py

@@ -82,6 +82,7 @@ class MessageCreate(BaseModel):
     name: Optional[str] = Field(None, description="The name of the participant.")
     otid: Optional[str] = Field(None, description="The offline threading id associated with this message")
     sender_id: Optional[str] = Field(None, description="The id of the sender of the message, can be an identity id or agent id")
+    group_id: Optional[str] = Field(None, description="The multi-agent group that the message was sent in")
 
     def model_dump(self, to_orm: bool = False, **kwargs) -> Dict[str, Any]:
         data = super().model_dump(**kwargs)

letta/schemas/response_format.py

@@ -0,0 +1,78 @@
+from enum import Enum
+from typing import Annotated, Any, Dict, Literal, Union
+
+from pydantic import BaseModel, Field, validator
+
+
+class ResponseFormatType(str, Enum):
+    """Enum defining the possible response format types."""
+
+    text = "text"
+    json_schema = "json_schema"
+    json_object = "json_object"
+
+
+class ResponseFormat(BaseModel):
+    """Base class for all response formats."""
+
+    type: ResponseFormatType = Field(
+        ...,
+        description="The type of the response format.",
+        # why use this?
+        example=ResponseFormatType.text,
+    )
+
+
+# ---------------------
+# Response Format Types
+# ---------------------
+
+# SQLAlchemy type for database mapping
+ResponseFormatDict = Dict[str, Any]
+
+
+class TextResponseFormat(ResponseFormat):
+    """Response format for plain text responses."""
+
+    type: Literal[ResponseFormatType.text] = Field(
+        ResponseFormatType.text,
+        description="The type of the response format.",
+    )
+
+
+class JsonSchemaResponseFormat(ResponseFormat):
+    """Response format for JSON schema-based responses."""
+
+    type: Literal[ResponseFormatType.json_schema] = Field(
+        ResponseFormatType.json_schema,
+        description="The type of the response format.",
+    )
+    json_schema: Dict[str, Any] = Field(
+        ...,
+        description="The JSON schema of the response.",
+    )
+
+    @validator("json_schema")
+    def validate_json_schema(cls, v: Dict[str, Any]) -> Dict[str, Any]:
+        """Validate that the provided schema is a valid JSON schema."""
+        if not isinstance(v, dict):
+            raise ValueError("JSON schema must be a dictionary")
+        if "schema" not in v:
+            raise ValueError("JSON schema should include a $schema property")
+        return v
+
+
+class JsonObjectResponseFormat(ResponseFormat):
+    """Response format for JSON object responses."""
+
+    type: Literal[ResponseFormatType.json_object] = Field(
+        ResponseFormatType.json_object,
+        description="The type of the response format.",
+    )
+
+
+# Pydantic type for validation
+ResponseFormatUnion = Annotated[
+    Union[TextResponseFormat | JsonSchemaResponseFormat | JsonObjectResponseFormat],
+    Field(discriminator="type"),
+]

letta/schemas/tool_execution_result.py

@@ -0,0 +1,14 @@
+from typing import Any, List, Literal, Optional
+
+from pydantic import BaseModel, Field
+
+from letta.schemas.agent import AgentState
+
+
+class ToolExecutionResult(BaseModel):
+    status: Literal["success", "error"] = Field(..., description="The status of the tool execution and return object")
+    func_return: Optional[Any] = Field(None, description="The function return object")
+    agent_state: Optional[AgentState] = Field(None, description="The agent state")
+    stdout: Optional[List[str]] = Field(None, description="Captured stdout (prints, logs) from function invocation")
+    stderr: Optional[List[str]] = Field(None, description="Captured stderr from the function invocation")
+    sandbox_config_fingerprint: Optional[str] = Field(None, description="The fingerprint of the config for the sandbox")

letta/server/rest_api/interface.py

@@ -1240,10 +1240,11 @@ class StreamingServerInterface(AgentChunkStreamingInterface):
                         and function_call.function.name == self.assistant_message_tool_name
                         and self.assistant_message_tool_kwarg in func_args
                     ):
+                        # Coerce content to `str` in cases where it's a JSON due to `response_format` being a JSON
                         processed_chunk = AssistantMessage(
                             id=msg_obj.id,
                             date=msg_obj.created_at,
-                            content=func_args[self.assistant_message_tool_kwarg],
+                            content=str(func_args[self.assistant_message_tool_kwarg]),
                             name=msg_obj.name,
                             otid=Message.generate_otid_from_id(msg_obj.id, chunk_index) if chunk_index is not None else None,
                         )

letta/server/rest_api/routers/openai/chat_completions/chat_completions.py

@@ -111,7 +111,7 @@ async def send_message_to_agent_chat_completions(
                 server.send_messages,
                 actor=actor,
                 agent_id=letta_agent.agent_state.id,
-                messages=messages,
+                input_messages=messages,
                 interface=streaming_interface,
                 put_inner_thoughts_first=False,
             )

letta/server/rest_api/routers/v1/agents.py

@@ -412,7 +412,7 @@ def list_blocks(
     """
     actor = server.user_manager.get_user_or_default(user_id=actor_id)
     try:
-        agent = server.agent_manager.get_agent_by_id(agent_id, actor=actor)
+        agent = server.agent_manager.get_agent_by_id(agent_id, actor)
         return agent.memory.blocks
     except NoResultFound as e:
         raise HTTPException(status_code=404, detail=str(e))
@@ -640,7 +640,7 @@ async def send_message(
         result = await server.send_message_to_agent(
             agent_id=agent_id,
             actor=actor,
-            messages=request.messages,
+            input_messages=request.messages,
             stream_steps=False,
             stream_tokens=False,
             # Support for AssistantMessage
@@ -703,7 +703,7 @@ async def send_message_streaming(
         result = await server.send_message_to_agent(
             agent_id=agent_id,
             actor=actor,
-            messages=request.messages,
+            input_messages=request.messages,
             stream_steps=True,
             stream_tokens=request.stream_tokens,
             # Support for AssistantMessage
@@ -730,7 +730,7 @@ async def process_message_background(
         result = await server.send_message_to_agent(
             agent_id=agent_id,
             actor=actor,
-            messages=messages,
+            input_messages=messages,
             stream_steps=False,  # NOTE(matt)
             stream_tokens=False,
             use_assistant_message=use_assistant_message,

letta/server/rest_api/routers/v1/groups.py

@@ -128,7 +128,7 @@ async def send_group_message(
     result = await server.send_group_message_to_agent(
         group_id=group_id,
         actor=actor,
-        messages=request.messages,
+        input_messages=request.messages,
         stream_steps=False,
         stream_tokens=False,
         # Support for AssistantMessage
@@ -167,7 +167,7 @@ async def send_group_message_streaming(
     result = await server.send_group_message_to_agent(
         group_id=group_id,
         actor=actor,
-        messages=request.messages,
+        input_messages=request.messages,
         stream_steps=True,
         stream_tokens=request.stream_tokens,
         # Support for AssistantMessage

letta/server/rest_api/routers/v1/messages.py

@@ -7,7 +7,7 @@ from starlette.requests import Request
 from letta.agents.letta_agent_batch import LettaAgentBatch
 from letta.log import get_logger
 from letta.orm.errors import NoResultFound
-from letta.schemas.job import BatchJob, JobStatus, JobType
+from letta.schemas.job import BatchJob, JobStatus, JobType, JobUpdate
 from letta.schemas.letta_request import CreateBatch
 from letta.server.rest_api.utils import get_letta_server
 from letta.server.server import SyncServer
@@ -43,18 +43,18 @@ async def create_messages_batch(
         if length > max_bytes:
             raise HTTPException(status_code=413, detail=f"Request too large ({length} bytes). Max is {max_bytes} bytes.")
 
+    actor = server.user_manager.get_user_or_default(user_id=actor_id)
+    batch_job = BatchJob(
+        user_id=actor.id,
+        status=JobStatus.running,
+        metadata={
+            "job_type": "batch_messages",
+        },
+        callback_url=str(payload.callback_url),
+    )
+
     try:
-        actor = server.user_manager.get_user_or_default(user_id=actor_id)
-
-        # Create a new job
-        batch_job = BatchJob(
-            user_id=actor.id,
-            status=JobStatus.created,
-            metadata={
-                "job_type": "batch_messages",
-            },
-            callback_url=str(payload.callback_url),
-        )
+        batch_job = server.job_manager.create_job(pydantic_job=batch_job, actor=actor)
 
         # create the batch runner
         batch_runner = LettaAgentBatch(
@@ -67,14 +67,17 @@ async def create_messages_batch(
             job_manager=server.job_manager,
             actor=actor,
         )
-        llm_batch_job = await batch_runner.step_until_request(batch_requests=payload.requests, letta_batch_job_id=batch_job.id)
+        await batch_runner.step_until_request(batch_requests=payload.requests, letta_batch_job_id=batch_job.id)
 
         # TODO: update run metadata
-        batch_job = server.job_manager.create_job(pydantic_job=batch_job, actor=actor)
-    except Exception:
+    except Exception as e:
         import traceback
 
+        print("Error creating batch job", e)
         traceback.print_exc()
+
+        # mark job as failed
+        server.job_manager.update_job_by_id(job_id=batch_job.id, job=BatchJob(status=JobStatus.failed), actor=actor)
         raise
     return batch_job
 
@@ -125,8 +128,19 @@ async def cancel_batch_run(
 
     try:
         job = server.job_manager.get_job_by_id(job_id=batch_id, actor=actor)
-        job.status = JobStatus.cancelled
-        server.job_manager.update_job_by_id(job_id=job, job=job)
-        # TODO: actually cancel it
+        job = server.job_manager.update_job_by_id(job_id=job.id, job_update=JobUpdate(status=JobStatus.cancelled), actor=actor)
+
+        # Get related llm batch jobs
+        llm_batch_jobs = server.batch_manager.list_llm_batch_jobs(letta_batch_id=job.id, actor=actor)
+        for llm_batch_job in llm_batch_jobs:
+            if llm_batch_job.status in {JobStatus.running, JobStatus.created}:
+                # TODO: Extend to providers beyond anthropic
+                # TODO: For now, we only support anthropic
+                # Cancel the job
+                anthropic_batch_id = llm_batch_job.create_batch_response.id
+                await server.anthropic_async_client.messages.batches.cancel(anthropic_batch_id)
+
+                # Update all the batch_job statuses
+                server.batch_manager.update_llm_batch_status(llm_batch_id=llm_batch_job.id, status=JobStatus.cancelled, actor=actor)
     except NoResultFound:
         raise HTTPException(status_code=404, detail="Run not found")