letta-nightly 0.11.0.dev20250807104511__py3-none-any.whl → 0.11.0.dev20250808055434__py3-none-any.whl

letta/orm/step_metrics.py

@@ -0,0 +1,71 @@
+from typing import TYPE_CHECKING, Optional
+
+from sqlalchemy import BigInteger, ForeignKey, String
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+from letta.orm.mixins import AgentMixin, ProjectMixin
+from letta.orm.sqlalchemy_base import SqlalchemyBase
+from letta.schemas.step_metrics import StepMetrics as PydanticStepMetrics
+
+if TYPE_CHECKING:
+    from letta.orm.agent import Agent
+    from letta.orm.job import Job
+    from letta.orm.step import Step
+
+
+class StepMetrics(SqlalchemyBase, ProjectMixin, AgentMixin):
+    """Tracks performance metrics for agent steps."""
+
+    __tablename__ = "step_metrics"
+    __pydantic_model__ = PydanticStepMetrics
+
+    id: Mapped[str] = mapped_column(
+        ForeignKey("steps.id", ondelete="CASCADE"),
+        primary_key=True,
+        doc="The unique identifier of the step this metric belongs to (also serves as PK)",
+    )
+    organization_id: Mapped[str] = mapped_column(
+        ForeignKey("organizations.id", ondelete="RESTRICT"),
+        nullable=True,
+        doc="The unique identifier of the organization",
+    )
+    provider_id: Mapped[Optional[str]] = mapped_column(
+        ForeignKey("providers.id", ondelete="RESTRICT"),
+        nullable=True,
+        doc="The unique identifier of the provider",
+    )
+    job_id: Mapped[Optional[str]] = mapped_column(
+        ForeignKey("jobs.id", ondelete="SET NULL"),
+        nullable=True,
+        doc="The unique identifier of the job",
+    )
+    llm_request_ns: Mapped[Optional[int]] = mapped_column(
+        BigInteger,
+        nullable=True,
+        doc="Time spent on the LLM request in nanoseconds",
+    )
+    tool_execution_ns: Mapped[Optional[int]] = mapped_column(
+        BigInteger,
+        nullable=True,
+        doc="Time spent on tool execution in nanoseconds",
+    )
+    step_ns: Mapped[Optional[int]] = mapped_column(
+        BigInteger,
+        nullable=True,
+        doc="Total time for the step in nanoseconds",
+    )
+    base_template_id: Mapped[Optional[str]] = mapped_column(
+        String,
+        nullable=True,
+        doc="The base template ID for the step",
+    )
+    template_id: Mapped[Optional[str]] = mapped_column(
+        String,
+        nullable=True,
+        doc="The template ID for the step",
+    )
+
+    # Relationships (foreign keys)
+    step: Mapped["Step"] = relationship("Step", back_populates="metrics", uselist=False)
+    job: Mapped[Optional["Job"]] = relationship("Job")
+    agent: Mapped[Optional["Agent"]] = relationship("Agent")

letta/schemas/enums.py

@@ -160,3 +160,12 @@ class SandboxType(str, Enum):
     E2B = "e2b"
     MODAL = "modal"
     LOCAL = "local"
+
+
+class StepStatus(str, Enum):
+    """Status of a step execution"""
+
+    PENDING = "pending"
+    SUCCESS = "success"
+    FAILED = "failed"
+    CANCELLED = "cancelled"

letta/schemas/llm_config.py

@@ -58,7 +58,7 @@ class LLMConfig(BaseModel):
     enable_reasoner: bool = Field(
         False, description="Whether or not the model should use extended thinking if it is a 'reasoning' style model"
     )
-    reasoning_effort: Optional[Literal["low", "medium", "high"]] = Field(
+    reasoning_effort: Optional[Literal["minimal", "low", "medium", "high"]] = Field(
         None,
         description="The reasoning effort to use when generating text reasoning models",
     )
@@ -188,6 +188,8 @@ class LLMConfig(BaseModel):
     @classmethod
     def apply_reasoning_setting_to_config(cls, config: "LLMConfig", reasoning: bool):
         if reasoning:
+            config.enable_reasoner = True
+
             if (
                 config.model_endpoint_type == "anthropic"
                 and ("claude-opus-4" in config.model or "claude-sonnet-4" in config.model or "claude-3-7-sonnet" in config.model)
@@ -195,19 +197,19 @@ class LLMConfig(BaseModel):
                 config.model_endpoint_type == "google_vertex" and ("gemini-2.5-flash" in config.model or "gemini-2.0-pro" in config.model)
             ):
                 config.put_inner_thoughts_in_kwargs = False
-                config.enable_reasoner = True
                 if config.max_reasoning_tokens == 0:
                     config.max_reasoning_tokens = 1024
             elif config.model_endpoint_type == "openai" and (
                 config.model.startswith("o1") or config.model.startswith("o3") or config.model.startswith("o4")
             ):
-                config.put_inner_thoughts_in_kwargs = True
-                config.enable_reasoner = True
+                config.put_inner_thoughts_in_kwargs = False
                 if config.reasoning_effort is None:
                     config.reasoning_effort = "medium"
             else:
                 config.put_inner_thoughts_in_kwargs = True
-                config.enable_reasoner = False
+
         else:
-            config.put_inner_thoughts_in_kwargs = False
             config.enable_reasoner = False
+            config.put_inner_thoughts_in_kwargs = False
+
+        return config

letta/schemas/providers/lmstudio.py

@@ -55,7 +55,7 @@ class LMStudioOpenAIProvider(OpenAIProvider):
                 LLMConfig(
                     model=model_name,
                     model_endpoint_type="openai",
-                    model_endpoint=self.base_url,
+                    model_endpoint=self.model_endpoint_url,
                     context_window=context_window_size,
                     handle=self.get_handle(model_name),
                     compatibility_type=compatibility_type,
@@ -94,7 +94,7 @@ class LMStudioOpenAIProvider(OpenAIProvider):
                 EmbeddingConfig(
                     embedding_model=model_name,
                     embedding_endpoint_type="openai",
-                    embedding_endpoint=self.base_url,
+                    embedding_endpoint=self.model_endpoint_url,
                     embedding_dim=768,  # Default embedding dimension, not context window
                     embedding_chunk_size=DEFAULT_EMBEDDING_CHUNK_SIZE,  # NOTE: max is 2048
                     handle=self.get_handle(model_name),

letta/schemas/providers/openai.py

@@ -11,7 +11,7 @@ from letta.schemas.providers.base import Provider
 
 logger = get_logger(__name__)
 
-ALLOWED_PREFIXES = {"gpt-4", "o1", "o3", "o4"}
+ALLOWED_PREFIXES = {"gpt-4", "gpt-5", "o1", "o3", "o4"}
 DISALLOWED_KEYWORDS = {"transcribe", "search", "realtime", "tts", "audio", "computer", "o1-mini", "o1-preview", "o1-pro"}
 DEFAULT_EMBEDDING_BATCH_SIZE = 1024
 

letta/schemas/step.py

@@ -3,6 +3,7 @@ from typing import Dict, List, Literal, Optional
 
 from pydantic import Field
 
+from letta.schemas.enums import StepStatus
 from letta.schemas.letta_base import LettaBase
 from letta.schemas.letta_stop_reason import StopReasonType
 from letta.schemas.message import Message
@@ -40,6 +41,11 @@ class Step(StepBase):
     )
     project_id: Optional[str] = Field(None, description="The project that the agent that executed this step belongs to (cloud only).")
 
+    # error tracking fields
+    error_type: Optional[str] = Field(None, description="The type/class of the error that occurred")
+    error_data: Optional[Dict] = Field(None, description="Error details including message, traceback, and additional context")
+    status: Optional[StepStatus] = Field(StepStatus.PENDING, description="Step status: pending, success, or failed")
+
 
 class StepProgression(int, Enum):
     START = auto()

letta/schemas/step_metrics.py

@@ -0,0 +1,23 @@
+from typing import Optional
+
+from pydantic import Field
+
+from letta.schemas.letta_base import LettaBase
+
+
+class StepMetricsBase(LettaBase):
+    __id_prefix__ = "step"
+
+
+class StepMetrics(StepMetricsBase):
+    id: str = Field(..., description="The id of the step this metric belongs to (matches steps.id).")
+    organization_id: Optional[str] = Field(None, description="The unique identifier of the organization.")
+    provider_id: Optional[str] = Field(None, description="The unique identifier of the provider.")
+    job_id: Optional[str] = Field(None, description="The unique identifier of the job.")
+    agent_id: Optional[str] = Field(None, description="The unique identifier of the agent.")
+    llm_request_ns: Optional[int] = Field(None, description="Time spent on LLM requests in nanoseconds.")
+    tool_execution_ns: Optional[int] = Field(None, description="Time spent on tool execution in nanoseconds.")
+    step_ns: Optional[int] = Field(None, description="Total time for the step in nanoseconds.")
+    base_template_id: Optional[str] = Field(None, description="The base template ID that the step belongs to (cloud only).")
+    template_id: Optional[str] = Field(None, description="The template ID that the step belongs to (cloud only).")
+    project_id: Optional[str] = Field(None, description="The project that the step belongs to (cloud only).")

letta/schemas/tool_rule.py

@@ -23,26 +23,24 @@ class BaseToolRule(LettaBase):
     def get_valid_tools(self, tool_call_history: List[str], available_tools: Set[str], last_function_response: Optional[str]) -> set[str]:
         raise NotImplementedError
 
-    def render_prompt(self) -> Optional[str]:
+    def render_prompt(self) -> str | None:
         """Render the prompt template with this rule's attributes."""
-        template_to_use = self.prompt_template or self._get_default_template()
-        if not template_to_use:
+        if not self.prompt_template:
             return None
 
         try:
-            template = Template(template_to_use)
+            template = Template(self.prompt_template)
             return template.render(**self.model_dump())
         except Exception as e:
             logger.warning(
-                f"Failed to render prompt template for tool rule '{self.tool_name}' (type: {self.type}). "
-                f"Template: '{template_to_use}'. Error: {e}"
+                "Failed to render prompt template for tool rule '%s' (type: %s). Template: '%s'. Error: %s",
+                self.tool_name,
+                self.type,
+                self.prompt_template,
+                e,
             )
             return None
 
-    def _get_default_template(self) -> Optional[str]:
-        """Get the default template for this rule type. Override in subclasses."""
-        return None
-
 
 class ChildToolRule(BaseToolRule):
     """
@@ -60,9 +58,6 @@ class ChildToolRule(BaseToolRule):
         last_tool = tool_call_history[-1] if tool_call_history else None
         return set(self.children) if last_tool == self.tool_name else available_tools
 
-    def _get_default_template(self) -> Optional[str]:
-        return "<tool_rule>\nAfter using {{ tool_name }}, you must use one of these tools: {{ children | join(', ') }}\n</tool_rule>"
-
 
 class ParentToolRule(BaseToolRule):
     """
@@ -80,9 +75,6 @@ class ParentToolRule(BaseToolRule):
         last_tool = tool_call_history[-1] if tool_call_history else None
         return set(self.children) if last_tool == self.tool_name else available_tools - set(self.children)
 
-    def _get_default_template(self) -> Optional[str]:
-        return "<tool_rule>\n{{ children | join(', ') }} can only be used after {{ tool_name }}\n</tool_rule>"
-
 
 class ConditionalToolRule(BaseToolRule):
     """
@@ -125,7 +117,8 @@ class ConditionalToolRule(BaseToolRule):
 
         return {self.default_child} if self.default_child else available_tools
 
-    def _matches_key(self, function_output: str, key: Any) -> bool:
+    @staticmethod
+    def _matches_key(function_output: str, key: Any) -> bool:
         """Helper function to determine if function output matches a mapping key."""
         if isinstance(key, bool):
             return function_output.lower() == "true" if key else function_output.lower() == "false"
@@ -142,9 +135,6 @@ class ConditionalToolRule(BaseToolRule):
         else:  # Assume string
             return str(function_output) == str(key)
 
-    def _get_default_template(self) -> Optional[str]:
-        return "<tool_rule>\n{{ tool_name }} will determine which tool to use next based on its output\n</tool_rule>"
-
 
 class InitToolRule(BaseToolRule):
     """
@@ -165,9 +155,6 @@ class TerminalToolRule(BaseToolRule):
         description="Optional Jinja2 template for generating agent prompt about this tool rule.",
     )
 
-    def _get_default_template(self) -> Optional[str]:
-        return "<tool_rule>\n{{ tool_name }} ends your response (yields control) when called\n</tool_rule>"
-
 
 class ContinueToolRule(BaseToolRule):
     """
@@ -196,9 +183,6 @@ class RequiredBeforeExitToolRule(BaseToolRule):
         """Returns all available tools - the logic for preventing exit is handled elsewhere."""
         return available_tools
 
-    def _get_default_template(self) -> Optional[str]:
-        return "<tool_rule>{{ tool_name }} must be called before ending the conversation</tool_rule>"
-
 
 class MaxCountPerStepToolRule(BaseToolRule):
     """
@@ -222,9 +206,6 @@ class MaxCountPerStepToolRule(BaseToolRule):
 
         return available_tools
 
-    def _get_default_template(self) -> Optional[str]:
-        return "<tool_rule>\n{{ tool_name }}: at most {{ max_count_limit }} use(s) per response\n</tool_rule>"
-
 
 ToolRule = Annotated[
     Union[

letta/services/step_manager.py

@@ -1,6 +1,6 @@
 from datetime import datetime
 from enum import Enum
-from typing import List, Literal, Optional
+from typing import Dict, List, Literal, Optional
 
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -12,6 +12,7 @@ from letta.orm.job import Job as JobModel
 from letta.orm.sqlalchemy_base import AccessType
 from letta.orm.step import Step as StepModel
 from letta.otel.tracing import get_trace_id, trace_method
+from letta.schemas.enums import StepStatus
 from letta.schemas.letta_stop_reason import LettaStopReason, StopReasonType
 from letta.schemas.openai.chat_completion_response import UsageStatistics
 from letta.schemas.step import Step as PydanticStep
@@ -87,6 +88,10 @@ class StepManager:
         job_id: Optional[str] = None,
         step_id: Optional[str] = None,
         project_id: Optional[str] = None,
+        stop_reason: Optional[LettaStopReason] = None,
+        status: Optional[StepStatus] = None,
+        error_type: Optional[str] = None,
+        error_data: Optional[Dict] = None,
     ) -> PydanticStep:
         step_data = {
             "origin": None,
@@ -106,9 +111,14 @@ class StepManager:
             "tid": None,
             "trace_id": get_trace_id(),  # Get the current trace ID
             "project_id": project_id,
+            "status": status if status else StepStatus.PENDING,
+            "error_type": error_type,
+            "error_data": error_data,
         }
         if step_id:
             step_data["id"] = step_id
+        if stop_reason:
+            step_data["stop_reason"] = stop_reason.stop_reason
         with db_registry.session() as session:
             if job_id:
                 self._verify_job_access(session, job_id, actor, access=["write"])
@@ -133,6 +143,9 @@ class StepManager:
         step_id: Optional[str] = None,
         project_id: Optional[str] = None,
         stop_reason: Optional[LettaStopReason] = None,
+        status: Optional[StepStatus] = None,
+        error_type: Optional[str] = None,
+        error_data: Optional[Dict] = None,
     ) -> PydanticStep:
         step_data = {
             "origin": None,
@@ -152,6 +165,9 @@ class StepManager:
             "tid": None,
             "trace_id": get_trace_id(),  # Get the current trace ID
             "project_id": project_id,
+            "status": status if status else StepStatus.PENDING,
+            "error_type": error_type,
+            "error_data": error_data,
         }
         if step_id:
             step_data["id"] = step_id
@@ -236,6 +252,126 @@ class StepManager:
             await session.commit()
             return step
 
+    @enforce_types
+    @trace_method
+    async def update_step_error_async(
+        self,
+        actor: PydanticUser,
+        step_id: str,
+        error_type: str,
+        error_message: str,
+        error_traceback: str,
+        error_details: Optional[Dict] = None,
+        stop_reason: Optional[LettaStopReason] = None,
+    ) -> PydanticStep:
+        """Update a step with error information.
+
+        Args:
+            actor: The user making the request
+            step_id: The ID of the step to update
+            error_type: The type/class of the error
+            error_message: The error message
+            error_traceback: Full error traceback
+            error_details: Additional error context
+            stop_reason: The stop reason to set
+
+        Returns:
+            The updated step
+
+        Raises:
+            NoResultFound: If the step does not exist
+        """
+        async with db_registry.async_session() as session:
+            step = await session.get(StepModel, step_id)
+            if not step:
+                raise NoResultFound(f"Step with id {step_id} does not exist")
+            if step.organization_id != actor.organization_id:
+                raise Exception("Unauthorized")
+
+            step.status = StepStatus.FAILED
+            step.error_type = error_type
+            step.error_data = {"message": error_message, "traceback": error_traceback, "details": error_details}
+            if stop_reason:
+                step.stop_reason = stop_reason.stop_reason
+
+            await session.commit()
+            return step.to_pydantic()
+
+    @enforce_types
+    @trace_method
+    async def update_step_success_async(
+        self,
+        actor: PydanticUser,
+        step_id: str,
+        usage: UsageStatistics,
+        stop_reason: Optional[LettaStopReason] = None,
+    ) -> PydanticStep:
+        """Update a step with success status and final usage statistics.
+
+        Args:
+            actor: The user making the request
+            step_id: The ID of the step to update
+            usage: Final usage statistics
+            stop_reason: The stop reason to set
+
+        Returns:
+            The updated step
+
+        Raises:
+            NoResultFound: If the step does not exist
+        """
+        async with db_registry.async_session() as session:
+            step = await session.get(StepModel, step_id)
+            if not step:
+                raise NoResultFound(f"Step with id {step_id} does not exist")
+            if step.organization_id != actor.organization_id:
+                raise Exception("Unauthorized")
+
+            step.status = StepStatus.SUCCESS
+            step.completion_tokens = usage.completion_tokens
+            step.prompt_tokens = usage.prompt_tokens
+            step.total_tokens = usage.total_tokens
+            if stop_reason:
+                step.stop_reason = stop_reason.stop_reason
+
+            await session.commit()
+            return step.to_pydantic()
+
+    @enforce_types
+    @trace_method
+    async def update_step_cancelled_async(
+        self,
+        actor: PydanticUser,
+        step_id: str,
+        stop_reason: Optional[LettaStopReason] = None,
+    ) -> PydanticStep:
+        """Update a step with cancelled status.
+
+        Args:
+            actor: The user making the request
+            step_id: The ID of the step to update
+            stop_reason: The stop reason to set
+
+        Returns:
+            The updated step
+
+        Raises:
+            NoResultFound: If the step does not exist
+        """
+        async with db_registry.async_session() as session:
+            step = await session.get(StepModel, step_id)
+            if not step:
+                raise NoResultFound(f"Step with id {step_id} does not exist")
+            if step.organization_id != actor.organization_id:
+                raise Exception("Unauthorized")
+
+            step.status = StepStatus.CANCELLED
+            if stop_reason:
+                step.stop_reason = stop_reason.stop_reason
+
+            await session.commit()
+            return step.to_pydantic()
+
     def _verify_job_access(
         self,
         session: Session,
@@ -319,6 +455,10 @@ class NoopStepManager(StepManager):
         job_id: Optional[str] = None,
         step_id: Optional[str] = None,
         project_id: Optional[str] = None,
+        stop_reason: Optional[LettaStopReason] = None,
+        status: Optional[StepStatus] = None,
+        error_type: Optional[str] = None,
+        error_data: Optional[Dict] = None,
     ) -> PydanticStep:
         return
 
@@ -339,5 +479,43 @@ class NoopStepManager(StepManager):
         step_id: Optional[str] = None,
         project_id: Optional[str] = None,
         stop_reason: Optional[LettaStopReason] = None,
+        status: Optional[StepStatus] = None,
+        error_type: Optional[str] = None,
+        error_data: Optional[Dict] = None,
+    ) -> PydanticStep:
+        return
+
+    @enforce_types
+    @trace_method
+    async def update_step_error_async(
+        self,
+        actor: PydanticUser,
+        step_id: str,
+        error_type: str,
+        error_message: str,
+        error_traceback: str,
+        error_details: Optional[Dict] = None,
+        stop_reason: Optional[LettaStopReason] = None,
+    ) -> PydanticStep:
+        return
+
+    @enforce_types
+    @trace_method
+    async def update_step_success_async(
+        self,
+        actor: PydanticUser,
+        step_id: str,
+        usage: UsageStatistics,
+        stop_reason: Optional[LettaStopReason] = None,
+    ) -> PydanticStep:
+        return
+
+    @enforce_types
+    @trace_method
+    async def update_step_cancelled_async(
+        self,
+        actor: PydanticUser,
+        step_id: str,
+        stop_reason: Optional[LettaStopReason] = None,
     ) -> PydanticStep:
         return

letta/services/tool_executor/builtin_tool_executor.py

@@ -210,7 +210,10 @@ class LettaBuiltinToolExecutor(ToolExecutor):
         logger.info(f"[DEBUG] Starting Firecrawl search for query: '{task.query}' with limit={limit}")
 
         # Perform the search for this task
-        search_result = await app.search(task.query, limit=limit, scrape_options=ScrapeOptions(formats=["markdown"]))
+        scrape_options = ScrapeOptions(
+            formats=["markdown"], excludeTags=["#ad", "#footer"], onlyMainContent=True, parsePDF=True, removeBase64Images=True
+        )
+        search_result = await app.search(task.query, limit=limit, scrape_options=scrape_options)
 
         logger.info(
             f"[DEBUG] Firecrawl search completed for '{task.query}': {len(search_result.get('data', [])) if search_result else 0} results"

letta/services/tool_executor/core_tool_executor.py

@@ -230,14 +230,7 @@ class LettaCoreToolExecutor(ToolExecutor):
         await AgentManager().update_memory_if_changed_async(agent_id=agent_state.id, new_memory=agent_state.memory, actor=actor)
         return None
 
-    async def memory_replace(
-        self,
-        agent_state: AgentState,
-        actor: User,
-        label: str,
-        old_str: str,
-        new_str: Optional[str] = None,
-    ) -> str:
+    async def memory_replace(self, agent_state: AgentState, actor: User, label: str, old_str: str, new_str: str) -> str:
         """
         The memory_replace command allows you to replace a specific string in a memory
         block with a new string. This is used for making precise edits.
@@ -246,8 +239,7 @@ class LettaCoreToolExecutor(ToolExecutor):
             label (str): Section of the memory to be edited, identified by its label.
             old_str (str): The text to replace (must match exactly, including whitespace
                 and indentation). Do not include line number prefixes.
-            new_str (Optional[str]): The new text to insert in place of the old text.
-                Omit this argument to delete the old_str. Do not include line number prefixes.
+            new_str (str): The new text to insert in place of the old text. Do not include line number prefixes.
 
         Returns:
             str: The success message