letta-nightly 0.13.0.dev20251030104218__py3-none-any.whl → 0.13.1.dev20251031234110__py3-none-any.whl

letta/services/run_manager.py

@@ -14,7 +14,7 @@ from letta.orm.run_metrics import RunMetrics as RunMetricsModel
 from letta.orm.sqlalchemy_base import AccessType
 from letta.orm.step import Step as StepModel
 from letta.otel.tracing import log_event, trace_method
-from letta.schemas.enums import AgentType, ComparisonOperator, MessageRole, RunStatus, PrimitiveType
+from letta.schemas.enums import AgentType, ComparisonOperator, MessageRole, PrimitiveType, RunStatus
 from letta.schemas.job import LettaRequestConfig
 from letta.schemas.letta_message import LettaMessage, LettaMessageUnion
 from letta.schemas.letta_response import LettaResponse
@@ -63,14 +63,16 @@ class RunManager:
 
             run = RunModel(**run_data)
             run.organization_id = organization_id
-            run = await run.create_async(session, actor=actor, no_commit=True, no_refresh=True)
-
-            # Create run metrics with start timestamp
-            import time
 
             # Get the project_id from the agent
             agent = await session.get(AgentModel, agent_id)
             project_id = agent.project_id if agent else None
+            run.project_id = project_id
+
+            run = await run.create_async(session, actor=actor, no_commit=True, no_refresh=True)
+
+            # Create run metrics with start timestamp
+            import time
 
             metrics = RunMetricsModel(
                 id=run.id,
@@ -95,6 +97,34 @@ class RunManager:
                 raise NoResultFound(f"Run with id {run_id} not found")
             return run.to_pydantic()
 
+    @enforce_types
+    async def get_run_with_status(self, run_id: str, actor: PydanticUser) -> PydanticRun:
+        """Get a run by its ID and update status from Lettuce if applicable."""
+        run = await self.get_run_by_id(run_id=run_id, actor=actor)
+
+        use_lettuce = run.metadata and run.metadata.get("lettuce")
+        if use_lettuce and run.status not in [RunStatus.completed, RunStatus.failed, RunStatus.cancelled]:
+            try:
+                from letta.services.lettuce import LettuceClient
+
+                lettuce_client = await LettuceClient.create()
+                status = await lettuce_client.get_status(run_id=run_id)
+
+                # Map the status to our enum
+                if status == "RUNNING":
+                    run.status = RunStatus.running
+                elif status == "COMPLETED":
+                    run.status = RunStatus.completed
+                elif status == "FAILED":
+                    run.status = RunStatus.failed
+                elif status == "CANCELLED":
+                    run.status = RunStatus.cancelled
+            except Exception as e:
+                logger.error(f"Failed to get status from Lettuce for run {run_id}: {str(e)}")
+                # Return run with current status from DB if Lettuce fails
+
+        return run
+
     @enforce_types
     async def list_runs(
         self,
@@ -112,12 +142,27 @@ class RunManager:
         step_count: Optional[int] = None,
         step_count_operator: ComparisonOperator = ComparisonOperator.EQ,
         tools_used: Optional[List[str]] = None,
+        project_id: Optional[str] = None,
+        order_by: Literal["created_at", "duration"] = "created_at",
+        duration_percentile: Optional[int] = None,
+        duration_filter: Optional[dict] = None,
+        start_date: Optional[datetime] = None,
+        end_date: Optional[datetime] = None,
     ) -> List[PydanticRun]:
         """List runs with filtering options."""
         async with db_registry.async_session() as session:
-            from sqlalchemy import or_, select
+            from sqlalchemy import func, or_, select
 
-            query = select(RunModel).filter(RunModel.organization_id == actor.organization_id)
+            # Always join with run_metrics to get duration data
+            query = (
+                select(RunModel, RunMetricsModel.run_ns)
+                .outerjoin(RunMetricsModel, RunModel.id == RunMetricsModel.id)
+                .filter(RunModel.organization_id == actor.organization_id)
+            )
+
+            # Filter by project_id if provided
+            if project_id:
+                query = query.filter(RunModel.project_id == project_id)
 
             # Handle agent filtering
             if agent_id:
@@ -141,41 +186,107 @@ class RunManager:
             if template_family:
                 query = query.filter(RunModel.base_template_id == template_family)
 
-            # Filter by step_count and/or tools_used - join with run_metrics
-            if step_count is not None or tools_used:
-                query = query.join(RunMetricsModel, RunModel.id == RunMetricsModel.id)
-
-                # Filter by step_count with the specified operator
-                if step_count is not None:
-                    if step_count_operator == ComparisonOperator.EQ:
-                        query = query.filter(RunMetricsModel.num_steps == step_count)
-                    elif step_count_operator == ComparisonOperator.GTE:
-                        query = query.filter(RunMetricsModel.num_steps >= step_count)
-                    elif step_count_operator == ComparisonOperator.LTE:
-                        query = query.filter(RunMetricsModel.num_steps <= step_count)
-
-                # Filter by tools used ids
-                if tools_used:
-                    from sqlalchemy import String, cast as sa_cast, type_coerce
-                    from sqlalchemy.dialects.postgresql import ARRAY, JSONB
-
-                    # Use ?| operator to check if any tool_id exists in the array (OR logic)
-                    jsonb_tools = sa_cast(RunMetricsModel.tools_used, JSONB)
-                    tools_array = type_coerce(tools_used, ARRAY(String))
-                    query = query.filter(jsonb_tools.op("?|")(tools_array))
-
-            # Apply pagination
-            from letta.services.helpers.run_manager_helper import _apply_pagination_async
+            # Filter by date range
+            if start_date:
+                query = query.filter(RunModel.created_at >= start_date)
+            if end_date:
+                query = query.filter(RunModel.created_at <= end_date)
+
+            # Filter by step_count with the specified operator
+            if step_count is not None:
+                if step_count_operator == ComparisonOperator.EQ:
+                    query = query.filter(RunMetricsModel.num_steps == step_count)
+                elif step_count_operator == ComparisonOperator.GTE:
+                    query = query.filter(RunMetricsModel.num_steps >= step_count)
+                elif step_count_operator == ComparisonOperator.LTE:
+                    query = query.filter(RunMetricsModel.num_steps <= step_count)
+
+            # Filter by tools used ids
+            if tools_used:
+                from sqlalchemy import String, cast as sa_cast, type_coerce
+                from sqlalchemy.dialects.postgresql import ARRAY, JSONB
+
+                # Use ?| operator to check if any tool_id exists in the array (OR logic)
+                jsonb_tools = sa_cast(RunMetricsModel.tools_used, JSONB)
+                tools_array = type_coerce(tools_used, ARRAY(String))
+                query = query.filter(jsonb_tools.op("?|")(tools_array))
+
+            # Ensure run_ns is not null when working with duration
+            if order_by == "duration" or duration_percentile is not None or duration_filter is not None:
+                query = query.filter(RunMetricsModel.run_ns.isnot(None))
+
+            # Apply duration filter if requested
+            if duration_filter is not None:
+                duration_value = duration_filter.get("value") if isinstance(duration_filter, dict) else duration_filter.value
+                duration_operator = duration_filter.get("operator") if isinstance(duration_filter, dict) else duration_filter.operator
+
+                if duration_operator == "gt":
+                    query = query.filter(RunMetricsModel.run_ns > duration_value)
+                elif duration_operator == "lt":
+                    query = query.filter(RunMetricsModel.run_ns < duration_value)
+                elif duration_operator == "eq":
+                    query = query.filter(RunMetricsModel.run_ns == duration_value)
+
+            # Apply duration percentile filter if requested
+            if duration_percentile is not None:
+                # Calculate the percentile threshold
+                percentile_query = (
+                    select(func.percentile_cont(duration_percentile / 100.0).within_group(RunMetricsModel.run_ns))
+                    .select_from(RunMetricsModel)
+                    .join(RunModel, RunModel.id == RunMetricsModel.id)
+                    .filter(RunModel.organization_id == actor.organization_id)
+                    .filter(RunMetricsModel.run_ns.isnot(None))
+                )
 
-            query = await _apply_pagination_async(query, before, after, session, ascending=ascending)
+                # Apply same filters to percentile calculation
+                if project_id:
+                    percentile_query = percentile_query.filter(RunModel.project_id == project_id)
+                if agent_ids:
+                    percentile_query = percentile_query.filter(RunModel.agent_id.in_(agent_ids))
+                if statuses:
+                    percentile_query = percentile_query.filter(RunModel.status.in_(statuses))
+
+                # Execute percentile query
+                percentile_result = await session.execute(percentile_query)
+                percentile_threshold = percentile_result.scalar()
+
+                # Filter by percentile threshold (runs slower than the percentile)
+                if percentile_threshold is not None:
+                    query = query.filter(RunMetricsModel.run_ns >= percentile_threshold)
+
+            # Apply sorting based on order_by
+            if order_by == "duration":
+                # Sort by duration
+                if ascending:
+                    query = query.order_by(RunMetricsModel.run_ns.asc())
+                else:
+                    query = query.order_by(RunMetricsModel.run_ns.desc())
+            else:
+                # Apply pagination for created_at ordering
+                from letta.services.helpers.run_manager_helper import _apply_pagination_async
+
+                query = await _apply_pagination_async(query, before, after, session, ascending=ascending)
 
             # Apply limit
             if limit:
                 query = query.limit(limit)
 
             result = await session.execute(query)
-            runs = result.scalars().all()
-            return [run.to_pydantic() for run in runs]
+            rows = result.all()
+
+            # Populate total_duration_ns from run_metrics.run_ns
+            pydantic_runs = []
+            for row in rows:
+                run_model = row[0]
+                run_ns = row[1]
+
+                pydantic_run = run_model.to_pydantic()
+                if run_ns is not None:
+                    pydantic_run.total_duration_ns = run_ns
+
+                pydantic_runs.append(pydantic_run)
+
+            return pydantic_runs
 
     @enforce_types
     @raise_on_invalid_id(param_name="run_id", expected_prefix=PrimitiveType.RUN)
@@ -323,8 +434,8 @@ class RunManager:
             logger.error(error_message)
             result["callback_error"] = error_message
             # Continue silently - callback failures should not affect run completion
-        finally:
-            return result
+
+        return result
 
     @enforce_types
     @raise_on_invalid_id(param_name="run_id", expected_prefix=PrimitiveType.RUN)

letta/services/source_manager.py

@@ -326,13 +326,25 @@ class SourceManager:
     @enforce_types
     @trace_method
     @raise_on_invalid_id(param_name="source_id", expected_prefix=PrimitiveType.SOURCE)
-    async def get_agents_for_source_id(self, source_id: str, actor: PydanticUser) -> List[str]:
+    async def get_agents_for_source_id(
+        self,
+        source_id: str,
+        actor: PydanticUser,
+        before: Optional[str] = None,
+        after: Optional[str] = None,
+        limit: Optional[int] = 50,
+        ascending: bool = True,
+    ) -> List[str]:
         """
         Get all agent IDs associated with a given source ID.
 
         Args:
             source_id: ID of the source to find agents for
             actor: User performing the action
+            before: Agent ID cursor for pagination (upper bound)
+            after: Agent ID cursor for pagination (lower bound)
+            limit: Maximum number of agent IDs to return
+            ascending: Sort direction by creation time
 
         Returns:
             List[str]: List of agent IDs that have this source attached
@@ -341,8 +353,84 @@ class SourceManager:
             # Verify source exists and user has permission to access it
             await self._validate_source_exists_async(session, source_id, actor)
 
-            # Query the junction table directly for performance
-            query = select(SourcesAgents.agent_id).where(SourcesAgents.source_id == source_id)
+            # Get reference objects for pagination
+            before_obj = None
+            after_obj = None
+
+            if before:
+                before_obj = await session.get(AgentModel, before)
+                if not before_obj:
+                    from letta.orm.errors import NoResultFound
+
+                    raise NoResultFound(f"No Agent found with id {before}")
+
+            if after:
+                after_obj = await session.get(AgentModel, after)
+                if not after_obj:
+                    from letta.orm.errors import NoResultFound
+
+                    raise NoResultFound(f"No Agent found with id {after}")
+
+            # Build query with join to AgentModel for ordering and pagination
+            query = (
+                select(AgentModel.id)
+                .join(SourcesAgents, AgentModel.id == SourcesAgents.agent_id)
+                .where(
+                    SourcesAgents.source_id == source_id,
+                    AgentModel.organization_id == actor.organization_id,
+                    AgentModel.is_deleted == False,
+                )
+            )
+
+            # Apply pagination conditions
+            if before_obj or after_obj:
+                from sqlalchemy import and_, or_
+
+                conditions = []
+
+                if before_obj and after_obj:
+                    # Window-based query
+                    conditions.append(
+                        or_(
+                            AgentModel.created_at < before_obj.created_at,
+                            and_(AgentModel.created_at == before_obj.created_at, AgentModel.id < before_obj.id),
+                        )
+                    )
+                    conditions.append(
+                        or_(
+                            AgentModel.created_at > after_obj.created_at,
+                            and_(AgentModel.created_at == after_obj.created_at, AgentModel.id > after_obj.id),
+                        )
+                    )
+                else:
+                    if before_obj:
+                        conditions.append(
+                            or_(
+                                AgentModel.created_at < before_obj.created_at
+                                if ascending
+                                else AgentModel.created_at > before_obj.created_at,
+                                and_(AgentModel.created_at == before_obj.created_at, AgentModel.id < before_obj.id),
+                            )
+                        )
+                    if after_obj:
+                        conditions.append(
+                            or_(
+                                AgentModel.created_at > after_obj.created_at if ascending else AgentModel.created_at < after_obj.created_at,
+                                and_(AgentModel.created_at == after_obj.created_at, AgentModel.id > after_obj.id),
+                            )
+                        )
+
+                if conditions:
+                    query = query.where(and_(*conditions))
+
+            # Apply ordering
+            if ascending:
+                query = query.order_by(AgentModel.created_at.asc(), AgentModel.id.asc())
+            else:
+                query = query.order_by(AgentModel.created_at.desc(), AgentModel.id.desc())
+
+            # Apply limit
+            query = query.limit(limit)
 
             result = await session.execute(query)
             agent_ids = result.scalars().all()

letta/services/step_manager.py

@@ -278,15 +278,14 @@ class StepManager:
         ascending: bool = False,
     ) -> List[PydanticMessage]:
         async with db_registry.async_session() as session:
-            messages = MessageModel.list(
+            messages = await MessageModel.list_async(
                 db_session=session,
                 before=before,
                 after=after,
                 ascending=ascending,
                 limit=limit,
                 actor=actor,
-                join_model=StepModel,
-                join_conditions=[MessageModel.step.id == step_id],
+                step_id=step_id,
             )
             return [message.to_pydantic() for message in messages]
 

letta/services/streaming_service.py

@@ -31,6 +31,7 @@ from letta.schemas.letta_message import AssistantMessage, MessageType
 from letta.schemas.letta_message_content import TextContent
 from letta.schemas.letta_request import LettaStreamingRequest
 from letta.schemas.letta_response import LettaResponse
+from letta.schemas.letta_stop_reason import LettaStopReason, StopReasonType
 from letta.schemas.message import MessageCreate
 from letta.schemas.run import Run as PydanticRun, RunUpdate
 from letta.schemas.usage import LettaUsageStatistics
@@ -273,6 +274,11 @@ class StreamingService:
 
         async def error_aware_stream():
             """Stream that handles early LLM errors gracefully in streaming format."""
+            run_status = None
+            run_update_metadata = None
+            stop_reason = None
+            error_data = None
+
             try:
                 stream = agent_loop.stream(
                     input_messages=messages,
@@ -287,23 +293,23 @@ class StreamingService:
                 async for chunk in stream:
                     yield chunk
 
-                # update run status after completion
-                if run_id and self.runs_manager:
-                    if agent_loop.stop_reason.stop_reason.value == "cancelled":
-                        run_status = RunStatus.cancelled
-                    else:
-                        run_status = RunStatus.completed
-
-                    await self.runs_manager.update_run_by_id_async(
-                        run_id=run_id,
-                        update=RunUpdate(status=run_status, stop_reason=agent_loop.stop_reason.stop_reason.value),
-                        actor=actor,
-                    )
+                # set run status after successful completion
+                if agent_loop.stop_reason.stop_reason.value == "cancelled":
+                    run_status = RunStatus.cancelled
+                else:
+                    run_status = RunStatus.completed
+                stop_reason = agent_loop.stop_reason.stop_reason.value
 
             except LLMTimeoutError as e:
+                run_status = RunStatus.failed
                 error_data = {"error": {"type": "llm_timeout", "message": "The LLM request timed out. Please try again.", "detail": str(e)}}
+                stop_reason = StopReasonType.llm_api_error
+                logger.error(f"Run {run_id} stopped with LLM timeout error: {e}, error_data: {error_data}")
                 yield (f"data: {json.dumps(error_data)}\n\n", 504)
+                # Send [DONE] marker to properly close the stream
+                yield "data: [DONE]\n\n"
             except LLMRateLimitError as e:
+                run_status = RunStatus.failed
                 error_data = {
                     "error": {
                         "type": "llm_rate_limit",
@@ -311,8 +317,13 @@ class StreamingService:
                         "detail": str(e),
                     }
                 }
+                stop_reason = StopReasonType.llm_api_error
+                logger.warning(f"Run {run_id} stopped with LLM rate limit error: {e}, error_data: {error_data}")
                 yield (f"data: {json.dumps(error_data)}\n\n", 429)
+                # Send [DONE] marker to properly close the stream
+                yield "data: [DONE]\n\n"
             except LLMAuthenticationError as e:
+                run_status = RunStatus.failed
                 error_data = {
                     "error": {
                         "type": "llm_authentication",
@@ -320,13 +331,41 @@ class StreamingService:
                         "detail": str(e),
                     }
                 }
+                logger.warning(f"Run {run_id} stopped with LLM authentication error: {e}, error_data: {error_data}")
+                stop_reason = StopReasonType.llm_api_error
                 yield (f"data: {json.dumps(error_data)}\n\n", 401)
+                # Send [DONE] marker to properly close the stream
+                yield "data: [DONE]\n\n"
             except LLMError as e:
+                run_status = RunStatus.failed
                 error_data = {"error": {"type": "llm_error", "message": "An error occurred with the LLM request.", "detail": str(e)}}
+                logger.error(f"Run {run_id} stopped with LLM error: {e}, error_data: {error_data}")
                 yield (f"data: {json.dumps(error_data)}\n\n", 502)
+                # Send [DONE] marker to properly close the stream
+                stop_reason = StopReasonType.llm_api_error
+                yield "data: [DONE]\n\n"
             except Exception as e:
-                error_data = {"error": {"type": "internal_error", "message": "An internal server error occurred.", "detail": str(e)}}
+                run_status = RunStatus.failed
+                error_data = {
+                    "error": {
+                        "type": "internal_error",
+                        "message": "An unknown error occurred with the LLM streaming request.",
+                        "detail": str(e),
+                    }
+                }
+                logger.error(f"Run {run_id} stopped with unknown error: {e}, error_data: {error_data}")
+                stop_reason = StopReasonType.error
                 yield (f"data: {json.dumps(error_data)}\n\n", 500)
+                # Re-raise to ensure proper error handling and Sentry capture
+                raise
+            finally:
+                # always update run status, whether success or failure
+                if run_id and self.runs_manager and run_status:
+                    await self.runs_manager.update_run_by_id_async(
+                        run_id=run_id,
+                        update=RunUpdate(status=run_status, stop_reason=stop_reason, metadata=error_data),
+                        actor=actor,
+                    )
 
         return error_aware_stream()
 

letta/services/summarizer/summarizer.py

@@ -4,7 +4,13 @@ import traceback
 from typing import List, Optional, Tuple, Union
 
 from letta.agents.ephemeral_summary_agent import EphemeralSummaryAgent
-from letta.constants import DEFAULT_MESSAGE_TOOL, DEFAULT_MESSAGE_TOOL_KWARG, MESSAGE_SUMMARY_REQUEST_ACK
+from letta.constants import (
+    DEFAULT_MESSAGE_TOOL,
+    DEFAULT_MESSAGE_TOOL_KWARG,
+    MESSAGE_SUMMARY_REQUEST_ACK,
+    TOOL_RETURN_TRUNCATION_CHARS,
+)
+from letta.errors import ContextWindowExceededError
 from letta.helpers.message_helper import convert_message_creates_to_messages
 from letta.llm_api.llm_client import LLMClient
 from letta.log import get_logger
@@ -394,7 +400,27 @@ async def simple_summary(messages: List[Message], llm_config: LLMConfig, actor:
         response_data = await llm_client.request_async(request_data, summarizer_llm_config)
     except Exception as e:
         # handle LLM error (likely a context window exceeded error)
-        raise llm_client.handle_llm_error(e)
+        try:
+            raise llm_client.handle_llm_error(e)
+        except ContextWindowExceededError as context_error:
+            logger.warning(
+                f"Context window exceeded during summarization, falling back to truncated tool returns. Original error: {context_error}"
+            )
+
+            # Fallback: rebuild request with truncated tool returns
+            request_data = llm_client.build_request_data(
+                AgentType.letta_v1_agent,
+                input_messages_obj,
+                summarizer_llm_config,
+                tools=[],
+                tool_return_truncation_chars=TOOL_RETURN_TRUNCATION_CHARS,
+            )
+
+            try:
+                response_data = await llm_client.request_async(request_data, summarizer_llm_config)
+            except Exception as fallback_error:
+                logger.error(f"Fallback summarization also failed: {fallback_error}")
+                raise llm_client.handle_llm_error(fallback_error)
     response = llm_client.convert_response_to_chat_completion(response_data, input_messages_obj, summarizer_llm_config)
     if response.choices[0].message.content is None:
         logger.warning("No content returned from summarizer")

letta/services/tool_executor/builtin_tool_executor.py

@@ -189,7 +189,7 @@ class LettaBuiltinToolExecutor(ToolExecutor):
             return json.dumps(response, indent=2, ensure_ascii=False)
 
         except Exception as e:
-            logger.error(f"Exa search failed for query '{query}': {str(e)}")
+            logger.info(f"Exa search failed for query '{query}': {str(e)}")
             return json.dumps({"query": query, "error": f"Search failed: {str(e)}"})
 
     async def fetch_webpage(self, agent_state: "AgentState", url: str) -> str: