letta-nightly 0.8.15.dev20250719104256__py3-none-any.whl → 0.8.16.dev20250721070720__py3-none-any.whl

letta/server/rest_api/app.py

@@ -2,8 +2,10 @@ import importlib.util
 import json
 import logging
 import os
+import platform
 import sys
 from contextlib import asynccontextmanager
+from functools import partial
 from pathlib import Path
 from typing import Optional
 
@@ -34,32 +36,25 @@ from letta.server.db import db_registry
 from letta.server.rest_api.auth.index import setup_auth_router  # TODO: probably remove right?
 from letta.server.rest_api.interface import StreamingServerInterface
 from letta.server.rest_api.routers.openai.chat_completions.chat_completions import router as openai_chat_completions_router
-
-# from letta.orm.utilities import get_db_session  # TODO(ethan) reenable once we merge ORM
 from letta.server.rest_api.routers.v1 import ROUTERS as v1_routes
 from letta.server.rest_api.routers.v1.organizations import router as organizations_router
 from letta.server.rest_api.routers.v1.users import router as users_router  # TODO: decide on admin
 from letta.server.rest_api.static_files import mount_static_files
+from letta.server.rest_api.utils import SENTRY_ENABLED
 from letta.server.server import SyncServer
 from letta.settings import settings
 
-# TODO(ethan)
+if SENTRY_ENABLED:
+    import sentry_sdk
+
+IS_WINDOWS = platform.system() == "Windows"
+
 # NOTE(charles): @ethan I had to add this to get the global as the bottom to work
-interface: StreamingServerInterface = StreamingServerInterface
+interface: type = StreamingServerInterface
 server = SyncServer(default_interface_factory=lambda: interface())
 logger = get_logger(__name__)
 
 
-import logging
-import platform
-
-from fastapi import FastAPI
-
-is_windows = platform.system() == "Windows"
-
-log = logging.getLogger("uvicorn")
-
-
 def generate_openapi_schema(app: FastAPI):
     # Update the OpenAPI schema
     if not app.openapi_schema:
@@ -157,6 +152,17 @@ async def lifespan(app_: FastAPI):
         logger.info(f"[Worker {worker_id}] Scheduler shutdown completed")
     except Exception as e:
         logger.error(f"[Worker {worker_id}] Scheduler shutdown failed: {e}", exc_info=True)
+
+    # Cleanup SQLAlchemy instrumentation
+    if not settings.disable_tracing and settings.sqlalchemy_tracing:
+        try:
+            from letta.otel.sqlalchemy_instrumentation_integration import teardown_letta_db_instrumentation
+
+            teardown_letta_db_instrumentation()
+            logger.info(f"[Worker {worker_id}] SQLAlchemy instrumentation shutdown completed")
+        except Exception as e:
+            logger.warning(f"[Worker {worker_id}] SQLAlchemy instrumentation shutdown failed: {e}")
+
     logger.info(f"[Worker {worker_id}] Lifespan shutdown completed")
 
 
@@ -166,9 +172,7 @@ def create_application() -> "FastAPI":
     # server = SyncServer(default_interface_factory=lambda: interface())
     print(f"\n[[ Letta server // v{letta_version} ]]")
 
-    if (os.getenv("SENTRY_DSN") is not None) and (os.getenv("SENTRY_DSN") != ""):
-        import sentry_sdk
-
+    if SENTRY_ENABLED:
         sentry_sdk.init(
             dsn=os.getenv("SENTRY_DSN"),
             traces_sample_rate=1.0,
@@ -176,6 +180,7 @@ def create_application() -> "FastAPI":
                 "continuous_profiling_auto_start": True,
             },
         )
+        logger.info("Sentry enabled.")
 
     debug_mode = "--debug" in sys.argv
     app = FastAPI(
@@ -188,31 +193,13 @@ def create_application() -> "FastAPI":
         lifespan=lifespan,
     )
 
-    @app.exception_handler(IncompatibleAgentType)
-    async def handle_incompatible_agent_type(request: Request, exc: IncompatibleAgentType):
-        return JSONResponse(
-            status_code=400,
-            content={
-                "detail": str(exc),
-                "expected_type": exc.expected_type,
-                "actual_type": exc.actual_type,
-            },
-        )
+    # === Exception Handlers ===
+    # TODO (cliandy): move to separate file
 
     @app.exception_handler(Exception)
     async def generic_error_handler(request: Request, exc: Exception):
-        # Log the actual error for debugging
-        log.error(f"Unhandled error: {str(exc)}", exc_info=True)
-        print(f"Unhandled error: {str(exc)}")
-
-        import traceback
-
-        # Print the stack trace
-        print(f"Stack trace: {traceback.format_exc()}")
-
-        if (os.getenv("SENTRY_DSN") is not None) and (os.getenv("SENTRY_DSN") != ""):
-            import sentry_sdk
-
+        logger.error(f"Unhandled error: {str(exc)}", exc_info=True)
+        if SENTRY_ENABLED:
             sentry_sdk.capture_exception(exc)
 
         return JSONResponse(
@@ -224,62 +211,70 @@ def create_application() -> "FastAPI":
             },
         )
 
-    @app.exception_handler(NoResultFound)
-    async def no_result_found_handler(request: Request, exc: NoResultFound):
-        logger.error(f"NoResultFound: {exc}")
+    async def error_handler_with_code(request: Request, exc: Exception, code: int, detail: str | None = None):
+        logger.error(f"{type(exc).__name__}", exc_info=exc)
+        if SENTRY_ENABLED:
+            sentry_sdk.capture_exception(exc)
 
+        if not detail:
+            detail = str(exc)
         return JSONResponse(
-            status_code=404,
-            content={"detail": str(exc)},
+            status_code=code,
+            content={"detail": detail},
         )
 
-    @app.exception_handler(ForeignKeyConstraintViolationError)
-    async def foreign_key_constraint_handler(request: Request, exc: ForeignKeyConstraintViolationError):
-        logger.error(f"ForeignKeyConstraintViolationError: {exc}")
+    _error_handler_400 = partial(error_handler_with_code, code=400)
+    _error_handler_404 = partial(error_handler_with_code, code=404)
+    _error_handler_404_agent = partial(_error_handler_404, detail="Agent not found")
+    _error_handler_404_user = partial(_error_handler_404, detail="User not found")
+    _error_handler_409 = partial(error_handler_with_code, code=409)
 
-        return JSONResponse(
-            status_code=409,
-            content={"detail": str(exc)},
-        )
+    app.add_exception_handler(ValueError, _error_handler_400)
+    app.add_exception_handler(NoResultFound, _error_handler_404)
+    app.add_exception_handler(LettaAgentNotFoundError, _error_handler_404_agent)
+    app.add_exception_handler(LettaUserNotFoundError, _error_handler_404_user)
+    app.add_exception_handler(ForeignKeyConstraintViolationError, _error_handler_409)
+    app.add_exception_handler(UniqueConstraintViolationError, _error_handler_409)
 
-    @app.exception_handler(UniqueConstraintViolationError)
-    async def unique_key_constraint_handler(request: Request, exc: UniqueConstraintViolationError):
-        logger.error(f"UniqueConstraintViolationError: {exc}")
+    @app.exception_handler(IncompatibleAgentType)
+    async def handle_incompatible_agent_type(request: Request, exc: IncompatibleAgentType):
+        logger.error("Incompatible agent types. Expected: %s, Actual: %s", exc.expected_type, exc.actual_type)
+        if SENTRY_ENABLED:
+            sentry_sdk.capture_exception(exc)
 
         return JSONResponse(
-            status_code=409,
-            content={"detail": str(exc)},
+            status_code=400,
+            content={
+                "detail": str(exc),
+                "expected_type": exc.expected_type,
+                "actual_type": exc.actual_type,
+            },
         )
 
     @app.exception_handler(DatabaseTimeoutError)
     async def database_timeout_error_handler(request: Request, exc: DatabaseTimeoutError):
         logger.error(f"Timeout occurred: {exc}. Original exception: {exc.original_exception}")
+        if SENTRY_ENABLED:
+            sentry_sdk.capture_exception(exc)
+
         return JSONResponse(
             status_code=503,
             content={"detail": "The database is temporarily unavailable. Please try again later."},
         )
 
-    @app.exception_handler(ValueError)
-    async def value_error_handler(request: Request, exc: ValueError):
-        return JSONResponse(status_code=400, content={"detail": str(exc)})
-
-    @app.exception_handler(LettaAgentNotFoundError)
-    async def agent_not_found_handler(request: Request, exc: LettaAgentNotFoundError):
-        return JSONResponse(status_code=404, content={"detail": "Agent not found"})
-
-    @app.exception_handler(LettaUserNotFoundError)
-    async def user_not_found_handler(request: Request, exc: LettaUserNotFoundError):
-        return JSONResponse(status_code=404, content={"detail": "User not found"})
-
     @app.exception_handler(BedrockPermissionError)
     async def bedrock_permission_error_handler(request, exc: BedrockPermissionError):
+        logger.error(f"Bedrock permission denied.")
+        if SENTRY_ENABLED:
+            sentry_sdk.capture_exception(exc)
+
         return JSONResponse(
             status_code=403,
             content={
                 "error": {
                     "type": "bedrock_permission_denied",
                     "message": "Unable to access the required AI model. Please check your Bedrock permissions or contact support.",
-                    "details": {"model_arn": exc.model_arn, "reason": str(exc)},
+                    "detail": {str(exc)},
                 }
             },
         )
@@ -290,6 +285,9 @@ def create_application() -> "FastAPI":
         print(f"▶ Using secure mode with password: {random_password}")
         app.add_middleware(CheckPasswordMiddleware)
 
+    # Add reverse proxy middleware to handle X-Forwarded-* headers
+    # app.add_middleware(ReverseProxyMiddleware, base_path=settings.server_base_path)
+
     app.add_middleware(
         CORSMiddleware,
         allow_origins=settings.cors_origins,
@@ -314,6 +312,20 @@ def create_application() -> "FastAPI":
         )
         setup_metrics(endpoint=otlp_endpoint, app=app, service_name=service_name)
 
+        # Set up SQLAlchemy synchronous operation instrumentation
+        if settings.sqlalchemy_tracing:
+            from letta.otel.sqlalchemy_instrumentation_integration import setup_letta_db_instrumentation
+
+            try:
+                setup_letta_db_instrumentation(
+                    enable_joined_monitoring=True,  # Monitor joined loading operations
+                    sql_truncate_length=1500,  # Longer SQL statements for debugging
+                )
+                print("▶ SQLAlchemy synchronous operation instrumentation enabled")
+            except Exception as e:
+                logger.warning(f"Failed to setup SQLAlchemy instrumentation: {e}")
+                # Don't fail startup if instrumentation fails
+
     for route in v1_routes:
         app.include_router(route, prefix=API_PREFIX)
         # this gives undocumented routes for "latest" and bare api calls.
@@ -371,8 +383,8 @@ def start_server(
 
     # Experimental UV Loop Support
     try:
-        if importlib.util.find_spec("uvloop") is not None and settings.use_uvloop:
-            print("Running server on uvloop...")
+        if settings.use_uvloop:
+            print("Running server asyncio loop on uvloop...")
             import asyncio
 
             import uvloop
@@ -383,7 +395,7 @@ def start_server(
 
     if (os.getenv("LOCAL_HTTPS") == "true") or "--localhttps" in sys.argv:
         print(f"▶ Server running at: https://{host or 'localhost'}:{port or REST_DEFAULT_PORT}")
-        print(f"▶ View using ADE at: https://app.letta.com/development-servers/local/dashboard\n")
+        print("▶ View using ADE at: https://app.letta.com/development-servers/local/dashboard\n")
         if importlib.util.find_spec("granian") is not None and settings.use_granian:
             from granian import Granian
 
@@ -417,7 +429,7 @@ def start_server(
             )
 
     else:
-        if is_windows:
+        if IS_WINDOWS:
             # Windows doesn't those the fancy unicode characters
             print(f"Server running at: http://{host or 'localhost'}:{port or REST_DEFAULT_PORT}")
             print(f"View using ADE at: https://app.letta.com/development-servers/local/dashboard\n")

letta/server/rest_api/routers/v1/agents.py

@@ -636,6 +636,9 @@ async def list_messages(
     use_assistant_message: bool = Query(True, description="Whether to use assistant messages"),
     assistant_message_tool_name: str = Query(DEFAULT_MESSAGE_TOOL, description="The name of the designated message tool."),
     assistant_message_tool_kwarg: str = Query(DEFAULT_MESSAGE_TOOL_KWARG, description="The name of the message argument."),
+    include_err: bool | None = Query(
+        None, description="Whether to include error messages and error statuses. For debugging purposes only."
+    ),
     actor_id: str | None = Header(None, alias="user_id"),  # Extract user_id from header, default to None if not present
 ):
     """
@@ -654,6 +657,7 @@ async def list_messages(
         use_assistant_message=use_assistant_message,
         assistant_message_tool_name=assistant_message_tool_name,
         assistant_message_tool_kwarg=assistant_message_tool_kwarg,
+        include_err=include_err,
         actor=actor,
     )
 
@@ -701,28 +705,32 @@ async def send_message(
     model_compatible = agent.llm_config.model_endpoint_type in ["anthropic", "openai", "together", "google_ai", "google_vertex", "bedrock"]
 
     # Create a new run for execution tracking
-    job_status = JobStatus.created
-    run = await server.job_manager.create_job_async(
-        pydantic_job=Run(
-            user_id=actor.id,
-            status=job_status,
-            metadata={
-                "job_type": "send_message",
-                "agent_id": agent_id,
-            },
-            request_config=LettaRequestConfig(
-                use_assistant_message=request.use_assistant_message,
-                assistant_message_tool_name=request.assistant_message_tool_name,
-                assistant_message_tool_kwarg=request.assistant_message_tool_kwarg,
-                include_return_message_types=request.include_return_message_types,
+    if settings.track_agent_run:
+        job_status = JobStatus.created
+        run = await server.job_manager.create_job_async(
+            pydantic_job=Run(
+                user_id=actor.id,
+                status=job_status,
+                metadata={
+                    "job_type": "send_message",
+                    "agent_id": agent_id,
+                },
+                request_config=LettaRequestConfig(
+                    use_assistant_message=request.use_assistant_message,
+                    assistant_message_tool_name=request.assistant_message_tool_name,
+                    assistant_message_tool_kwarg=request.assistant_message_tool_kwarg,
+                    include_return_message_types=request.include_return_message_types,
+                ),
             ),
-        ),
-        actor=actor,
-    )
+            actor=actor,
+        )
+    else:
+        run = None
+
     job_update_metadata = None
     # TODO (cliandy): clean this up
     redis_client = await get_redis_client()
-    await redis_client.set(f"{REDIS_RUN_ID_PREFIX}:{agent_id}", run.id)
+    await redis_client.set(f"{REDIS_RUN_ID_PREFIX}:{agent_id}", run.id if run else None)
 
     try:
         if agent_eligible and model_compatible:
@@ -737,7 +745,7 @@ async def send_message(
                     job_manager=server.job_manager,
                     actor=actor,
                     group=agent.multi_agent_group,
-                    current_run_id=run.id,
+                    current_run_id=run.id if run else None,
                 )
             else:
                 agent_loop = LettaAgent(
@@ -750,7 +758,7 @@ async def send_message(
                     actor=actor,
                     step_manager=server.step_manager,
                     telemetry_manager=server.telemetry_manager if settings.llm_api_logging else NoopTelemetryManager(),
-                    current_run_id=run.id,
+                    current_run_id=run.id if run else None,
                     # summarizer settings to be added here
                     summarizer_mode=(
                         SummarizationMode.STATIC_MESSAGE_BUFFER
@@ -786,12 +794,13 @@ async def send_message(
         job_status = JobStatus.failed
         raise
     finally:
-        await server.job_manager.safe_update_job_status_async(
-            job_id=run.id,
-            new_status=job_status,
-            actor=actor,
-            metadata=job_update_metadata,
-        )
+        if settings.track_agent_run:
+            await server.job_manager.safe_update_job_status_async(
+                job_id=run.id,
+                new_status=job_status,
+                actor=actor,
+                metadata=job_update_metadata,
+            )
 
 
 # noinspection PyInconsistentReturns
@@ -832,29 +841,32 @@ async def send_message_streaming(
     not_letta_endpoint = agent.llm_config.model_endpoint != LETTA_MODEL_ENDPOINT
 
     # Create a new job for execution tracking
-    job_status = JobStatus.created
-    run = await server.job_manager.create_job_async(
-        pydantic_job=Run(
-            user_id=actor.id,
-            status=job_status,
-            metadata={
-                "job_type": "send_message_streaming",
-                "agent_id": agent_id,
-            },
-            request_config=LettaRequestConfig(
-                use_assistant_message=request.use_assistant_message,
-                assistant_message_tool_name=request.assistant_message_tool_name,
-                assistant_message_tool_kwarg=request.assistant_message_tool_kwarg,
-                include_return_message_types=request.include_return_message_types,
+    if settings.track_agent_run:
+        job_status = JobStatus.created
+        run = await server.job_manager.create_job_async(
+            pydantic_job=Run(
+                user_id=actor.id,
+                status=job_status,
+                metadata={
+                    "job_type": "send_message_streaming",
+                    "agent_id": agent_id,
+                },
+                request_config=LettaRequestConfig(
+                    use_assistant_message=request.use_assistant_message,
+                    assistant_message_tool_name=request.assistant_message_tool_name,
+                    assistant_message_tool_kwarg=request.assistant_message_tool_kwarg,
+                    include_return_message_types=request.include_return_message_types,
+                ),
             ),
-        ),
-        actor=actor,
-    )
+            actor=actor,
+        )
+    else:
+        run = None
 
     job_update_metadata = None
     # TODO (cliandy): clean this up
     redis_client = await get_redis_client()
-    await redis_client.set(f"{REDIS_RUN_ID_PREFIX}:{agent_id}", run.id)
+    await redis_client.set(f"{REDIS_RUN_ID_PREFIX}:{agent_id}", run.id if run else None)
 
     try:
         if agent_eligible and model_compatible:
@@ -871,7 +883,7 @@ async def send_message_streaming(
                     step_manager=server.step_manager,
                     telemetry_manager=server.telemetry_manager if settings.llm_api_logging else NoopTelemetryManager(),
                     group=agent.multi_agent_group,
-                    current_run_id=run.id,
+                    current_run_id=run.id if run else None,
                 )
             else:
                 agent_loop = LettaAgent(
@@ -884,7 +896,7 @@ async def send_message_streaming(
                     actor=actor,
                     step_manager=server.step_manager,
                     telemetry_manager=server.telemetry_manager if settings.llm_api_logging else NoopTelemetryManager(),
-                    current_run_id=run.id,
+                    current_run_id=run.id if run else None,
                     # summarizer settings to be added here
                     summarizer_mode=(
                         SummarizationMode.STATIC_MESSAGE_BUFFER
@@ -937,12 +949,13 @@ async def send_message_streaming(
         job_status = JobStatus.failed
         raise
     finally:
-        await server.job_manager.safe_update_job_status_async(
-            job_id=run.id,
-            new_status=job_status,
-            actor=actor,
-            metadata=job_update_metadata,
-        )
+        if settings.track_agent_run:
+            await server.job_manager.safe_update_job_status_async(
+                job_id=run.id,
+                new_status=job_status,
+                actor=actor,
+                metadata=job_update_metadata,
+            )
 
 
 @router.post("/{agent_id}/messages/cancel", operation_id="cancel_agent_run")
@@ -959,6 +972,8 @@ async def cancel_agent_run(
     """
 
     actor = await server.user_manager.get_actor_or_default_async(actor_id=actor_id)
+    if not settings.track_agent_run:
+        raise HTTPException(status_code=400, detail="Agent run tracking is disabled")
     if not run_ids:
         redis_client = await get_redis_client()
         run_id = await redis_client.get(f"{REDIS_RUN_ID_PREFIX}:{agent_id}")
@@ -1156,7 +1171,7 @@ async def list_agent_groups(
 ):
     """Lists the groups for an agent"""
     actor = await server.user_manager.get_actor_or_default_async(actor_id=actor_id)
-    print("in list agents with manager_type", manager_type)
+    logger.info("in list agents with manager_type", manager_type)
     return server.agent_manager.list_groups(agent_id=agent_id, manager_type=manager_type, actor=actor)
 
 

letta/server/rest_api/routers/v1/blocks.py

@@ -72,14 +72,14 @@ async def modify_block(
     return await server.block_manager.update_block_async(block_id=block_id, block_update=block_update, actor=actor)
 
 
-@router.delete("/{block_id}", response_model=Block, operation_id="delete_block")
+@router.delete("/{block_id}", operation_id="delete_block")
 async def delete_block(
     block_id: str,
     server: SyncServer = Depends(get_letta_server),
     actor_id: Optional[str] = Header(None, alias="user_id"),
 ):
     actor = await server.user_manager.get_actor_or_default_async(actor_id=actor_id)
-    return await server.block_manager.delete_block_async(block_id=block_id, actor=actor)
+    await server.block_manager.delete_block_async(block_id=block_id, actor=actor)
 
 
 @router.get("/{block_id}", response_model=Block, operation_id="retrieve_block")

letta/server/rest_api/routers/v1/jobs.py

@@ -7,6 +7,7 @@ from letta.schemas.enums import JobStatus
 from letta.schemas.job import Job
 from letta.server.rest_api.utils import get_letta_server
 from letta.server.server import SyncServer
+from letta.settings import settings
 
 router = APIRouter(prefix="/jobs", tags=["jobs"])
 
@@ -93,6 +94,8 @@ async def cancel_job(
     agent execution to terminate as soon as possible.
     """
     actor = await server.user_manager.get_actor_or_default_async(actor_id=actor_id)
+    if not settings.track_agent_run:
+        raise HTTPException(status_code=400, detail="Agent run tracking is disabled")
 
     try:
         # First check if the job exists and is in a cancellable state

letta/server/rest_api/routers/v1/organizations.py

@@ -52,7 +52,7 @@ async def delete_org(
     try:
         org = await server.organization_manager.get_organization_by_id_async(org_id=org_id)
         if org is None:
-            raise HTTPException(status_code=404, detail=f"Organization does not exist")
+            raise HTTPException(status_code=404, detail="Organization does not exist")
         await server.organization_manager.delete_organization_by_id_async(org_id=org_id)
     except HTTPException:
         raise
@@ -70,7 +70,7 @@ async def update_org(
     try:
         org = await server.organization_manager.get_organization_by_id_async(org_id=org_id)
         if org is None:
-            raise HTTPException(status_code=404, detail=f"Organization does not exist")
+            raise HTTPException(status_code=404, detail="Organization does not exist")
         org = await server.organization_manager.update_organization_async(org_id=org_id, name=request.name)
     except HTTPException:
         raise

letta/server/rest_api/routers/v1/sources.py

@@ -99,6 +99,7 @@ async def get_source_id_by_name(
 async def get_sources_metadata(
     server: "SyncServer" = Depends(get_letta_server),
     actor_id: Optional[str] = Header(None, alias="user_id"),
+    include_detailed_per_source_metadata: bool = False,
 ):
     """
     Get aggregated metadata for all sources in an organization.
@@ -107,10 +108,12 @@ async def get_sources_metadata(
     - Total number of sources
     - Total number of files across all sources
     - Total size of all files
-    - Per-source breakdown with file details (file_name, file_size per file)
+    - Per-source breakdown with file details (file_name, file_size per file) if include_detailed_per_source_metadata is True
     """
     actor = await server.user_manager.get_actor_or_default_async(actor_id=actor_id)
-    return await server.file_manager.get_organization_sources_metadata(actor=actor)
+    return await server.file_manager.get_organization_sources_metadata(
+        actor=actor, include_detailed_per_source_metadata=include_detailed_per_source_metadata
+    )
 
 
 @router.get("/", response_model=List[Source], operation_id="list_sources")
@@ -321,6 +324,19 @@ async def upload_file_to_source(
     return file_metadata
 
 
+@router.get("/{source_id}/agents", response_model=List[str], operation_id="get_agents_for_source")
+async def get_agents_for_source(
+    source_id: str,
+    server: SyncServer = Depends(get_letta_server),
+    actor_id: Optional[str] = Header(None, alias="user_id"),
+):
+    """
+    Get all agent IDs that have the specified source attached.
+    """
+    actor = await server.user_manager.get_actor_or_default_async(actor_id=actor_id)
+    return await server.source_manager.get_agents_for_source_id(source_id=source_id, actor=actor)
+
+
 @router.get("/{source_id}/passages", response_model=List[Passage], operation_id="list_source_passages")
 async def list_source_passages(
     source_id: str,