letta-nightly 0.6.27.dev20250220104103__py3-none-any.whl → 0.6.29.dev20250221033538__py3-none-any.whl

letta/server/rest_api/routers/openai/chat_completions/chat_completions.py

@@ -1,21 +1,30 @@
 import asyncio
-from typing import TYPE_CHECKING, Iterable, List, Optional, Union, cast
+from typing import TYPE_CHECKING, List, Optional, Union
 
+import httpx
+import openai
 from fastapi import APIRouter, Body, Depends, Header, HTTPException
 from fastapi.responses import StreamingResponse
-from openai.types.chat import ChatCompletionMessageParam
 from openai.types.chat.completion_create_params import CompletionCreateParams
+from starlette.concurrency import run_in_threadpool
 
 from letta.agent import Agent
 from letta.constants import DEFAULT_MESSAGE_TOOL, DEFAULT_MESSAGE_TOOL_KWARG
 from letta.log import get_logger
-from letta.schemas.message import MessageCreate
-from letta.schemas.openai.chat_completion_response import Message
+from letta.schemas.message import Message, MessageCreate
 from letta.schemas.user import User
 from letta.server.rest_api.chat_completions_interface import ChatCompletionsStreamingInterface
 
 # TODO this belongs in a controller!
-from letta.server.rest_api.utils import get_letta_server, sse_async_generator
+from letta.server.rest_api.utils import (
+    convert_letta_messages_to_openai,
+    create_assistant_message_from_openai_response,
+    create_user_message,
+    get_letta_server,
+    get_messages_from_completion_request,
+    sse_async_generator,
+)
+from letta.settings import model_settings
 
 if TYPE_CHECKING:
     from letta.server.server import SyncServer
@@ -25,6 +34,88 @@ router = APIRouter(prefix="/v1", tags=["chat_completions"])
 logger = get_logger(__name__)
 
 
+@router.post(
+    "/fast/chat/completions",
+    response_model=None,
+    operation_id="create_fast_chat_completions",
+    responses={
+        200: {
+            "description": "Successful response",
+            "content": {
+                "text/event-stream": {"description": "Server-Sent Events stream"},
+            },
+        }
+    },
+)
+async def create_fast_chat_completions(
+    completion_request: CompletionCreateParams = Body(...),
+    server: "SyncServer" = Depends(get_letta_server),
+    user_id: Optional[str] = Header(None, alias="user_id"),
+):
+    # TODO: This is necessary, we need to factor out CompletionCreateParams due to weird behavior
+    agent_id = str(completion_request.get("user", None))
+    if agent_id is None:
+        error_msg = "Must pass agent_id in the 'user' field"
+        logger.error(error_msg)
+        raise HTTPException(status_code=400, detail=error_msg)
+    model = completion_request.get("model")
+
+    actor = server.user_manager.get_user_or_default(user_id=user_id)
+    client = openai.AsyncClient(
+        api_key=model_settings.openai_api_key,
+        max_retries=0,
+        http_client=httpx.AsyncClient(
+            timeout=httpx.Timeout(connect=15.0, read=5.0, write=5.0, pool=5.0),
+            follow_redirects=True,
+            limits=httpx.Limits(
+                max_connections=50,
+                max_keepalive_connections=50,
+                keepalive_expiry=120,
+            ),
+        ),
+    )
+
+    # Magic message manipulating
+    input_message = get_messages_from_completion_request(completion_request)[-1]
+    completion_request.pop("messages")
+
+    # Get in context messages
+    in_context_messages = server.agent_manager.get_in_context_messages(agent_id=agent_id, actor=actor)
+    openai_dict_in_context_messages = convert_letta_messages_to_openai(in_context_messages)
+    openai_dict_in_context_messages.append(input_message)
+
+    async def event_stream():
+        # TODO: Factor this out into separate interface
+        response_accumulator = []
+
+        stream = await client.chat.completions.create(**completion_request, messages=openai_dict_in_context_messages)
+
+        async with stream:
+            async for chunk in stream:
+                if chunk.choices and chunk.choices[0].delta.content:
+                    # TODO: This does not support tool calling right now
+                    response_accumulator.append(chunk.choices[0].delta.content)
+                yield f"data: {chunk.model_dump_json()}\n\n"
+
+        # Construct messages
+        user_message = create_user_message(input_message=input_message, agent_id=agent_id, actor=actor)
+        assistant_message = create_assistant_message_from_openai_response(
+            response_text="".join(response_accumulator), agent_id=agent_id, model=str(model), actor=actor
+        )
+
+        # Persist both in one synchronous DB call, done in a threadpool
+        await run_in_threadpool(
+            server.agent_manager.append_to_in_context_messages,
+            [user_message, assistant_message],
+            agent_id=agent_id,
+            actor=actor,
+        )
+
+        yield "data: [DONE]\n\n"
+
+    return StreamingResponse(event_stream(), media_type="text/event-stream")
+
+
 @router.post(
     "/chat/completions",
     response_model=None,
@@ -44,26 +135,8 @@ async def create_chat_completions(
     user_id: Optional[str] = Header(None, alias="user_id"),
 ):
     # Validate and process fields
-    try:
-        messages = list(cast(Iterable[ChatCompletionMessageParam], completion_request["messages"]))
-    except KeyError:
-        # Handle the case where "messages" is not present in the request
-        raise HTTPException(status_code=400, detail="The 'messages' field is missing in the request.")
-    except TypeError:
-        # Handle the case where "messages" is not iterable
-        raise HTTPException(status_code=400, detail="The 'messages' field must be an iterable.")
-    except Exception as e:
-        # Catch any other unexpected errors and include the exception message
-        raise HTTPException(status_code=400, detail=f"An error occurred while processing 'messages': {str(e)}")
-
-    if messages[-1]["role"] != "user":
-        logger.error(f"The last message does not have a `user` role: {messages}")
-        raise HTTPException(status_code=400, detail="'messages[-1].role' must be a 'user'")
-
+    messages = get_messages_from_completion_request(completion_request)
     input_message = messages[-1]
-    if not isinstance(input_message["content"], str):
-        logger.error(f"The input message does not have valid content: {input_message}")
-        raise HTTPException(status_code=400, detail="'messages[-1].content' must be a 'string'")
 
     # Process remaining fields
     if not completion_request["stream"]:
@@ -138,6 +211,7 @@ async def send_message_to_agent_chat_completions(
                 agent_id=letta_agent.agent_state.id,
                 messages=messages,
                 interface=streaming_interface,
+                put_inner_thoughts_first=False,
             )
         )
 

letta/server/rest_api/routers/v1/__init__.py

@@ -1,6 +1,7 @@
 from letta.server.rest_api.routers.v1.agents import router as agents_router
 from letta.server.rest_api.routers.v1.blocks import router as blocks_router
 from letta.server.rest_api.routers.v1.health import router as health_router
+from letta.server.rest_api.routers.v1.identities import router as identities_router
 from letta.server.rest_api.routers.v1.jobs import router as jobs_router
 from letta.server.rest_api.routers.v1.llms import router as llm_router
 from letta.server.rest_api.routers.v1.providers import router as providers_router
@@ -15,6 +16,7 @@ ROUTERS = [
     tools_router,
     sources_router,
     agents_router,
+    identities_router,
     llm_router,
     blocks_router,
     jobs_router,

letta/server/rest_api/routers/v1/agents.py

@@ -1,3 +1,4 @@
+import traceback
 from datetime import datetime
 from typing import Annotated, List, Optional
 
@@ -23,6 +24,7 @@ from letta.schemas.tool import Tool
 from letta.schemas.user import User
 from letta.server.rest_api.utils import get_letta_server
 from letta.server.server import SyncServer
+from letta.tracing import trace_method
 
 # These can be forward refs, but because Fastapi needs them at runtime the must be imported normally
 
@@ -50,6 +52,7 @@ def list_agents(
     project_id: Optional[str] = Query(None, description="Search agents by project id"),
     template_id: Optional[str] = Query(None, description="Search agents by template id"),
     base_template_id: Optional[str] = Query(None, description="Search agents by base template id"),
+    identifier_keys: Optional[List[str]] = Query(None, description="Search agents by identifier keys"),
 ):
     """
     List all agents associated with a given user.
@@ -78,6 +81,7 @@ def list_agents(
         query_text=query_text,
         tags=tags,
         match_all_tags=match_all_tags,
+        identifier_keys=identifier_keys,
         **kwargs,
     )
     return agents
@@ -111,12 +115,17 @@ def create_agent(
     agent: CreateAgentRequest = Body(...),
     server: "SyncServer" = Depends(get_letta_server),
     user_id: Optional[str] = Header(None, alias="user_id"),  # Extract user_id from header, default to None if not present
+    x_project: Optional[str] = Header(None, alias="X-Project"),  # Only handled by next js middleware
 ):
     """
     Create a new agent with the specified configuration.
     """
-    actor = server.user_manager.get_user_or_default(user_id=user_id)
-    return server.create_agent(agent, actor=actor)
+    try:
+        actor = server.user_manager.get_user_or_default(user_id=user_id)
+        return server.create_agent(agent, actor=actor)
+    except Exception as e:
+        traceback.print_exc()
+        raise HTTPException(status_code=500, detail=str(e))
 
 
 @router.patch("/{agent_id}", response_model=AgentState, operation_id="modify_agent")
@@ -460,6 +469,7 @@ def modify_message(
     response_model=LettaResponse,
     operation_id="send_message",
 )
+@trace_method("POST /v1/agents/{agent_id}/messages")
 async def send_message(
     agent_id: str,
     server: SyncServer = Depends(get_letta_server),
@@ -498,6 +508,7 @@ async def send_message(
         }
     },
 )
+@trace_method("POST /v1/agents/{agent_id}/messages/stream")
 async def send_message_streaming(
     agent_id: str,
     server: SyncServer = Depends(get_letta_server),
@@ -509,7 +520,6 @@ async def send_message_streaming(
     This endpoint accepts a message from a user and processes it through the agent.
     It will stream the steps of the response always, and stream the tokens if 'stream_tokens' is set to True.
     """
-
     actor = server.user_manager.get_user_or_default(user_id=user_id)
     result = await server.send_message_to_agent(
         agent_id=agent_id,
@@ -574,6 +584,7 @@ async def process_message_background(
     response_model=Run,
     operation_id="create_agent_message_async",
 )
+@trace_method("POST /v1/agents/{agent_id}/messages/async")
 async def send_message_async(
     agent_id: str,
     background_tasks: BackgroundTasks,

letta/server/rest_api/routers/v1/identities.py

@@ -0,0 +1,121 @@
+from typing import TYPE_CHECKING, List, Optional
+
+from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query
+
+from letta.orm.errors import NoResultFound
+from letta.schemas.identity import Identity, IdentityCreate, IdentityType, IdentityUpdate
+from letta.server.rest_api.utils import get_letta_server
+
+if TYPE_CHECKING:
+    from letta.server.server import SyncServer
+
+router = APIRouter(prefix="/identities", tags=["identities"])
+
+
+@router.get("/", tags=["identities"], response_model=List[Identity], operation_id="list_identities")
+def list_identities(
+    name: Optional[str] = Query(None),
+    project_id: Optional[str] = Query(None),
+    identifier_key: Optional[str] = Query(None),
+    identity_type: Optional[IdentityType] = Query(None),
+    before: Optional[str] = Query(None),
+    after: Optional[str] = Query(None),
+    limit: Optional[int] = Query(50),
+    server: "SyncServer" = Depends(get_letta_server),
+    user_id: Optional[str] = Header(None, alias="user_id"),  # Extract user_id from header, default to None if not present
+):
+    """
+    Get a list of all identities in the database
+    """
+    try:
+        actor = server.user_manager.get_user_or_default(user_id=user_id)
+
+        identities = server.identity_manager.list_identities(
+            name=name,
+            project_id=project_id,
+            identifier_key=identifier_key,
+            identity_type=identity_type,
+            before=before,
+            after=after,
+            limit=limit,
+            actor=actor,
+        )
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"{e}")
+    return identities
+
+
+@router.get("/{identity_id}", tags=["identities"], response_model=Identity, operation_id="retrieve_identity")
+def retrieve_identity(
+    identity_id: str,
+    server: "SyncServer" = Depends(get_letta_server),
+    user_id: Optional[str] = Header(None, alias="user_id"),  # Extract user_id from header, default to None if not present
+):
+    try:
+        actor = server.user_manager.get_user_or_default(user_id=user_id)
+        return server.identity_manager.get_identity(identity_id=identity_id, actor=actor)
+    except NoResultFound as e:
+        raise HTTPException(status_code=404, detail=str(e))
+
+
+@router.post("/", tags=["identities"], response_model=Identity, operation_id="create_identity")
+def create_identity(
+    identity: IdentityCreate = Body(...),
+    server: "SyncServer" = Depends(get_letta_server),
+    user_id: Optional[str] = Header(None, alias="user_id"),  # Extract user_id from header, default to None if not present
+    x_project: Optional[str] = Header(None, alias="X-Project"),  # Only handled by next js middleware
+):
+    try:
+        actor = server.user_manager.get_user_or_default(user_id=user_id)
+        return server.identity_manager.create_identity(identity=identity, actor=actor)
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"{e}")
+
+
+@router.put("/", tags=["identities"], response_model=Identity, operation_id="upsert_identity")
+def upsert_identity(
+    identity: IdentityCreate = Body(...),
+    server: "SyncServer" = Depends(get_letta_server),
+    user_id: Optional[str] = Header(None, alias="user_id"),  # Extract user_id from header, default to None if not present
+    x_project: Optional[str] = Header(None, alias="X-Project"),  # Only handled by next js middleware
+):
+    try:
+        actor = server.user_manager.get_user_or_default(user_id=user_id)
+        return server.identity_manager.upsert_identity(identity=identity, actor=actor)
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"{e}")
+
+
+@router.patch("/{identity_id}", tags=["identities"], response_model=Identity, operation_id="update_identity")
+def modify_identity(
+    identity_id: str,
+    identity: IdentityUpdate = Body(...),
+    server: "SyncServer" = Depends(get_letta_server),
+    user_id: Optional[str] = Header(None, alias="user_id"),  # Extract user_id from header, default to None if not present
+):
+    try:
+        actor = server.user_manager.get_user_or_default(user_id=user_id)
+        return server.identity_manager.update_identity(identity_id=identity_id, identity=identity, actor=actor)
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"{e}")
+
+
+@router.delete("/{identity_id}", tags=["identities"], operation_id="delete_identity")
+def delete_identity(
+    identity_id: str,
+    server: "SyncServer" = Depends(get_letta_server),
+    user_id: Optional[str] = Header(None, alias="user_id"),  # Extract user_id from header, default to None if not present
+):
+    """
+    Delete an identity by its identifier key
+    """
+    actor = server.user_manager.get_user_or_default(user_id=user_id)
+    server.identity_manager.delete_identity(identity_id=identity_id, actor=actor)

letta/server/rest_api/utils.py

@@ -1,23 +1,33 @@
 import asyncio
 import json
 import os
+import uuid
 import warnings
+from datetime import datetime, timezone
 from enum import Enum
-from typing import TYPE_CHECKING, AsyncGenerator, Optional, Union
+from typing import TYPE_CHECKING, AsyncGenerator, Dict, Iterable, List, Optional, Union, cast
 
-from fastapi import Header
+import pytz
+from fastapi import Header, HTTPException
+from openai.types.chat import ChatCompletionMessageParam
+from openai.types.chat.chat_completion_message_tool_call import ChatCompletionMessageToolCall as OpenAIToolCall
+from openai.types.chat.chat_completion_message_tool_call import Function as OpenAIFunction
+from openai.types.chat.completion_create_params import CompletionCreateParams
 from pydantic import BaseModel
 
+from letta.constants import DEFAULT_MESSAGE_TOOL, DEFAULT_MESSAGE_TOOL_KWARG
 from letta.errors import ContextWindowExceededError, RateLimitExceededError
 from letta.log import get_logger
+from letta.schemas.enums import MessageRole
+from letta.schemas.letta_message import TextContent
+from letta.schemas.message import Message
 from letta.schemas.usage import LettaUsageStatistics
+from letta.schemas.user import User
 from letta.server.rest_api.interface import StreamingServerInterface
 
 if TYPE_CHECKING:
     from letta.server.server import SyncServer
 
-# from letta.orm.user import User
-# from letta.orm.utilities import get_db_session
 
 SSE_PREFIX = "data: "
 SSE_SUFFIX = "\n\n"
@@ -128,3 +138,172 @@ def log_error_to_sentry(e):
         import sentry_sdk
 
         sentry_sdk.capture_exception(e)
+
+
+def create_user_message(input_message: dict, agent_id: str, actor: User) -> Message:
+    """
+    Converts a user input message into the internal structured format.
+    """
+    # Generate timestamp in the correct format
+    now = datetime.now(pytz.timezone("US/Pacific")).strftime("%Y-%m-%d %I:%M:%S %p %Z%z")
+
+    # Format message as structured JSON
+    structured_message = {"type": "user_message", "message": input_message["content"], "time": now}
+
+    # Construct the Message object
+    user_message = Message(
+        id=f"message-{uuid.uuid4()}",
+        role=MessageRole.user,
+        content=[TextContent(text=json.dumps(structured_message, indent=2))],  # Store structured JSON
+        organization_id=actor.organization_id,
+        agent_id=agent_id,
+        model=None,
+        tool_calls=None,
+        tool_call_id=None,
+        created_at=datetime.now(timezone.utc),
+    )
+
+    return user_message
+
+
+def create_assistant_message_from_openai_response(
+    response_text: str,
+    agent_id: str,
+    model: str,
+    actor: User,
+) -> Message:
+    """
+    Converts an OpenAI response into a Message that follows the internal
+    paradigm where LLM responses are structured as tool calls instead of content.
+    """
+    tool_call_id = str(uuid.uuid4())
+
+    # Construct the tool call with the assistant's message
+    tool_call = OpenAIToolCall(
+        id=tool_call_id,
+        function=OpenAIFunction(
+            name=DEFAULT_MESSAGE_TOOL,
+            arguments='{\n  "' + DEFAULT_MESSAGE_TOOL_KWARG + '": ' + f'"{response_text}",\n  "request_heartbeat": true\n' + "}",
+        ),
+        type="function",
+    )
+
+    # Construct the Message object
+    assistant_message = Message(
+        id=f"message-{uuid.uuid4()}",
+        role=MessageRole.assistant,
+        content=[],
+        organization_id=actor.organization_id,
+        agent_id=agent_id,
+        model=model,
+        tool_calls=[tool_call],
+        tool_call_id=None,
+        created_at=datetime.now(timezone.utc),
+    )
+
+    return assistant_message
+
+
+def convert_letta_messages_to_openai(messages: List[Message]) -> List[dict]:
+    """
+    Flattens Letta's messages (with system, user, assistant, tool roles, etc.)
+    into standard OpenAI chat messages (system, user, assistant).
+
+    Transformation rules:
+      1. Assistant + send_message tool_call => content = tool_call's "message"
+      2. Tool (role=tool) referencing send_message => skip
+      3. User messages might store actual text inside JSON => parse that into content
+      4. System => pass through as normal
+    """
+
+    openai_messages = []
+
+    for msg in messages:
+        # 1. Assistant + 'send_message' tool_calls => flatten
+        if msg.role == MessageRole.assistant and msg.tool_calls:
+            # Find any 'send_message' tool_calls
+            send_message_calls = [tc for tc in msg.tool_calls if tc.function.name == "send_message"]
+            if send_message_calls:
+                # If we have multiple calls, just pick the first or merge them
+                # Typically there's only one.
+                tc = send_message_calls[0]
+                arguments = json.loads(tc.function.arguments)
+                # Extract the "message" string
+                extracted_text = arguments.get("message", "")
+
+                # Create a new content with the extracted text
+                msg = Message(
+                    id=msg.id,
+                    role=msg.role,
+                    content=[TextContent(text=extracted_text)],
+                    organization_id=msg.organization_id,
+                    agent_id=msg.agent_id,
+                    model=msg.model,
+                    name=msg.name,
+                    tool_calls=None,  # no longer needed
+                    tool_call_id=None,
+                    created_at=msg.created_at,
+                )
+
+        # 2. If role=tool and it's referencing send_message => skip
+        if msg.role == MessageRole.tool and msg.name == "send_message":
+            # Usually 'tool' messages with `send_message` are just status/OK messages
+            # that OpenAI doesn't need to see. So skip them.
+            continue
+
+        # 3. User messages might store text in JSON => parse it
+        if msg.role == MessageRole.user:
+            # Example: content=[TextContent(text='{"type": "user_message","message":"Hello"}')]
+            # Attempt to parse JSON and extract "message"
+            if msg.content and msg.content[0].text.strip().startswith("{"):
+                try:
+                    parsed = json.loads(msg.content[0].text)
+                    # If there's a "message" field, use that as the content
+                    if "message" in parsed:
+                        actual_user_text = parsed["message"]
+                        msg = Message(
+                            id=msg.id,
+                            role=msg.role,
+                            content=[TextContent(text=actual_user_text)],
+                            organization_id=msg.organization_id,
+                            agent_id=msg.agent_id,
+                            model=msg.model,
+                            name=msg.name,
+                            tool_calls=msg.tool_calls,
+                            tool_call_id=msg.tool_call_id,
+                            created_at=msg.created_at,
+                        )
+                except json.JSONDecodeError:
+                    pass  # It's not JSON, leave as-is
+
+        # 4. System is left as-is (or any other role that doesn't need special handling)
+        #
+        # Finally, convert to dict using your existing method
+        openai_messages.append(msg.to_openai_dict())
+
+    return openai_messages
+
+
+def get_messages_from_completion_request(completion_request: CompletionCreateParams) -> List[Dict]:
+    try:
+        messages = list(cast(Iterable[ChatCompletionMessageParam], completion_request["messages"]))
+    except KeyError:
+        # Handle the case where "messages" is not present in the request
+        raise HTTPException(status_code=400, detail="The 'messages' field is missing in the request.")
+    except TypeError:
+        # Handle the case where "messages" is not iterable
+        raise HTTPException(status_code=400, detail="The 'messages' field must be an iterable.")
+    except Exception as e:
+        # Catch any other unexpected errors and include the exception message
+        raise HTTPException(status_code=400, detail=f"An error occurred while processing 'messages': {str(e)}")
+
+    if messages[-1]["role"] != "user":
+        logger.error(f"The last message does not have a `user` role: {messages}")
+        raise HTTPException(status_code=400, detail="'messages[-1].role' must be a 'user'")
+
+    input_message = messages[-1]
+    if not isinstance(input_message["content"], str):
+        logger.error(f"The input message does not have valid content: {input_message}")
+        raise HTTPException(status_code=400, detail="'messages[-1].content' must be a 'string'")
+
+    return messages