letta-nightly 0.8.0.dev20250606195656__py3-none-any.whl → 0.8.3.dev20250607000559__py3-none-any.whl

letta/otel/metric_registry.py

@@ -0,0 +1,122 @@
+from dataclasses import dataclass, field
+from functools import partial
+
+from opentelemetry import metrics
+from opentelemetry.metrics import Counter, Histogram
+
+from letta.helpers.singleton import singleton
+from letta.otel.metrics import get_letta_meter
+
+
+@singleton
+@dataclass(frozen=True)
+class MetricRegistry:
+    """Registry of all application metrics
+
+    Metrics are composed of the following:
+        - name
+        - description
+        - unit: UCUM unit of the metric (i.e. 'By' for bytes, 'ms' for milliseconds, '1' for count
+        - bucket_bounds (list[float] | None): the explicit bucket bounds for histogram metrics
+
+        and instruments are of types Counter, Histogram, and Gauge
+
+    The relationship between the various models is as follows:
+        project_id -N:1-> base_template_id -N:1-> template_id -N:1-> agent_id
+        agent_id -1:1+-> model_name
+        agent_id -1:N -> tool_name
+    """
+
+    Instrument = Counter | Histogram
+    _metrics: dict[str, Instrument] = field(default_factory=dict, init=False)
+    _meter: metrics.Meter = field(init=False)
+
+    def __post_init__(self):
+        object.__setattr__(self, "_meter", get_letta_meter())
+
+    def _get_or_create_metric(self, name: str, factory):
+        """Lazy initialization of metrics."""
+        if name not in self._metrics:
+            self._metrics[name] = factory()
+        return self._metrics[name]
+
+    # (includes base attributes: project, template_base, template, agent)
+    @property
+    def user_message_counter(self) -> Counter:
+        return self._get_or_create_metric(
+            "count_user_message",
+            partial(
+                self._meter.create_counter,
+                name="count_user_message",
+                description="Counts the number of messages sent by the user",
+                unit="1",
+            ),
+        )
+
+    # (includes tool_name, tool_execution_success, & step_id on failure)
+    @property
+    def tool_execution_counter(self) -> Counter:
+        return self._get_or_create_metric(
+            "count_tool_execution",
+            partial(self._meter.create_counter, name="count_tool_execution", description="Counts the number of tools executed.", unit="1"),
+        )
+
+    # project_id + model
+    @property
+    def ttft_ms_histogram(self) -> Histogram:
+        return self._get_or_create_metric(
+            "hist_ttft_ms",
+            partial(self._meter.create_histogram, name="hist_ttft_ms", description="Histogram for the Time to First Token (ms)", unit="ms"),
+        )
+
+    # (includes model name)
+    @property
+    def llm_execution_time_ms_histogram(self) -> Histogram:
+        return self._get_or_create_metric(
+            "hist_llm_execution_time_ms",
+            partial(
+                self._meter.create_histogram,
+                name="hist_llm_execution_time_ms",
+                description="Histogram for LLM execution time (ms)",
+                unit="ms",
+            ),
+        )
+
+    # (includes tool name)
+    @property
+    def tool_execution_time_ms_histogram(self) -> Histogram:
+        return self._get_or_create_metric(
+            "hist_tool_execution_time_ms",
+            partial(
+                self._meter.create_histogram,
+                name="hist_tool_execution_time_ms",
+                description="Histogram for tool execution time (ms)",
+                unit="ms",
+            ),
+        )
+
+    # TODO (cliandy): instrument this
+    @property
+    def message_cost(self) -> Histogram:
+        return self._get_or_create_metric(
+            "hist_message_cost_usd",
+            partial(
+                self._meter.create_histogram,
+                name="hist_message_cost_usd",
+                description="Histogram for cost of messages (usd) per step",
+                unit="usd",
+            ),
+        )
+
+    # (includes model name)
+    @property
+    def message_output_tokens(self) -> Histogram:
+        return self._get_or_create_metric(
+            "hist_message_output_tokens",
+            partial(
+                self._meter.create_histogram,
+                name="hist_message_output_tokens",
+                description="Histogram for output tokens generated by LLM per step",
+                unit="1",
+            ),
+        )

letta/otel/metrics.py

@@ -0,0 +1,66 @@
+from fastapi import FastAPI, Request
+from opentelemetry import metrics
+from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
+from opentelemetry.metrics import NoOpMeter
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
+
+from letta.log import get_logger
+from letta.otel.context import add_ctx_attribute
+from letta.otel.resource import get_resource, is_pytest_environment
+
+logger = get_logger(__name__)
+
+_meter: metrics.Meter = NoOpMeter("noop")
+_is_metrics_initialized: bool = False
+
+
+async def _otel_metric_middleware(request: Request, call_next):
+    if not _is_metrics_initialized:
+        return await call_next(request)
+
+    header_attributes = {
+        "x-organization-id": "organization.id",
+        "x-project-id": "project.id",
+        "x-base-template-id": "base_template.id",
+        "x-template-id": "template.id",
+        "x-agent-id": "agent.id",
+    }
+    try:
+        for header_key, otel_key in header_attributes.items():
+            header_value = request.headers.get(header_key)
+            if header_value:
+                add_ctx_attribute(otel_key, header_value)
+        return await call_next(request)
+    except Exception:
+        raise
+
+
+def setup_metrics(
+    endpoint: str,
+    app: FastAPI | None = None,
+    service_name: str = "memgpt-server",
+) -> None:
+    if is_pytest_environment():
+        return
+    assert endpoint
+
+    global _is_metrics_initialized, _meter
+
+    otlp_metric_exporter = OTLPMetricExporter(endpoint=endpoint)
+    metric_reader = PeriodicExportingMetricReader(exporter=otlp_metric_exporter)
+    meter_provider = MeterProvider(resource=get_resource(service_name), metric_readers=[metric_reader])
+    metrics.set_meter_provider(meter_provider)
+    _meter = metrics.get_meter(__name__)
+
+    if app:
+        app.middleware("http")(_otel_metric_middleware)
+
+    _is_metrics_initialized = True
+
+
+def get_letta_meter() -> metrics.Meter | None:
+    """Returns the global letta meter if metrics are initialized."""
+    if not _is_metrics_initialized or isinstance(_meter, NoOpMeter):
+        logger.warning("Metrics are not initialized or meter is not available.")
+    return _meter

letta/otel/resource.py

@@ -0,0 +1,26 @@
+import os
+import sys
+import uuid
+
+from opentelemetry.sdk.resources import Resource
+
+from letta import __version__ as letta_version
+
+_resources = {}
+
+
+def get_resource(service_name: str) -> Resource:
+    _env = os.getenv("LETTA_ENVIRONMENT")
+    if service_name not in _resources:
+        resource_dict = {
+            "service.name": service_name,
+            "letta.version": letta_version,
+        }
+        if _env != "PRODUCTION":
+            resource_dict["device.id"] = uuid.getnode()  # MAC address as unique device identifier,
+        _resources[(service_name, _env)] = Resource.create(resource_dict)
+    return _resources[(service_name, _env)]
+
+
+def is_pytest_environment():
+    return "pytest" in sys.modules

letta/{tracing.py → otel/tracing.py}

@@ -1,6 +1,5 @@
 import inspect
 import re
-import sys
 import time
 from functools import wraps
 from typing import Any, Dict, List, Optional
@@ -11,15 +10,18 @@ from fastapi.responses import JSONResponse
 from opentelemetry import trace
 from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
 from opentelemetry.instrumentation.requests import RequestsInstrumentor
-from opentelemetry.sdk.resources import Resource
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import BatchSpanProcessor
 from opentelemetry.trace import Status, StatusCode
 
-from letta import __version__ as letta_version
+from letta.log import get_logger
+from letta.otel.resource import get_resource, is_pytest_environment
+from letta.settings import settings
 
+logger = get_logger(__name__)  # TODO: set up logger config for this
 tracer = trace.get_tracer(__name__)
 _is_tracing_initialized = False
+
 _excluded_v1_endpoints_regex: List[str] = [
     # "^GET /v1/agents/(?P<agent_id>[^/]+)/messages$",
     # "^GET /v1/agents/(?P<agent_id>[^/]+)/context$",
@@ -30,11 +32,7 @@ _excluded_v1_endpoints_regex: List[str] = [
 ]
 
 
-def is_pytest_environment():
-    return "pytest" in sys.modules
-
-
-async def trace_request_middleware(request: Request, call_next):
+async def _trace_request_middleware(request: Request, call_next):
     if not _is_tracing_initialized:
         return await call_next(request)
     initial_span_name = f"{request.method} {request.url.path}"
@@ -56,7 +54,7 @@ async def trace_request_middleware(request: Request, call_next):
             raise
 
 
-async def update_trace_attributes(request: Request):
+async def _update_trace_attributes(request: Request):
     """Dependency to update trace attributes after FastAPI has processed the request"""
     if not _is_tracing_initialized:
         return
@@ -78,35 +76,19 @@ async def update_trace_attributes(request: Request):
     for key, value in request.path_params.items():
         span.set_attribute(f"http.{key}", value)
 
-    # Add user ID if available
-    user_id = request.headers.get("user_id")
-    if user_id:
-        span.set_attribute("user.id", user_id)
-
-    # Add organization_id if available
-    organization_id = request.headers.get("x-organization-id")
-    if organization_id:
-        span.set_attribute("organization.id", organization_id)
-
-    # Add project_id if available
-    project_id = request.headers.get("x-project-id")
-    if project_id:
-        span.set_attribute("project.id", project_id)
-
-    # Add agent_id if available
-    agent_id = request.headers.get("x-agent-id")
-    if agent_id:
-        span.set_attribute("agent.id", agent_id)
-
-    # Add template_id if available
-    template_id = request.headers.get("x-template-id")
-    if template_id:
-        span.set_attribute("template.id", template_id)
-
-    # Add base_template_id if available
-    base_template_id = request.headers.get("x-base-template-id")
-    if base_template_id:
-        span.set_attribute("base_template.id", base_template_id)
+    # Add the following headers to span if available
+    header_attributes = {
+        "user_id": "user.id",
+        "x-organization-id": "organization.id",
+        "x-project-id": "project.id",
+        "x-agent-id": "agent.id",
+        "x-template-id": "template.id",
+        "x-base-template-id": "base_template.id",
+    }
+    for header_key, span_key in header_attributes.items():
+        header_value = request.headers.get(header_key)
+        if header_value:
+            span.set_attribute(span_key, header_value)
 
     # Add request body if available
     try:
@@ -117,7 +99,7 @@ async def update_trace_attributes(request: Request):
         pass
 
 
-async def trace_error_handler(_request: Request, exc: Exception) -> JSONResponse:
+async def _trace_error_handler(_request: Request, exc: Exception) -> JSONResponse:
     status_code = getattr(exc, "status_code", 500)
     error_msg = str(exc)
 
@@ -142,49 +124,44 @@ def setup_tracing(
 ) -> None:
     if is_pytest_environment():
         return
+    assert endpoint
 
     global _is_tracing_initialized
 
-    provider = TracerProvider(resource=Resource.create({"service.name": service_name}))
-    import uuid
+    tracer_provider = TracerProvider(resource=get_resource(service_name))
+    tracer_provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint)))
+    _is_tracing_initialized = True
+    trace.set_tracer_provider(tracer_provider)
 
-    provider = TracerProvider(
-        resource=Resource.create(
-            {
-                "service.name": service_name,
-                "device.id": uuid.getnode(),  # MAC address as unique device identifier,
-                "letta.version": letta_version,
-            }
-        )
-    )
-    if endpoint:
-        provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint)))
-        _is_tracing_initialized = True
-        trace.set_tracer_provider(provider)
-
-        def requests_callback(span: trace.Span, _: Any, response: Any) -> None:
-            if hasattr(response, "status_code"):
-                span.set_status(Status(StatusCode.OK if response.status_code < 400 else StatusCode.ERROR))
-
-        RequestsInstrumentor().instrument(response_hook=requests_callback)
-
-        if app:
-            # Add middleware first
-            app.middleware("http")(trace_request_middleware)
-
-            # Add dependency to v1 routes
-            from letta.server.rest_api.routers.v1 import ROUTERS as v1_routes
-
-            for router in v1_routes:
-                for route in router.routes:
-                    full_path = ((next(iter(route.methods)) + " ") if route.methods else "") + "/v1" + route.path
-                    if not any(re.match(regex, full_path) for regex in _excluded_v1_endpoints_regex):
-                        route.dependencies.append(Depends(update_trace_attributes))
-
-            # Register exception handlers
-            app.exception_handler(HTTPException)(trace_error_handler)
-            app.exception_handler(RequestValidationError)(trace_error_handler)
-            app.exception_handler(Exception)(trace_error_handler)
+    # Instrumentors (e.g., RequestsInstrumentor)
+    def requests_callback(span: trace.Span, _: Any, response: Any) -> None:
+        if hasattr(response, "status_code"):
+            span.set_status(Status(StatusCode.OK if response.status_code < 400 else StatusCode.ERROR))
+
+    RequestsInstrumentor().instrument(response_hook=requests_callback)
+
+    if settings.sqlalchemy_tracing:
+        from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
+
+        SQLAlchemyInstrumentor().instrument()
+
+    if app:
+        # Add middleware first
+        app.middleware("http")(_trace_request_middleware)
+
+        # Add dependency to v1 routes
+        from letta.server.rest_api.routers.v1 import ROUTERS as V1_ROUTES
+
+        for router in V1_ROUTES:
+            for route in router.routes:
+                full_path = ((next(iter(route.methods)) + " ") if route.methods else "") + "/v1" + route.path
+                if not any(re.match(regex, full_path) for regex in _excluded_v1_endpoints_regex):
+                    route.dependencies.append(Depends(_update_trace_attributes))
+
+        # Register exception handlers for tracing
+        app.exception_handler(HTTPException)(_trace_error_handler)
+        app.exception_handler(RequestValidationError)(_trace_error_handler)
+        app.exception_handler(Exception)(_trace_error_handler)
 
 
 def trace_method(func):

letta/plugins/README.md

@@ -0,0 +1,22 @@
+### Plugins
+
+Plugins enable plug and play for various components.
+
+Plugin configurations can be set in `letta.settings.settings`.
+
+The plugins will take a delimited list of consisting of individual plugin configs:
+
+`<plugin_name>.<config_name>=<class_or_function>`
+
+joined by `;`
+
+In the default configuration, the top level keys have values `plugin_name`,
+the `config_name` is nested under and the `class_or_function` is defined
+after in format `<module_path>:<name>`.
+
+```
+DEFAULT_PLUGINS = {
+    "experimental_check": {
+        "default": "letta.plugins.defaults:is_experimental_enabled",
+        ...
+```

letta/plugins/defaults.py

@@ -0,0 +1,11 @@
+from letta.settings import settings
+
+
+def is_experimental_enabled(feature_name: str, **kwargs) -> bool:
+    if feature_name in ("async_agent_loop", "summarize"):
+        if not (kwargs.get("eligibility", False) and settings.use_experimental):
+            return False
+        return True
+
+    # Err on safety here, disabling experimental if not handled here.
+    return False

letta/plugins/plugins.py

@@ -0,0 +1,72 @@
+import importlib
+from typing import Protocol, runtime_checkable
+
+from letta.settings import settings
+
+
+@runtime_checkable
+class SummarizerProtocol(Protocol):
+    """What a summarizer must implement"""
+
+    async def summarize(self, text: str) -> str: ...
+    def get_name(self) -> str: ...
+
+
+# Currently this supports one of each plugin type. This can be expanded in the future.
+DEFAULT_PLUGINS = {
+    "experimental_check": {
+        "protocol": None,
+        "target": "letta.plugins.defaults:is_experimental_enabled",
+    },
+    "summarizer": {
+        "protocol": SummarizerProtocol,
+        "target": "letta.services.summarizer.summarizer:Summarizer",
+    },
+}
+
+
+def get_plugin(plugin_type: str):
+    """Get a plugin instance"""
+    plugin_register = dict(DEFAULT_PLUGINS, **settings.plugin_register_dict)
+    if plugin_type in plugin_register:
+        impl_path = plugin_register[plugin_type]["target"]
+        module_path, name = impl_path.split(":")
+        module = importlib.import_module(module_path)
+        plugin = getattr(module, name)
+        if type(plugin).__name__ == "function":
+            return plugin
+        elif type(plugin).__name__ == "class":
+            if plugin_register["protocol"] and not isinstance(plugin, type(plugin_register["protocol"])):
+                raise TypeError(f'{plugin} does not implement {type(plugin_register["protocol"]).__name__}')
+            return plugin()
+    raise TypeError("Unknown plugin type")
+
+
+_experimental_checker = None
+_summarizer = None
+
+
+# TODO handle coroutines
+# Convenience functions
+def get_experimental_checker():
+    global _experimental_checker
+    if _experimental_checker is None:
+        _experimental_checker = get_plugin("experimental_check")
+    return _experimental_checker
+
+
+def get_summarizer():
+    global _summarizer
+    if _summarizer is None:
+        _summarizer = get_plugin("summarizer")
+    return _summarizer
+
+
+def reset_experimental_checker():
+    global _experimental_checker
+    _experimental_checker = None
+
+
+def reset_summarizer():
+    global _summarizer
+    _summarizer = None

letta/schemas/enums.py

@@ -87,3 +87,11 @@ class ToolRuleType(str, Enum):
     constrain_child_tools = "constrain_child_tools"
     max_count_per_step = "max_count_per_step"
     parent_last_tool = "parent_last_tool"
+
+
+class FileProcessingStatus(str, Enum):
+    PENDING = "pending"
+    PARSING = "parsing"
+    EMBEDDING = "embedding"
+    COMPLETED = "completed"
+    ERROR = "error"

letta/schemas/file.py

@@ -4,6 +4,7 @@ from typing import Optional
 
 from pydantic import Field
 
+from letta.schemas.enums import FileProcessingStatus
 from letta.schemas.letta_base import LettaBase
 
 
@@ -34,12 +35,22 @@ class FileMetadata(FileMetadataBase):
     file_size: Optional[int] = Field(None, description="The size of the file in bytes.")
     file_creation_date: Optional[str] = Field(None, description="The creation date of the file.")
     file_last_modified_date: Optional[str] = Field(None, description="The last modified date of the file.")
+    processing_status: FileProcessingStatus = Field(
+        default=FileProcessingStatus.PENDING,
+        description="The current processing status of the file (e.g. pending, parsing, embedding, completed, error).",
+    )
+    error_message: Optional[str] = Field(default=None, description="Optional error message if the file failed processing.")
 
     # orm metadata, optional fields
     created_at: Optional[datetime] = Field(default_factory=datetime.utcnow, description="The creation date of the file.")
     updated_at: Optional[datetime] = Field(default_factory=datetime.utcnow, description="The update date of the file.")
     is_deleted: bool = Field(False, description="Whether this file is deleted or not.")
 
+    # This is optional, and only occasionally pulled in since it can be very large
+    content: Optional[str] = Field(
+        default=None, description="Optional full-text content of the file; only populated on demand due to its size."
+    )
+
 
 class FileAgentBase(LettaBase):
     """Base class for the FileMetadata-⇄-Agent association schemas"""
@@ -67,6 +78,7 @@ class FileAgent(FileAgentBase):
     )
     agent_id: str = Field(..., description="Unique identifier of the agent.")
     file_id: str = Field(..., description="Unique identifier of the file.")
+    file_name: str = Field(..., description="Name of the file.")
     is_open: bool = Field(True, description="True if the agent currently has the file open.")
     visible_content: Optional[str] = Field(
         None,

letta/schemas/letta_request.py

@@ -3,6 +3,7 @@ from typing import List, Optional
 from pydantic import BaseModel, Field, HttpUrl
 
 from letta.constants import DEFAULT_MESSAGE_TOOL, DEFAULT_MESSAGE_TOOL_KWARG
+from letta.schemas.letta_message import MessageType
 from letta.schemas.message import MessageCreate
 
 
@@ -21,6 +22,11 @@ class LettaRequest(BaseModel):
         description="The name of the message argument in the designated message tool.",
     )
 
+    # filter to only return specific message types
+    include_return_message_types: Optional[List[MessageType]] = Field(
+        default=None, description="Only return specified message types in the response. If `None` (default) returns all messages."
+    )
+
 
 class LettaStreamingRequest(LettaRequest):
     stream_tokens: bool = Field(

letta/schemas/passage.py

@@ -23,6 +23,7 @@ class PassageBase(OrmMetadataBase):
 
     # file association
     file_id: Optional[str] = Field(None, description="The unique identifier of the file associated with the passage.")
+    file_name: Optional[str] = Field(None, description="The name of the file (only for source passages).")
     metadata: Optional[Dict] = Field({}, validation_alias="metadata_", description="The metadata of the passage.")
 
 

letta/schemas/tool.py

@@ -7,6 +7,7 @@ from letta.constants import (
     FUNCTION_RETURN_CHAR_LIMIT,
     LETTA_BUILTIN_TOOL_MODULE_NAME,
     LETTA_CORE_TOOL_MODULE_NAME,
+    LETTA_FILES_TOOL_MODULE_NAME,
     LETTA_MULTI_AGENT_TOOL_MODULE_NAME,
     LETTA_VOICE_TOOL_MODULE_NAME,
     MCP_TOOL_TAG_NAME_PREFIX,
@@ -106,6 +107,9 @@ class Tool(BaseTool):
         elif self.tool_type in {ToolType.LETTA_BUILTIN}:
             # If it's letta voice tool, we generate the json_schema on the fly here
             self.json_schema = get_json_schema_from_module(module_name=LETTA_BUILTIN_TOOL_MODULE_NAME, function_name=self.name)
+        elif self.tool_type in {ToolType.LETTA_FILES_CORE}:
+            # If it's letta files tool, we generate the json_schema on the fly here
+            self.json_schema = get_json_schema_from_module(module_name=LETTA_FILES_TOOL_MODULE_NAME, function_name=self.name)
         elif self.tool_type in {ToolType.EXTERNAL_COMPOSIO}:
             # Composio schemas handled separately
             pass

letta/server/db.py

@@ -13,8 +13,8 @@ from sqlalchemy.orm import sessionmaker
 
 from letta.config import LettaConfig
 from letta.log import get_logger
+from letta.otel.tracing import trace_method
 from letta.settings import settings
-from letta.tracing import trace_method
 
 logger = get_logger(__name__)
 
@@ -131,7 +131,12 @@ class DatabaseRegistry:
             # Create async session factory
             self._async_engines["default"] = async_engine
             self._async_session_factories["default"] = async_sessionmaker(
-                close_resets_only=False, autocommit=False, autoflush=False, bind=self._async_engines["default"], class_=AsyncSession
+                expire_on_commit=True,
+                close_resets_only=False,
+                autocommit=False,
+                autoflush=False,
+                bind=self._async_engines["default"],
+                class_=AsyncSession,
             )
             self._initialized["async"] = True
 
@@ -207,11 +212,6 @@ class DatabaseRegistry:
         self.initialize_sync()
         return self._engines.get(name)
 
-    def get_async_engine(self, name: str = "default") -> AsyncEngine:
-        """Get an async database engine by name."""
-        self.initialize_async()
-        return self._async_engines.get(name)
-
     def get_session_factory(self, name: str = "default") -> sessionmaker:
         """Get a session factory by name."""
         self.initialize_sync()