arize-phoenix 11.23.1__py3-none-any.whl → 12.28.1__py3-none-any.whl

phoenix/server/main.py

@@ -7,7 +7,7 @@ from pathlib import Path
 from ssl import CERT_REQUIRED
 from threading import Thread
 from time import sleep, time
-from typing import Optional
+from typing import Awaitable, Callable, Optional
 from urllib.parse import urljoin
 
 from jinja2 import BaseLoader, Environment
@@ -37,6 +37,7 @@ from phoenix.config import (
     get_env_password_reset_token_expiry,
     get_env_port,
     get_env_refresh_token_expiry,
+    get_env_scarf_sh_pixel_id,
     get_env_smtp_hostname,
     get_env_smtp_mail_from,
     get_env_smtp_password,
@@ -371,7 +372,10 @@ def main() -> None:
         start_prometheus()
 
     engine = create_engine_and_run_migrations(db_connection_str)
-    instrumentation_cleanups = instrument_engine_if_enabled(engine)
+    shutdown_callbacks: list[Callable[[], None | Awaitable[None]]] = []
+    shutdown_callbacks.extend(instrument_engine_if_enabled(engine))
+    # Ensure engine is disposed on shutdown to properly close database connections
+    shutdown_callbacks.append(engine.dispose)
     factory = DbSessionFactory(db=_db(engine), dialect=engine.dialect.name)
     corpus_model = (
         None if corpus_inferences is None else create_model_from_inferences(corpus_inferences)
@@ -448,7 +452,7 @@ def main() -> None:
         initial_spans=fixture_spans,
         initial_evaluations=fixture_evals,
         startup_callbacks=[lambda: print(msg)],
-        shutdown_callbacks=instrumentation_cleanups,
+        shutdown_callbacks=shutdown_callbacks,
         secret=auth_settings.phoenix_secret,
         password_reset_token_expiry=get_env_password_reset_token_expiry(),
         access_token_expiry=get_env_access_token_expiry(),
@@ -456,6 +460,7 @@ def main() -> None:
         scaffolder_config=scaffolder_config,
         email_sender=email_sender,
         oauth2_client_configs=get_env_oauth2_settings(),
+        ldap_config=auth_settings.ldap_config,
         allowed_origins=allowed_origins,
         management_url=management_url,
     )
@@ -498,6 +503,7 @@ def initialize_settings() -> None:
     Settings.log_migrations = get_env_log_migrations()
     Settings.disable_migrations = get_env_disable_migrations()
     Settings.fullstory_org = get_env_fullstory_org()
+    Settings.scarf_sh_pixel_id = get_env_scarf_sh_pixel_id()
 
 
 if __name__ == "__main__":

phoenix/server/oauth2.py

@@ -1,12 +1,16 @@
-from collections.abc import Iterable
-from typing import Any, Iterator, Optional
+import logging
+from collections.abc import Iterable, Mapping
+from typing import Any, Iterator, Optional, get_args
 
+import jmespath
 from authlib.integrations.base_client import BaseApp
 from authlib.integrations.base_client.async_app import AsyncOAuth2Mixin
 from authlib.integrations.base_client.async_openid import AsyncOpenIDMixin
 from authlib.integrations.httpx_client import AsyncOAuth2Client as AsyncHttpxOAuth2Client
 
-from phoenix.config import OAuth2ClientConfig
+from phoenix.config import AssignableUserRoleName, OAuth2ClientConfig
+
+logger = logging.getLogger(__name__)
 
 
 class OAuth2Client(AsyncOAuth2Mixin, AsyncOpenIDMixin, BaseApp):  # type:ignore[misc]
@@ -25,13 +29,78 @@ class OAuth2Client(AsyncOAuth2Mixin, AsyncOpenIDMixin, BaseApp):  # type:ignore[
         display_name: str,
         allow_sign_up: bool,
         auto_login: bool,
+        use_pkce: bool = False,
+        groups_attribute_path: Optional[str] = None,
+        allowed_groups: Optional[list[str]] = None,
+        role_attribute_path: Optional[str] = None,
+        role_mapping: Optional[Mapping[str, AssignableUserRoleName]] = None,
+        role_attribute_strict: bool = False,
         **kwargs: Any,
     ) -> None:
         self._display_name = display_name
         self._allow_sign_up = allow_sign_up
         self._auto_login = auto_login
+        self._use_pkce = use_pkce
+
+        self._groups_attribute_path = (
+            groups_attribute_path.strip()
+            if groups_attribute_path and groups_attribute_path.strip()
+            else None
+        )
+
+        if allowed_groups:
+            self._allowed_groups = {g for g in allowed_groups if g.strip()}
+        else:
+            self._allowed_groups = set()
+
+        if self._allowed_groups and not self._groups_attribute_path:
+            raise ValueError(
+                "groups_attribute_path must be specified when allowed_groups is configured. "
+                "Group-based access control requires both parameters to be set."
+            )
+
+        if self._groups_attribute_path and not self._allowed_groups:
+            raise ValueError(
+                "allowed_groups must be specified when groups_attribute_path is configured. "
+                "Group-based access control requires both parameters to be set. "
+                "If you don't need group-based access control, remove groups_attribute_path."
+            )
+
+        self._compiled_groups_path = self._compile_jmespath_expression(
+            self._groups_attribute_path, "GROUPS_ATTRIBUTE_PATH"
+        )
+
+        # Role mapping configuration
+        self._role_attribute_path = (
+            role_attribute_path.strip()
+            if role_attribute_path and role_attribute_path.strip()
+            else None
+        )
+        self._role_mapping = role_mapping
+        self._role_attribute_strict = role_attribute_strict
+        self._compiled_role_path = self._compile_jmespath_expression(
+            self._role_attribute_path, "ROLE_ATTRIBUTE_PATH"
+        )
+
         super().__init__(framework=None, *args, **kwargs)
-        self._allow_sign_up = allow_sign_up
+
+    @staticmethod
+    def _compile_jmespath_expression(
+        path: Optional[str], attribute_name: str
+    ) -> Optional[jmespath.parser.ParsedResult]:
+        """Validate and compile JMESPath expression at startup for fail-fast behavior."""
+        if not path:
+            return None
+
+        try:
+            return jmespath.compile(path)
+        except (jmespath.exceptions.JMESPathError, jmespath.exceptions.ParseError) as e:
+            raise ValueError(
+                f"Invalid JMESPath expression in {attribute_name}: '{path}'. Error: {e}. "
+                "Hint: Claim keys with special characters (colons, dots, slashes, hyphens) "
+                "must be enclosed in double quotes. "
+                "Examples: '\"cognito:groups\"', '\"https://myapp.com/groups\"'"
+            ) from e
 
     @property
     def allow_sign_up(self) -> bool:
@@ -45,6 +114,240 @@ class OAuth2Client(AsyncOAuth2Mixin, AsyncOpenIDMixin, BaseApp):  # type:ignore[
     def display_name(self) -> str:
         return self._display_name
 
+    @property
+    def use_pkce(self) -> bool:
+        return self._use_pkce
+
+    def has_sufficient_claims(self, claims: dict[str, Any]) -> bool:
+        """
+        Check if the ID token contains all application-required claims.
+
+        OIDC Core §2 mandates that ID tokens contain authentication claims (iss, sub, aud,
+        exp, iat), but user profile claims (email, name, groups, roles) are optional and may
+        only be available via UserInfo endpoint (§5.4, §5.5). This method determines if we
+        need to call UserInfo.
+
+        Application-required claims:
+        - email: Required for user identification and account creation
+        - groups: Required if group-based access control is configured
+        - roles: Required if role mapping is configured
+
+        If any required claim is missing, returns False to trigger UserInfo endpoint call.
+
+        Args:
+            claims: Claims from ID token (OIDC Core §3.1.3.3)
+
+        Returns:
+            True if all application-required claims are present (UserInfo not needed)
+            False if additional claims must be fetched from UserInfo endpoint
+        """
+        # Check for email claim (required by application)
+        email = claims.get("email")
+        if not email or not isinstance(email, str) or not email.strip():
+            # Email missing or invalid, need UserInfo
+            return False
+
+        # Check for group claims if group-based access control is configured
+        if self._compiled_groups_path:
+            groups = self._extract_groups_from_claims(claims)
+            if len(groups) == 0:
+                # Groups required but not present, need UserInfo
+                return False
+
+        # Check for role claims if role mapping is configured
+        if self._compiled_role_path:
+            # Check if role claim EXISTS (not whether it maps successfully)
+            # Optimization: If the claim exists but doesn't map, UserInfo won't help
+            result = self._compiled_role_path.search(claims)
+            role_value = self._normalize_to_single_string(result)
+            if not role_value:
+                # Role claim missing - UserInfo might have a mappable role
+                # (could upgrade from default VIEWER to ADMIN/MEMBER)
+                return False
+            # Role exists - UserInfo won't help even if role doesn't map
+            # (UserInfo will have the same unmappable role)
+
+        # All required claims present
+        return True
+
+    def validate_access(self, user_claims: dict[str, Any]) -> None:
+        """
+        Validate that the user has access based on configured claim-based access control.
+
+        Currently supports group-based access control. In the future, this may be extended
+        to support organization-based or other claim-based authorization mechanisms.
+
+        Args:
+            user_claims: Claims from the OIDC ID token (OIDC Core §3.1.3.3) or userinfo
+                endpoint (OIDC Core §5.3). Custom claims for groups/roles are extracted
+                per OIDC Core §5.1.2 (Additional Claims).
+
+        Raises:
+            PermissionError: If user doesn't meet the access requirements
+        """
+        if not self._allowed_groups or not self._groups_attribute_path:
+            return
+
+        user_groups = self._extract_groups_from_claims(user_claims)
+
+        if not any(group in self._allowed_groups for group in user_groups):
+            raise PermissionError(
+                "Access denied. Your account does not belong to any authorized groups."
+            )
+
+    def _extract_groups_from_claims(self, claims: dict[str, Any]) -> list[str]:
+        """Extract group values from claims using the configured JMESPath expression."""
+        if not self._compiled_groups_path:
+            return []
+
+        result = self._compiled_groups_path.search(claims)
+        return self._normalize_to_string_list(result)
+
+    @staticmethod
+    def _normalize_to_string_list(value: Any) -> list[str]:
+        """
+        Normalize a JMESPath result to a list of strings.
+
+        Handles common OIDC claim formats: single values, lists, and scalar types.
+        Non-scalar items (dicts, nested lists) are silently skipped.
+
+        Args:
+            value: Result from JMESPath query
+
+        Returns:
+            List of string values, or empty list if value cannot be normalized
+        """
+        if value is None:
+            return []
+
+        if isinstance(value, str):
+            return [value]
+
+        if isinstance(value, (int, float, bool)):
+            return [str(value)]
+
+        if isinstance(value, list):
+            return [
+                str(item) if isinstance(item, (int, float, bool)) else item
+                for item in value
+                if isinstance(item, (str, int, float, bool))
+            ]
+
+        return []
+
+    def extract_and_map_role(self, user_claims: dict[str, Any]) -> Optional[AssignableUserRoleName]:
+        """
+        Extract and map user role from OIDC claims.
+
+        This method extracts the role claim using the configured JMESPath expression,
+        optionally applies role mapping to translate IDP role values to Phoenix roles,
+        and handles missing/invalid roles based on the strict mode setting.
+
+        Role Mapping Flow:
+        1. Extract role claim using ROLE_ATTRIBUTE_PATH (JMESPath)
+           - Supports simple paths: "role", "user.org.role"
+           - Supports conditional logic: "contains(groups[*], 'admin') && 'ADMIN' || 'VIEWER'"
+        2. Apply ROLE_MAPPING (if configured) to translate IDP role → Phoenix role
+           - If ROLE_MAPPING not set, use extracted value directly if valid (ADMIN/MEMBER/VIEWER)
+           - This allows JMESPath expressions to return Phoenix roles directly
+        3. Validate Phoenix role (ADMIN, MEMBER, VIEWER - SYSTEM excluded for OAuth)
+        4. Handle missing/invalid roles:
+           - strict=True: Raise PermissionError (deny access)
+           - strict=False: Return "VIEWER" (default, least privilege)
+
+        IMPORTANT: Backward Compatibility
+        - If ROLE_ATTRIBUTE_PATH is NOT configured, returns None
+        - This preserves existing users' roles (no unwanted downgrades)
+        - Caller should only apply "VIEWER" default for NEW users
+
+        Args:
+            user_claims: Claims from the OIDC ID token or userinfo endpoint
+
+        Returns:
+            Phoenix role name (ADMIN, MEMBER, or VIEWER), or None if role attribute
+            path is not configured (to preserve existing user roles)
+
+        Raises:
+            PermissionError: If strict mode is enabled and role cannot be determined
+        """
+        # If no role mapping configured, return None to preserve existing user roles
+        if not self._compiled_role_path:
+            return None
+
+        # Extract role from claims
+        result = self._compiled_role_path.search(user_claims)
+        role_value = self._normalize_to_single_string(result)
+
+        # If role claim is missing or empty
+        if not role_value:
+            if self._role_attribute_strict:
+                raise PermissionError(
+                    f"Access denied: Role claim not found in user claims. "
+                    f"Role attribute path '{self._role_attribute_path}' is configured with "
+                    f"strict mode enabled."
+                )
+            return "VIEWER"  # Non-strict: default to least privilege
+
+        # Apply role mapping if configured
+        if self._role_mapping:
+            mapped_role = self._role_mapping.get(role_value)
+            if not mapped_role:
+                # Role value doesn't match any mapping
+                if self._role_attribute_strict:
+                    raise PermissionError(
+                        f"Access denied: Role '{role_value}' is not mapped to a Phoenix role. "
+                        f"Role mapping is configured with strict mode enabled."
+                    )
+                return "VIEWER"  # Non-strict: default to least privilege
+            return mapped_role
+
+        # No role mapping configured, but role path exists
+        # Try to use the raw role value directly if it's a valid Phoenix role
+        # Note: SYSTEM is excluded from valid roles for OIDC (validated at config parsing)
+        role_upper = role_value.upper()
+        if role_upper in get_args(AssignableUserRoleName):
+            return role_upper  # type: ignore[return-value]
+
+        # Role value is not a valid Phoenix role
+        if self._role_attribute_strict:
+            raise PermissionError(
+                f"Access denied: Role '{role_value}' is not a valid Phoenix role "
+                f"(expected ADMIN, MEMBER, or VIEWER). Strict mode is enabled."
+            )
+        return "VIEWER"  # Non-strict: default to least privilege
+
+    @staticmethod
+    def _normalize_to_single_string(value: Any) -> Optional[str]:
+        """
+        Normalize a JMESPath result to a single string value.
+
+        Handles common OIDC claim formats for single-value fields like role.
+        If the result is a list, takes the first element.
+
+        Args:
+            value: Result from JMESPath query
+
+        Returns:
+            String value or None if value cannot be normalized
+        """
+        if value is None:
+            return None
+
+        if isinstance(value, str):
+            return value.strip() or None
+
+        if isinstance(value, (int, float, bool)):
+            return str(value)
+
+        if isinstance(value, list) and len(value) > 0:
+            first = value[0]
+            if isinstance(first, str):
+                return first.strip() or None
+            if isinstance(first, (int, float, bool)):
+                return str(first)
+
+        return None
+
 
 class OAuth2Clients:
     def __init__(self) -> None:
@@ -67,26 +370,41 @@ class OAuth2Clients:
     def add_client(self, config: OAuth2ClientConfig) -> None:
         if (idp_name := config.idp_name) in self._clients:
             raise ValueError(f"oauth client already registered: {idp_name}")
+        # RFC 6749 §3.3: scope parameter (space-delimited list of scopes)
+        client_kwargs = {"scope": config.scopes}
+
+        if config.token_endpoint_auth_method:
+            # OIDC Core §9: Client authentication method at token endpoint
+            client_kwargs["token_endpoint_auth_method"] = config.token_endpoint_auth_method
+        if config.use_pkce:
+            # Always use S256 for PKCE (RFC 7636 §4.2: SHA-256 code challenge method)
+            client_kwargs["code_challenge_method"] = "S256"
+
         client = OAuth2Client(
             name=config.idp_name,
-            client_id=config.client_id,
-            client_secret=config.client_secret,
-            server_metadata_url=config.oidc_config_url,
-            client_kwargs={"scope": "openid email profile"},
+            client_id=config.client_id,  # RFC 6749 §2.2
+            client_secret=config.client_secret,  # RFC 6749 §2.3.1
+            server_metadata_url=config.oidc_config_url,  # OIDC Discovery §4
+            client_kwargs=client_kwargs,
             display_name=config.idp_display_name,
             allow_sign_up=config.allow_sign_up,
             auto_login=config.auto_login,
+            use_pkce=config.use_pkce,
+            groups_attribute_path=config.groups_attribute_path,
+            allowed_groups=config.allowed_groups,
+            role_attribute_path=config.role_attribute_path,
+            role_mapping=config.role_mapping,
+            role_attribute_strict=config.role_attribute_strict,
         )
+
         if config.auto_login:
             if self._auto_login_client:
                 raise ValueError("only one auto-login client is allowed")
             self._auto_login_client = client
         self._clients[config.idp_name] = client
 
-    def get_client(self, idp_name: str) -> OAuth2Client:
-        if (client := self._clients.get(idp_name)) is None:
-            raise ValueError(f"unknown or unregistered OAuth2 client: {idp_name}")
-        return client
+    def get_client(self, idp_name: str) -> Optional[OAuth2Client]:
+        return self._clients.get(idp_name)
 
     @classmethod
     def from_configs(cls, configs: Iterable[OAuth2ClientConfig]) -> "OAuth2Clients":

phoenix/server/prometheus.py

@@ -9,6 +9,7 @@ import psutil
 from prometheus_client import (
     Counter,
     Gauge,
+    Histogram,
     Summary,
     start_http_server,
 )
@@ -36,14 +37,19 @@ CPU_METRIC = Gauge(
     name="cpu_usage_percent",
     documentation="CPU usage percent",
 )
-BULK_LOADER_INSERTION_TIME = Summary(
-    name="bulk_loader_insertion_time_seconds_summary",
-    documentation="Summary of database insertion time (seconds)",
+BULK_LOADER_SPAN_INSERTION_TIME = Histogram(
+    namespace="phoenix",
+    name="bulk_loader_span_insertion_time_seconds",
+    documentation="Histogram of span database insertion time (seconds)",
+    buckets=[0.5, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0, 180.0],  # 500ms to 3min
 )
-BULK_LOADER_SPAN_INSERTIONS = Counter(
-    name="bulk_loader_span_insertions_total",
-    documentation="Total count of bulk loader span insertions",
+
+BULK_LOADER_SPAN_EXCEPTIONS = Counter(
+    namespace="phoenix",
+    name="bulk_loader_span_exceptions_total",
+    documentation="Total count of span insertion exceptions",
 )
+
 BULK_LOADER_EVALUATION_INSERTIONS = Counter(
     name="bulk_loader_evaluation_insertions_total",
     documentation="Total count of bulk loader evaluation insertions",
@@ -95,6 +101,37 @@ DB_DISK_USAGE_WARNING_EMAIL_ERRORS = Counter(
     documentation="Total count of database disk usage warning email send errors",
 )
 
+SPAN_QUEUE_REJECTIONS = Counter(
+    namespace="phoenix",
+    name="span_queue_rejections_total",
+    documentation="Total count of requests rejected due to span queue being full",
+)
+
+SPAN_QUEUE_SIZE = Gauge(
+    namespace="phoenix",
+    name="span_queue_size",
+    documentation="Current number of spans in the processing queue",
+)
+
+BULK_LOADER_LAST_ACTIVITY = Gauge(
+    namespace="phoenix",
+    name="bulk_loader_last_activity_timestamp_seconds",
+    documentation="Unix timestamp when bulk loader last processed items",
+)
+
+RETENTION_SWEEPER_LAST_RUN = Gauge(
+    namespace="phoenix",
+    name="retention_sweeper_last_run_seconds",
+    documentation="Unix timestamp (seconds since epoch) of the last retention sweeper run",
+)
+
+RETENTION_POLICY_EXECUTIONS = Counter(
+    namespace="phoenix",
+    name="retention_policy_executions_total",
+    documentation="Total number of retention policy executions",
+    labelnames=["status"],
+)
+
 
 class PrometheusMiddleware(BaseHTTPMiddleware):
     async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -> Response:

phoenix/server/rate_limiters.py

@@ -3,12 +3,7 @@ import time
 from collections import defaultdict
 from collections.abc import Callable, Coroutine
 from functools import partial
-from typing import (
-    Any,
-    Optional,
-    Pattern,  # import from re module when we drop support for 3.8
-    Union,
-)
+from typing import Any, Iterable
 
 from fastapi import HTTPException, Request
 
@@ -129,7 +124,7 @@ class ServerRateLimiter:
     def _fetch_token_bucket(self, key: str, request_time: float) -> TokenBucket:
         current_partition_index = self._current_partition_index(request_time)
         active_indices = self._active_partition_indices(current_partition_index)
-        bucket: Optional[TokenBucket] = None
+        bucket: TokenBucket | None = None
         for ii in active_indices:
             partition = self.cache_partitions[ii]
             if key in partition:
@@ -153,7 +148,7 @@ class ServerRateLimiter:
 
 
 def fastapi_ip_rate_limiter(
-    rate_limiter: ServerRateLimiter, paths: Optional[list[Union[str, Pattern[str]]]] = None
+    rate_limiter: ServerRateLimiter, paths: Iterable[str | re.Pattern[str]] | None = None
 ) -> Callable[[Request], Coroutine[Any, Any, Request]]:
     async def dependency(request: Request) -> Request:
         if paths is None or any(path_match(request.url.path, path) for path in paths):
@@ -182,7 +177,7 @@ def fastapi_route_rate_limiter(
     return dependency
 
 
-def path_match(path: str, match_pattern: Union[str, Pattern[str]]) -> bool:
+def path_match(path: str, match_pattern: str | re.Pattern[str]) -> bool:
     if isinstance(match_pattern, re.Pattern):
         return bool(match_pattern.match(path))
     return path == match_pattern

phoenix/server/retention.py

@@ -1,7 +1,9 @@
 from __future__ import annotations
 
+import logging
 from asyncio import create_task, gather, sleep
 from datetime import datetime, timedelta, timezone
+from time import time
 
 import sqlalchemy as sa
 from sqlalchemy.orm import selectinload
@@ -10,9 +12,15 @@ from phoenix.db.constants import DEFAULT_PROJECT_TRACE_RETENTION_POLICY_ID
 from phoenix.db.models import Project, ProjectTraceRetentionPolicy
 from phoenix.server.dml_event import SpanDeleteEvent
 from phoenix.server.dml_event_handler import DmlEventHandler
+from phoenix.server.prometheus import (
+    RETENTION_POLICY_EXECUTIONS,
+    RETENTION_SWEEPER_LAST_RUN,
+)
 from phoenix.server.types import DaemonTask, DbSessionFactory
 from phoenix.utilities import hour_of_week
 
+logger = logging.getLogger(__name__)
+
 
 class TraceDataSweeper(DaemonTask):
     def __init__(self, db: DbSessionFactory, dml_event_handler: DmlEventHandler):
@@ -24,15 +32,19 @@ class TraceDataSweeper(DaemonTask):
         """Check hourly and apply policies."""
         while self._running:
             await self._sleep_until_next_hour()
-            if not (policies := await self._get_policies()):
-                continue
-            current_hour = self._current_hour()
-            if tasks := [
-                create_task(self._apply(policy))
-                for policy in policies
-                if self._should_apply(policy, current_hour)
-            ]:
-                await gather(*tasks, return_exceptions=True)
+            RETENTION_SWEEPER_LAST_RUN.set(time())
+            try:
+                if not (policies := await self._get_policies()):
+                    continue
+                current_hour = self._current_hour()
+                if tasks := [
+                    create_task(self._apply(policy))
+                    for policy in policies
+                    if self._should_apply(policy, current_hour)
+                ]:
+                    await gather(*tasks, return_exceptions=True)
+            except Exception:
+                logger.exception("Unexpected error in retention sweeper main loop")
 
     async def _get_policies(self) -> list[ProjectTraceRetentionPolicy]:
         stmt = sa.select(ProjectTraceRetentionPolicy).options(
@@ -58,18 +70,19 @@ class TraceDataSweeper(DaemonTask):
         return True
 
     async def _apply(self, policy: ProjectTraceRetentionPolicy) -> None:
-        project_rowids = (
-            (
-                sa.select(Project.id)
-                .where(Project.trace_retention_policy_id.is_(None))
-                .scalar_subquery()
+        try:
+            project_rowids = (
+                (sa.select(Project.id).where(Project.trace_retention_policy_id.is_(None)))
+                if policy.id == DEFAULT_PROJECT_TRACE_RETENTION_POLICY_ID
+                else [p.id for p in policy.projects]
             )
-            if policy.id == DEFAULT_PROJECT_TRACE_RETENTION_POLICY_ID
-            else [p.id for p in policy.projects]
-        )
-        async with self._db() as session:
-            result = await policy.rule.delete_traces(session, project_rowids)
-        self._dml_event_handler.put(SpanDeleteEvent(tuple(result)))
+            async with self._db() as session:
+                result = await policy.rule.delete_traces(session, project_rowids)
+            self._dml_event_handler.put(SpanDeleteEvent(tuple(result)))
+            RETENTION_POLICY_EXECUTIONS.labels(status="success").inc()
+        except Exception:
+            logger.exception(f"Failed to apply retention policy '{policy.name}' (id={policy.id})")
+            RETENTION_POLICY_EXECUTIONS.labels(status="error").inc()
 
     async def _sleep_until_next_hour(self) -> None:
         next_hour = self._now().replace(minute=0, second=0, microsecond=0) + timedelta(hours=1)