arize-phoenix 3.25.0__py3-none-any.whl → 4.0.1__py3-none-any.whl

phoenix/server/api/types/Span.py

@@ -1,8 +1,7 @@
 import json
-from collections import defaultdict
 from datetime import datetime
 from enum import Enum
-from typing import Any, DefaultDict, Dict, Iterable, List, Mapping, Optional, Sized, cast
+from typing import Any, List, Mapping, Optional, Sized, cast
 
 import numpy as np
 import strawberry
@@ -11,13 +10,13 @@ from strawberry import ID, UNSET
 from strawberry.types import Info
 
 import phoenix.trace.schemas as trace_schema
-from phoenix.core.project import Project, WrappedSpan
-from phoenix.metrics.retrieval_metrics import RetrievalMetrics
+from phoenix.db import models
 from phoenix.server.api.context import Context
 from phoenix.server.api.types.DocumentRetrievalMetrics import DocumentRetrievalMetrics
 from phoenix.server.api.types.Evaluation import DocumentEvaluation, SpanEvaluation
 from phoenix.server.api.types.MimeType import MimeType
-from phoenix.trace.schemas import ComputedAttributes, SpanID
+from phoenix.server.api.types.node import Node
+from phoenix.trace.attributes import get_attribute_value
 
 EMBEDDING_EMBEDDINGS = SpanAttributes.EMBEDDING_EMBEDDINGS
 EMBEDDING_VECTOR = EmbeddingAttributes.EMBEDDING_VECTOR
@@ -40,18 +39,20 @@ class SpanKind(Enum):
     NB: this is actively under construction
     """
 
-    chain = trace_schema.SpanKind.CHAIN
-    tool = trace_schema.SpanKind.TOOL
-    llm = trace_schema.SpanKind.LLM
-    retriever = trace_schema.SpanKind.RETRIEVER
-    embedding = trace_schema.SpanKind.EMBEDDING
-    agent = trace_schema.SpanKind.AGENT
-    reranker = trace_schema.SpanKind.RERANKER
-    unknown = trace_schema.SpanKind.UNKNOWN
+    chain = "CHAIN"
+    tool = "TOOL"
+    llm = "LLM"
+    retriever = "RETRIEVER"
+    embedding = "EMBEDDING"
+    agent = "AGENT"
+    reranker = "RERANKER"
+    unknown = "UNKNOWN"
 
     @classmethod
     def _missing_(cls, v: Any) -> Optional["SpanKind"]:
-        return None if v else cls.unknown
+        if v and isinstance(v, str) and v.isascii() and not v.isupper():
+            return cls(v.upper())
+        return cls.unknown
 
 
 @strawberry.type
@@ -65,12 +66,18 @@ class SpanIOValue:
     mime_type: MimeType
     value: str
 
+    @strawberry.field(
+        description="Truncate value up to `chars` characters, appending '...' if truncated.",
+    )  # type: ignore
+    def truncated_value(self, chars: int = 100) -> str:
+        return f"{self.value[: max(0, chars - 3)]}..." if len(self.value) > chars else self.value
+
 
 @strawberry.enum
 class SpanStatusCode(Enum):
-    OK = trace_schema.SpanStatusCode.OK
-    ERROR = trace_schema.SpanStatusCode.ERROR
-    UNSET = trace_schema.SpanStatusCode.UNSET
+    OK = "OK"
+    ERROR = "ERROR"
+    UNSET = "UNSET"
 
     @classmethod
     def _missing_(cls, v: Any) -> Optional["SpanStatusCode"]:
@@ -84,19 +91,18 @@ class SpanEvent:
     timestamp: datetime
 
     @staticmethod
-    def from_event(
-        event: trace_schema.SpanEvent,
+    def from_dict(
+        event: Mapping[str, Any],
     ) -> "SpanEvent":
         return SpanEvent(
-            name=event.name,
-            message=cast(str, event.attributes.get(trace_schema.EXCEPTION_MESSAGE) or ""),
-            timestamp=event.timestamp,
+            name=event["name"],
+            message=cast(str, event["attributes"].get(trace_schema.EXCEPTION_MESSAGE) or ""),
+            timestamp=datetime.fromisoformat(event["timestamp"]),
         )
 
 
 @strawberry.type
-class Span:
-    project: strawberry.Private[Project]
+class Span(Node):
     name: str
     status_code: SpanStatusCode
     status_message: str
@@ -143,12 +149,8 @@ class Span:
         "an LLM, an evaluation may assess the helpfulness of its response with "
         "respect to its input."
     )  # type: ignore
-    def span_evaluations(self) -> List[SpanEvaluation]:
-        span_id = SpanID(str(self.context.span_id))
-        return [
-            SpanEvaluation.from_pb_evaluation(evaluation)
-            for evaluation in self.project.get_evaluations_by_span_id(span_id)
-        ]
+    async def span_evaluations(self, info: Info[Context, None]) -> List[SpanEvaluation]:
+        return await info.context.data_loaders.span_evaluations.load(self.id_attr)
 
     @strawberry.field(
         description="Evaluations of the documents associated with the span, e.g. "
@@ -158,68 +160,43 @@ class Span:
         "a list, and each evaluation is identified by its document's (zero-based) "
         "index in that list."
     )  # type: ignore
-    def document_evaluations(self) -> List[DocumentEvaluation]:
-        span_id = SpanID(str(self.context.span_id))
-        return [
-            DocumentEvaluation.from_pb_evaluation(evaluation)
-            for evaluation in self.project.get_document_evaluations_by_span_id(span_id)
-        ]
+    async def document_evaluations(self, info: Info[Context, None]) -> List[DocumentEvaluation]:
+        return await info.context.data_loaders.document_evaluations.load(self.id_attr)
 
     @strawberry.field(
         description="Retrieval metrics: NDCG@K, Precision@K, Reciprocal Rank, etc.",
     )  # type: ignore
-    def document_retrieval_metrics(
+    async def document_retrieval_metrics(
         self,
+        info: Info[Context, None],
         evaluation_name: Optional[str] = UNSET,
     ) -> List[DocumentRetrievalMetrics]:
         if not self.num_documents:
             return []
-        span_id = SpanID(str(self.context.span_id))
-        all_document_evaluation_names = self.project.get_document_evaluation_names(span_id)
-        if not all_document_evaluation_names:
-            return []
-        if evaluation_name is UNSET:
-            evaluation_names = all_document_evaluation_names
-        elif evaluation_name not in all_document_evaluation_names:
-            return []
-        else:
-            evaluation_names = [evaluation_name]
-        retrieval_metrics = []
-        for name in evaluation_names:
-            evaluation_scores = self.project.get_document_evaluation_scores(
-                span_id=span_id,
-                evaluation_name=name,
-                num_documents=self.num_documents,
-            )
-            retrieval_metrics.append(
-                DocumentRetrievalMetrics(
-                    evaluation_name=name,
-                    metrics=RetrievalMetrics(evaluation_scores),
-                )
-            )
-        return retrieval_metrics
+        return await info.context.data_loaders.document_retrieval_metrics.load(
+            (self.id_attr, evaluation_name or None, self.num_documents),
+        )
 
     @strawberry.field(
         description="All descendant spans (children, grandchildren, etc.)",
     )  # type: ignore
-    def descendants(
+    async def descendants(
         self,
         info: Info[Context, None],
     ) -> List["Span"]:
-        return [
-            to_gql_span(span, self.project)
-            for span in self.project.get_descendant_spans(SpanID(self.context.span_id))
-        ]
+        span_id = str(self.context.span_id)
+        spans = await info.context.data_loaders.span_descendants.load(span_id)
+        return [to_gql_span(span) for span in spans]
 
 
-def to_gql_span(span: WrappedSpan, project: Project) -> "Span":
-    events: List[SpanEvent] = list(map(SpanEvent.from_event, span.events))
-    input_value = cast(Optional[str], span.attributes.get(INPUT_VALUE))
-    output_value = cast(Optional[str], span.attributes.get(OUTPUT_VALUE))
-    retrieval_documents = span.attributes.get(RETRIEVAL_DOCUMENTS)
+def to_gql_span(span: models.Span) -> Span:
+    events: List[SpanEvent] = list(map(SpanEvent.from_dict, span.events))
+    input_value = cast(Optional[str], get_attribute_value(span.attributes, INPUT_VALUE))
+    output_value = cast(Optional[str], get_attribute_value(span.attributes, OUTPUT_VALUE))
+    retrieval_documents = get_attribute_value(span.attributes, RETRIEVAL_DOCUMENTS)
     num_documents = len(retrieval_documents) if isinstance(retrieval_documents, Sized) else None
     return Span(
-        project=project,
+        id_attr=span.id,
         name=span.name,
         status_code=SpanStatusCode(span.status_code),
         status_message=span.status_message,
@@ -227,50 +204,39 @@ def to_gql_span(span: WrappedSpan, project: Project) -> "Span":
         span_kind=SpanKind(span.span_kind),
         start_time=span.start_time,
         end_time=span.end_time,
-        latency_ms=cast(Optional[float], span[ComputedAttributes.LATENCY_MS]),
+        latency_ms=span.latency_ms,
         context=SpanContext(
-            trace_id=cast(ID, span.context.trace_id),
-            span_id=cast(ID, span.context.span_id),
-        ),
-        attributes=json.dumps(
-            _nested_attributes(_hide_embedding_vectors(span.attributes)),
-            cls=_JSONEncoder,
+            trace_id=cast(ID, span.trace.trace_id),
+            span_id=cast(ID, span.span_id),
         ),
-        metadata=_convert_metadata_to_string(span.attributes.get(METADATA)),
+        attributes=json.dumps(_hide_embedding_vectors(span.attributes), cls=_JSONEncoder),
+        metadata=_convert_metadata_to_string(get_attribute_value(span.attributes, METADATA)),
         num_documents=num_documents,
         token_count_total=cast(
             Optional[int],
-            span.attributes.get(LLM_TOKEN_COUNT_TOTAL),
+            get_attribute_value(span.attributes, LLM_TOKEN_COUNT_TOTAL),
         ),
         token_count_prompt=cast(
             Optional[int],
-            span.attributes.get(LLM_TOKEN_COUNT_PROMPT),
+            get_attribute_value(span.attributes, LLM_TOKEN_COUNT_PROMPT),
         ),
         token_count_completion=cast(
             Optional[int],
-            span.attributes.get(LLM_TOKEN_COUNT_COMPLETION),
-        ),
-        cumulative_token_count_total=cast(
-            Optional[int],
-            span[ComputedAttributes.CUMULATIVE_LLM_TOKEN_COUNT_TOTAL],
-        ),
-        cumulative_token_count_prompt=cast(
-            Optional[int],
-            span[ComputedAttributes.CUMULATIVE_LLM_TOKEN_COUNT_PROMPT],
-        ),
-        cumulative_token_count_completion=cast(
-            Optional[int],
-            span[ComputedAttributes.CUMULATIVE_LLM_TOKEN_COUNT_COMPLETION],
+            get_attribute_value(span.attributes, LLM_TOKEN_COUNT_COMPLETION),
         ),
+        cumulative_token_count_total=span.cumulative_llm_token_count_prompt
+        + span.cumulative_llm_token_count_completion,
+        cumulative_token_count_prompt=span.cumulative_llm_token_count_prompt,
+        cumulative_token_count_completion=span.cumulative_llm_token_count_completion,
         propagated_status_code=(
             SpanStatusCode.ERROR
-            if span[ComputedAttributes.CUMULATIVE_ERROR_COUNT]
+            if span.cumulative_error_count
             else SpanStatusCode(span.status_code)
         ),
         events=events,
         input=(
             SpanIOValue(
-                mime_type=MimeType(span.attributes.get(INPUT_MIME_TYPE)),
+                mime_type=MimeType(get_attribute_value(span.attributes, INPUT_MIME_TYPE)),
                 value=input_value,
             )
             if input_value is not None
@@ -278,7 +244,7 @@ def to_gql_span(span: WrappedSpan, project: Project) -> "Span":
         ),
         output=(
             SpanIOValue(
-                mime_type=MimeType(span.attributes.get(OUTPUT_MIME_TYPE)),
+                mime_type=MimeType(get_attribute_value(span.attributes, OUTPUT_MIME_TYPE)),
                 value=output_value,
             )
             if output_value is not None
@@ -287,6 +253,29 @@ def to_gql_span(span: WrappedSpan, project: Project) -> "Span":
     )
 
 
+def _hide_embedding_vectors(attributes: Mapping[str, Any]) -> Mapping[str, Any]:
+    if not (
+        isinstance(em := attributes.get("embedding"), dict)
+        and isinstance(embeddings := em.get("embeddings"), list)
+        and embeddings
+    ):
+        return attributes
+    embeddings = embeddings.copy()
+    for i, embedding in enumerate(embeddings):
+        if not (
+            isinstance(embedding, dict)
+            and isinstance(emb := embedding.get("embedding"), dict)
+            and isinstance(vector := emb.get("vector"), list)
+            and vector
+        ):
+            continue
+        embeddings[i] = {
+            **embedding,
+            "embedding": {**emb, "vector": f"<{len(vector)} dimensional vector>"},
+        }
+    return {**attributes, "embedding": {**em, "embeddings": embeddings}}
+
+
 class _JSONEncoder(json.JSONEncoder):
     def default(self, obj: Any) -> Any:
         if isinstance(obj, datetime):
@@ -302,39 +291,6 @@ class _JSONEncoder(json.JSONEncoder):
         return super().default(obj)
 
 
-def _trie() -> DefaultDict[str, Any]:
-    return defaultdict(_trie)
-
-
-def _nested_attributes(
-    attributes: Mapping[str, Any],
-) -> DefaultDict[str, Any]:
-    nested_attributes = _trie()
-    for attribute_name, attribute_value in attributes.items():
-        trie = nested_attributes
-        keys = attribute_name.split(".")
-        for key in keys[:-1]:
-            trie = trie[key]
-        trie[keys[-1]] = attribute_value
-    return nested_attributes
-
-
-def _hide_embedding_vectors(
-    attributes: Mapping[str, Any],
-) -> Dict[str, Any]:
-    _attributes = dict(attributes)
-    if not isinstance((embeddings := _attributes.get(EMBEDDING_EMBEDDINGS)), Iterable):
-        return _attributes
-    _embeddings = []
-    for embedding in embeddings:
-        _embedding = dict(embedding)
-        if isinstance((vector := _embedding.get(EMBEDDING_VECTOR)), Sized):
-            _embedding[EMBEDDING_VECTOR] = f"<{len(vector)} dimensional vector>"
-        _embeddings.append(_embedding)
-    _attributes[EMBEDDING_EMBEDDINGS] = _embeddings
-    return _attributes
-
-
 def _convert_metadata_to_string(metadata: Any) -> Optional[str]:
     """
     Converts metadata to a string representation.

phoenix/server/api/types/Trace.py

@@ -1,47 +1,56 @@
 from typing import List, Optional
 
 import strawberry
-from strawberry import ID, UNSET, Private
+from sqlalchemy import desc, select
+from sqlalchemy.orm import contains_eager
+from strawberry import UNSET
+from strawberry.types import Info
 
-from phoenix.core.project import Project
+from phoenix.db import models
+from phoenix.server.api.context import Context
 from phoenix.server.api.types.Evaluation import TraceEvaluation
+from phoenix.server.api.types.node import Node
 from phoenix.server.api.types.pagination import (
     Connection,
     ConnectionArgs,
-    Cursor,
+    CursorString,
     connection_from_list,
 )
 from phoenix.server.api.types.Span import Span, to_gql_span
-from phoenix.trace.schemas import TraceID
 
 
 @strawberry.type
-class Trace:
-    trace_id: ID
-    project: Private[Project]
-
+class Trace(Node):
     @strawberry.field
-    def spans(
+    async def spans(
         self,
+        info: Info[Context, None],
         first: Optional[int] = 50,
         last: Optional[int] = UNSET,
-        after: Optional[Cursor] = UNSET,
-        before: Optional[Cursor] = UNSET,
+        after: Optional[CursorString] = UNSET,
+        before: Optional[CursorString] = UNSET,
     ) -> Connection[Span]:
         args = ConnectionArgs(
             first=first,
-            after=after if isinstance(after, Cursor) else None,
+            after=after if isinstance(after, CursorString) else None,
             last=last,
-            before=before if isinstance(before, Cursor) else None,
+            before=before if isinstance(before, CursorString) else None,
         )
-        spans = sorted(
-            self.project.get_trace(TraceID(self.trace_id)),
-            key=lambda span: span.start_time,
+        stmt = (
+            select(models.Span)
+            .join(models.Trace)
+            .where(models.Trace.id == self.id_attr)
+            .options(contains_eager(models.Span.trace))
+            # Sort descending because the root span tends to show up later
+            # in the ingestion process.
+            .order_by(desc(models.Span.id))
+            .limit(first)
         )
-        data = [to_gql_span(span, self.project) for span in spans]
+        async with info.context.db() as session:
+            spans = await session.stream_scalars(stmt)
+            data = [to_gql_span(span) async for span in spans]
         return connection_from_list(data=data, args=args)
 
     @strawberry.field(description="Evaluations associated with the trace")  # type: ignore
-    def trace_evaluations(self) -> List[TraceEvaluation]:
-        evaluations = self.project.get_evaluations_by_trace_id(TraceID(self.trace_id))
-        return [TraceEvaluation.from_pb_evaluation(evaluation) for evaluation in evaluations]
+    async def trace_evaluations(self, info: Info[Context, None]) -> List[TraceEvaluation]:
+        return await info.context.data_loaders.trace_evaluations.load(self.id_attr)

phoenix/server/api/types/pagination.py

@@ -1,11 +1,16 @@
 import base64
 from dataclasses import dataclass
-from typing import Generic, List, Optional, TypeVar
+from datetime import datetime
+from enum import Enum, auto
+from typing import ClassVar, Generic, List, Optional, Tuple, TypeVar, Union
 
 import strawberry
 from strawberry import UNSET
+from typing_extensions import TypeAlias, assert_never
 
+ID: TypeAlias = int
 GenericType = TypeVar("GenericType")
+CursorSortColumnValue: TypeAlias = Union[str, int, float, datetime]
 
 
 @strawberry.type
@@ -35,11 +40,10 @@ class PageInfo:
     has_previous_page: bool
     start_cursor: Optional[str]
     end_cursor: Optional[str]
-    total_count: int
 
 
 # A type alias for the connection cursor implementation
-Cursor = str
+CursorString = str
 
 
 @strawberry.type
@@ -56,14 +60,132 @@ class Edge(Generic[GenericType]):
 CURSOR_PREFIX = "connection:"
 
 
-def offset_to_cursor(offset: int) -> Cursor:
+class CursorSortColumnDataType(Enum):
+    STRING = auto()
+    INT = auto()
+    FLOAT = auto()
+    DATETIME = auto()
+
+
+@dataclass
+class CursorSortColumn:
+    type: CursorSortColumnDataType
+    value: CursorSortColumnValue
+
+    def __str__(self) -> str:
+        if isinstance(self.value, str):
+            return self.value
+        if isinstance(self.value, (int, float)):
+            return str(self.value)
+        if isinstance(self.value, datetime):
+            return self.value.isoformat()
+        assert_never(self.type)
+
+    @classmethod
+    def from_string(cls, type: CursorSortColumnDataType, cursor_string: str) -> "CursorSortColumn":
+        value: CursorSortColumnValue
+        if type is CursorSortColumnDataType.STRING:
+            value = cursor_string
+        elif type is CursorSortColumnDataType.INT:
+            value = int(cursor_string)
+        elif type is CursorSortColumnDataType.FLOAT:
+            value = float(cursor_string)
+        elif type is CursorSortColumnDataType.DATETIME:
+            value = datetime.fromisoformat(cursor_string)
+        else:
+            assert_never(type)
+        return cls(type=type, value=value)
+
+
+@dataclass
+class Cursor:
+    """
+    Serializes and deserializes cursor strings for ID-based pagination.
+
+    In the simplest case, a cursor encodes the rowid of a record. In the case
+    that a sort has been applied, the cursor additionally encodes the data type
+    and value of the column indexed for sorting so that the sort position can be
+    efficiently found. The encoding ensures that the cursor string is opaque to
+    the client and discourages the client from making use of the encoded
+    content.
+
+    Examples:
+        # encodes "10"
+        Cursor(rowid=10)
+
+        # encodes "11:STRING:abc"
+        Cursor(
+            rowid=11,
+            sort_column=CursorSortColumn(
+                type=CursorSortColumnDataType.STRING,
+                value="abc"
+            )
+        )
+
+        # encodes "10:INT:5"
+        Cursor(
+            rowid=10,
+            sort_column=CursorSortColumn(
+                type=CursorSortColumnDataType.INT,
+                value=5
+            )
+        )
+
+        # encodes "17:FLOAT:5.7"
+        Cursor(
+            rowid=17,
+            sort_column=CursorSortColumn(
+                type=CursorSortColumnDataType.FLOAT,
+                value=5.7
+            )
+        )
+
+        # encodes "20:DATETIME:2024-05-05T04:25:29.911245+00:00"
+        Cursor(
+            rowid=20,
+            sort_column=CursorSortColumn(
+                type=CursorSortColumnDataType.DATETIME,
+                value=datetime.fromisoformat("2024-05-05T04:25:29.911245+00:00")
+            )
+        )
+    """
+
+    rowid: int
+    sort_column: Optional[CursorSortColumn] = None
+
+    _DELIMITER: ClassVar[str] = ":"
+
+    def __str__(self) -> str:
+        cursor_parts = [str(self.rowid)]
+        if (sort_column := self.sort_column) is not None:
+            cursor_parts.extend([sort_column.type.name, str(sort_column)])
+        return base64.b64encode(self._DELIMITER.join(cursor_parts).encode()).decode()
+
+    @classmethod
+    def from_string(cls, cursor: str) -> "Cursor":
+        decoded = base64.b64decode(cursor).decode()
+        rowid_string = decoded
+        sort_column = None
+        if (first_delimiter_index := decoded.find(cls._DELIMITER)) > -1:
+            rowid_string = decoded[:first_delimiter_index]
+            second_delimiter_index = decoded.index(cls._DELIMITER, first_delimiter_index + 1)
+            sort_column = CursorSortColumn.from_string(
+                type=CursorSortColumnDataType[
+                    decoded[first_delimiter_index + 1 : second_delimiter_index]
+                ],
+                cursor_string=decoded[second_delimiter_index + 1 :],
+            )
+        return cls(rowid=int(rowid_string), sort_column=sort_column)
+
+
+def offset_to_cursor(offset: int) -> CursorString:
     """
     Creates the cursor string from an offset.
     """
     return base64.b64encode(f"{CURSOR_PREFIX}{offset}".encode("utf-8")).decode()
 
 
-def cursor_to_offset(cursor: Cursor) -> int:
+def cursor_to_offset(cursor: CursorString) -> int:
     """
     Extracts the offset from the cursor string.
     """
@@ -71,13 +193,13 @@ def cursor_to_offset(cursor: Cursor) -> int:
     return int(offset)
 
 
-def get_offset_with_default(cursor: Optional[Cursor], default_offset: int) -> int:
+def get_offset_with_default(cursor: Optional[CursorString], default_offset: int) -> int:
     """
     Given an optional cursor and a default offset, returns the offset
     to use; if the cursor contains a valid offset, that will be used,
     otherwise it will be the default.
     """
-    if not isinstance(cursor, Cursor):
+    if not isinstance(cursor, CursorString):
         return default_offset
     offset = cursor_to_offset(cursor)
     return offset if isinstance(offset, int) else default_offset
@@ -90,9 +212,9 @@ class ConnectionArgs:
     """
 
     first: Optional[int] = UNSET
-    after: Optional[Cursor] = UNSET
+    after: Optional[CursorString] = UNSET
     last: Optional[int] = UNSET
-    before: Optional[Cursor] = UNSET
+    before: Optional[CursorString] = UNSET
 
 
 def connection_from_list(
@@ -169,6 +291,25 @@ def connection_from_list_slice(
             end_cursor=last_edge.cursor if last_edge else None,
             has_previous_page=start_offset > lower_bound if isinstance(args.last, int) else False,
             has_next_page=end_offset < upper_bound if isinstance(args.first, int) else False,
-            total_count=list_length,
+        ),
+    )
+
+
+def connections(
+    data: List[Tuple[Cursor, GenericType]],
+    has_previous_page: bool,
+    has_next_page: bool,
+) -> Connection[GenericType]:
+    edges = [Edge(node=node, cursor=str(cursor)) for cursor, node in data]
+    has_edges = len(edges) > 0
+    first_edge = edges[0] if has_edges else None
+    last_edge = edges[-1] if has_edges else None
+    return Connection(
+        edges=edges,
+        page_info=PageInfo(
+            start_cursor=first_edge.cursor if first_edge else None,
+            end_cursor=last_edge.cursor if last_edge else None,
+            has_previous_page=has_previous_page,
+            has_next_page=has_next_page,
         ),
     )