arize-phoenix 11.37.0__py3-none-any.whl → 12.0.0__py3-none-any.whl

phoenix/server/api/routers/v1/sessions.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+import warnings
+from typing import Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from pydantic import Field
+from sqlalchemy import select
+from starlette.requests import Request
+from starlette.status import HTTP_404_NOT_FOUND
+
+from phoenix.db import models
+from phoenix.db.helpers import SupportedSQLDialect
+from phoenix.db.insertion.helpers import as_kv, insert_on_conflict
+from phoenix.server.api.routers.v1.models import V1RoutesBaseModel
+from phoenix.server.authorization import is_not_locked
+from phoenix.server.bearer_auth import PhoenixUser
+
+from .annotations import SessionAnnotationData
+from .utils import RequestBody, ResponseBody, add_errors_to_responses
+
+router = APIRouter(tags=["sessions"])
+
+
+class InsertedSessionAnnotation(V1RoutesBaseModel):
+    id: str = Field(description="The ID of the inserted session annotation")
+
+
+class AnnotateSessionsRequestBody(RequestBody[list[SessionAnnotationData]]):
+    pass
+
+
+class AnnotateSessionsResponseBody(ResponseBody[list[InsertedSessionAnnotation]]):
+    pass
+
+
+@router.post(
+    "/session_annotations",
+    dependencies=[Depends(is_not_locked)],
+    operation_id="annotateSessions",
+    summary="Create session annotations",
+    responses=add_errors_to_responses(
+        [{"status_code": HTTP_404_NOT_FOUND, "description": "Session not found"}]
+    ),
+    response_description="Session annotations inserted successfully",
+    include_in_schema=True,
+)
+async def annotate_sessions(
+    request: Request,
+    request_body: AnnotateSessionsRequestBody,
+    sync: bool = Query(default=False, description="If true, fulfill request synchronously."),
+) -> AnnotateSessionsResponseBody:
+    if not request_body.data:
+        return AnnotateSessionsResponseBody(data=[])
+
+    user_id: Optional[int] = None
+    if request.app.state.authentication_enabled and isinstance(request.user, PhoenixUser):
+        user_id = int(request.user.identity)
+
+    session_annotations = request_body.data
+    filtered_session_annotations = list(filter(lambda d: d.name != "note", session_annotations))
+    if len(filtered_session_annotations) != len(session_annotations):
+        warnings.warn(
+            (
+                "Session annotations with the name 'note' are not supported in this endpoint. "
+                "They will be ignored."
+            ),
+            UserWarning,
+        )
+    precursors = [d.as_precursor(user_id=user_id) for d in filtered_session_annotations]
+    if not sync:
+        await request.state.enqueue_annotations(*precursors)
+        return AnnotateSessionsResponseBody(data=[])
+
+    session_ids = {p.session_id for p in precursors}
+    async with request.app.state.db() as session:
+        existing_sessions = {
+            session_id: rowid
+            async for session_id, rowid in await session.stream(
+                select(models.ProjectSession.session_id, models.ProjectSession.id).filter(
+                    models.ProjectSession.session_id.in_(session_ids)
+                )
+            )
+        }
+
+    missing_session_ids = session_ids - set(existing_sessions.keys())
+    # We prefer to fail the entire operation if there are missing sessions in sync mode
+    if missing_session_ids:
+        raise HTTPException(
+            detail=f"Sessions with IDs {', '.join(missing_session_ids)} do not exist.",
+            status_code=HTTP_404_NOT_FOUND,
+        )
+
+    async with request.app.state.db() as session:
+        inserted_ids = []
+        dialect = SupportedSQLDialect(session.bind.dialect.name)
+        for p in precursors:
+            values = dict(as_kv(p.as_insertable(existing_sessions[p.session_id]).row))
+            session_annotation_id = await session.scalar(
+                insert_on_conflict(
+                    values,
+                    dialect=dialect,
+                    table=models.ProjectSessionAnnotation,
+                    unique_by=("name", "project_session_id", "identifier"),
+                ).returning(models.ProjectSessionAnnotation.id)
+            )
+            inserted_ids.append(session_annotation_id)
+
+    return AnnotateSessionsResponseBody(
+        data=[InsertedSessionAnnotation(id=str(inserted_id)) for inserted_id in inserted_ids]
+    )

phoenix/server/api/routers/v1/traces.py

@@ -144,12 +144,11 @@ class AnnotateTracesResponseBody(ResponseBody[list[InsertedTraceAnnotation]]):
     responses=add_errors_to_responses(
         [{"status_code": HTTP_404_NOT_FOUND, "description": "Trace not found"}]
     ),
-    include_in_schema=False,
 )
 async def annotate_traces(
     request: Request,
     request_body: AnnotateTracesRequestBody,
-    sync: bool = Query(default=True, description="If true, fulfill request synchronously."),
+    sync: bool = Query(default=False, description="If true, fulfill request synchronously."),
 ) -> AnnotateTracesResponseBody:
     if not request_body.data:
         return AnnotateTracesResponseBody(data=[])

phoenix/server/api/routers/v1/users.py

@@ -217,6 +217,13 @@ async def create_user(
             detail="Cannot create users with SYSTEM role",
         )
 
+    # TODO: Implement VIEWER role
+    if role == "VIEWER":
+        raise HTTPException(
+            status_code=HTTP_400_BAD_REQUEST,
+            detail="VIEWER role not yet implemented",
+        )
+
     user: models.User
     if isinstance(user_data, LocalUserData):
         password = (user_data.password or secrets.token_hex()).strip()

phoenix/server/api/subscriptions.py

@@ -26,6 +26,7 @@ from typing_extensions import TypeAlias, assert_never
 from phoenix.config import PLAYGROUND_PROJECT_NAME
 from phoenix.datetime_utils import local_now, normalize_datetime
 from phoenix.db import models
+from phoenix.db.helpers import insert_experiment_with_examples_snapshot
 from phoenix.server.api.auth import IsLocked, IsNotReadOnly
 from phoenix.server.api.context import Context
 from phoenix.server.api.exceptions import BadRequest, CustomGraphQLError, NotFound
@@ -43,6 +44,7 @@ from phoenix.server.api.helpers.playground_spans import (
     get_db_trace,
     streaming_llm_span,
 )
+from phoenix.server.api.helpers.playground_users import get_user
 from phoenix.server.api.helpers.prompts.models import PromptTemplateFormat
 from phoenix.server.api.input_types.ChatCompletionInput import (
     ChatCompletionInput,
@@ -302,18 +304,19 @@ class Subscription:
                         description="Traces from prompt playground",
                     )
                 )
+            user_id = get_user(info)
             experiment = models.Experiment(
                 dataset_id=from_global_id_with_expected_type(input.dataset_id, Dataset.__name__),
                 dataset_version_id=resolved_version_id,
                 name=input.experiment_name
                 or _default_playground_experiment_name(input.prompt_name),
                 description=input.experiment_description,
-                repetitions=1,
+                repetitions=input.repetitions,
                 metadata_=input.experiment_metadata or dict(),
                 project_name=project_name,
+                user_id=user_id,
             )
-            session.add(experiment)
-            await session.flush()
+            await insert_experiment_with_examples_snapshot(session, experiment)
         yield ChatCompletionSubscriptionExperiment(
             experiment=to_gql_experiment(experiment)
         )  # eagerly yields experiment so it can be linked by consumers of the subscription
@@ -327,11 +330,13 @@ class Subscription:
                     llm_client=llm_client,
                     revision=revision,
                     results=results,
+                    repetition_number=repetition_number,
                     experiment_id=experiment.id,
                     project_id=playground_project_id,
                 ),
             )
             for revision in revisions
+            for repetition_number in range(1, input.repetitions + 1)
         ]
         in_progress: list[
             tuple[
@@ -409,6 +414,7 @@ async def _stream_chat_completion_over_dataset_example(
     input: ChatCompletionOverDatasetInput,
     llm_client: PlaygroundStreamingClient,
     revision: models.DatasetExampleRevision,
+    repetition_number: int,
     results: asyncio.Queue[ChatCompletionResult],
     experiment_id: int,
     project_id: int,
@@ -435,7 +441,11 @@ async def _stream_chat_completion_over_dataset_example(
         )
     except TemplateFormatterError as error:
         format_end_time = cast(datetime, normalize_datetime(dt=local_now(), tz=timezone.utc))
-        yield ChatCompletionSubscriptionError(message=str(error), dataset_example_id=example_id)
+        yield ChatCompletionSubscriptionError(
+            message=str(error),
+            dataset_example_id=example_id,
+            repetition_number=repetition_number,
+        )
         await results.put(
             (
                 example_id,
@@ -445,7 +455,7 @@ async def _stream_chat_completion_over_dataset_example(
                     dataset_example_id=revision.dataset_example_id,
                     trace_id=None,
                     output={},
-                    repetition_number=1,
+                    repetition_number=repetition_number,
                     start_time=format_start_time,
                     end_time=format_end_time,
                     error=str(error),
@@ -465,17 +475,24 @@ async def _stream_chat_completion_over_dataset_example(
         ):
             span.add_response_chunk(chunk)
             chunk.dataset_example_id = example_id
+            chunk.repetition_number = repetition_number
             yield chunk
         span.set_attributes(llm_client.attributes)
     db_trace = get_db_trace(span, project_id)
     db_span = get_db_span(span, db_trace)
     db_run = get_db_experiment_run(
-        db_span, db_trace, experiment_id=experiment_id, example_id=revision.dataset_example_id
+        db_span,
+        db_trace,
+        experiment_id=experiment_id,
+        example_id=revision.dataset_example_id,
+        repetition_number=repetition_number,
     )
     await results.put((example_id, db_span, db_run))
     if span.status_message is not None:
         yield ChatCompletionSubscriptionError(
-            message=span.status_message, dataset_example_id=example_id
+            message=span.status_message,
+            dataset_example_id=example_id,
+            repetition_number=repetition_number,
         )
 
 
@@ -511,6 +528,7 @@ async def _chat_completion_result_payloads(
             span=Span(span_rowid=span.id, db_span=span) if span else None,
             experiment_run=to_gql_experiment_run(run),
             dataset_example_id=example_id,
+            repetition_number=run.repetition_number,
         )
 
 

phoenix/server/api/types/ChatCompletionSubscriptionPayload.py

@@ -11,6 +11,7 @@ from .Span import Span
 @strawberry.interface
 class ChatCompletionSubscriptionPayload:
     dataset_example_id: Optional[GlobalID] = None
+    repetition_number: Optional[int] = None
 
 
 @strawberry.type

phoenix/server/api/types/DatasetExample.py

@@ -12,6 +12,7 @@ from phoenix.db import models
 from phoenix.server.api.context import Context
 from phoenix.server.api.exceptions import BadRequest
 from phoenix.server.api.types.DatasetExampleRevision import DatasetExampleRevision
+from phoenix.server.api.types.DatasetSplit import DatasetSplit, to_gql_dataset_split
 from phoenix.server.api.types.DatasetVersion import DatasetVersion
 from phoenix.server.api.types.ExperimentRepeatedRunGroup import (
     ExperimentRepeatedRunGroup,
@@ -131,3 +132,13 @@ class DatasetExample(Node):
             )
             for group in repeated_run_groups
         ]
+
+    @strawberry.field
+    async def dataset_splits(
+        self,
+        info: Info[Context, None],
+    ) -> list[DatasetSplit]:
+        return [
+            to_gql_dataset_split(split)
+            for split in await info.context.data_loaders.dataset_example_splits.load(self.id_attr)
+        ]

phoenix/server/api/types/DatasetSplit.py

@@ -0,0 +1,32 @@
+from datetime import datetime
+from typing import ClassVar, Optional
+
+import strawberry
+from strawberry.relay import Node, NodeID
+from strawberry.scalars import JSON
+
+from phoenix.db import models
+
+
+@strawberry.type
+class DatasetSplit(Node):
+    _table: ClassVar[type[models.Base]] = models.DatasetSplit
+    id_attr: NodeID[int]
+    name: str
+    description: Optional[str]
+    metadata: JSON
+    color: str
+    created_at: datetime
+    updated_at: datetime
+
+
+def to_gql_dataset_split(dataset_split: models.DatasetSplit) -> DatasetSplit:
+    return DatasetSplit(
+        id_attr=dataset_split.id,
+        name=dataset_split.name,
+        description=dataset_split.description,
+        color=dataset_split.color or "#ffffff",
+        metadata=dataset_split.metadata_,
+        created_at=dataset_split.created_at,
+        updated_at=dataset_split.updated_at,
+    )

phoenix/server/api/types/Experiment.py

@@ -193,10 +193,6 @@ class Experiment(Node):
                 async for token_type, is_prompt, cost, tokens in data
             ]
 
-    @strawberry.field
-    async def repetition_count(self, info: Info[Context, None]) -> int:
-        return await info.context.data_loaders.experiment_repetition_counts.load(self.id_attr)
-
 
 def to_gql_experiment(
     experiment: models.Experiment,

phoenix/server/api/types/Project.py

@@ -588,6 +588,22 @@ class Project(Node):
         async with info.context.db() as session:
             return list(await session.scalars(stmt))
 
+    @strawberry.field(
+        description="Names of all available annotations for sessions. "
+        "(The list contains no duplicates.)"
+    )  # type: ignore
+    async def session_annotation_names(
+        self,
+        info: Info[Context, None],
+    ) -> list[str]:
+        stmt = (
+            select(distinct(models.ProjectSessionAnnotation.name))
+            .join(models.ProjectSession)
+            .where(models.ProjectSession.project_id == self.project_rowid)
+        )
+        async with info.context.db() as session:
+            return list(await session.scalars(stmt))
+
     @strawberry.field(
         description="Names of available document evaluations.",
     )  # type: ignore

phoenix/server/api/types/ProjectSession.py

@@ -1,14 +1,19 @@
+from collections import defaultdict
+from dataclasses import asdict, dataclass
 from datetime import datetime
 from typing import TYPE_CHECKING, Annotated, ClassVar, Optional, Type
 
+import pandas as pd
 import strawberry
 from openinference.semconv.trace import SpanAttributes
 from sqlalchemy import select
 from strawberry import UNSET, Info, Private, lazy
-from strawberry.relay import Connection, GlobalID, Node, NodeID
+from strawberry.relay import Connection, Node, NodeID
 
 from phoenix.db import models
 from phoenix.server.api.context import Context
+from phoenix.server.api.input_types.AnnotationFilter import AnnotationFilter, satisfies_filter
+from phoenix.server.api.types.AnnotationSummary import AnnotationSummary
 from phoenix.server.api.types.CostBreakdown import CostBreakdown
 from phoenix.server.api.types.MimeType import MimeType
 from phoenix.server.api.types.pagination import ConnectionArgs, CursorString, connection_from_list
@@ -18,6 +23,8 @@ from phoenix.server.api.types.SpanIOValue import SpanIOValue
 from phoenix.server.api.types.TokenUsage import TokenUsage
 
 if TYPE_CHECKING:
+    from phoenix.server.api.types.Project import Project
+    from phoenix.server.api.types.ProjectSessionAnnotation import ProjectSessionAnnotation
     from phoenix.server.api.types.Trace import Trace
 
 
@@ -31,10 +38,13 @@ class ProjectSession(Node):
     end_time: datetime
 
     @strawberry.field
-    async def project_id(self) -> GlobalID:
+    async def project(
+        self,
+        info: Info[Context, None],
+    ) -> Annotated["Project", lazy(".Project")]:
         from phoenix.server.api.types.Project import Project
 
-        return GlobalID(type_name=Project.__name__, node_id=str(self.project_rowid))
+        return Project(project_rowid=self.project_rowid)
 
     @strawberry.field
     async def num_traces(
@@ -165,6 +175,81 @@ class ProjectSession(Node):
             for entry in summary
         ]
 
+    @strawberry.field
+    async def session_annotations(
+        self,
+        info: Info[Context, None],
+    ) -> list[Annotated["ProjectSessionAnnotation", lazy(".ProjectSessionAnnotation")]]:
+        """Get all annotations for this session."""
+        from phoenix.server.api.types.ProjectSessionAnnotation import (
+            to_gql_project_session_annotation,
+        )
+
+        stmt = select(models.ProjectSessionAnnotation).filter_by(project_session_id=self.id_attr)
+        async with info.context.db() as session:
+            annotations = await session.stream_scalars(stmt)
+            return [
+                to_gql_project_session_annotation(annotation) async for annotation in annotations
+            ]
+
+    @strawberry.field(
+        description="Summarizes each annotation (by name) associated with the session"
+    )  # type: ignore
+    async def session_annotation_summaries(
+        self,
+        info: Info[Context, None],
+        filter: Optional[AnnotationFilter] = None,
+    ) -> list[AnnotationSummary]:
+        """
+        Retrieves and summarizes annotations associated with this span.
+
+        This method aggregates annotation data by name and label, calculating metrics
+        such as count of occurrences and sum of scores. The results are organized
+        into a structured format that can be easily converted to a DataFrame.
+
+        Args:
+            info: GraphQL context information
+            filter: Optional filter to apply to annotations before processing
+
+        Returns:
+            A list of AnnotationSummary objects, each containing:
+            - name: The name of the annotation
+            - data: A list of dictionaries with label statistics
+        """
+        # Load all annotations for this span from the data loader
+        annotations = await info.context.data_loaders.session_annotations_by_session.load(
+            self.id_attr
+        )
+
+        # Apply filter if provided to narrow down the annotations
+        if filter:
+            annotations = [
+                annotation for annotation in annotations if satisfies_filter(annotation, filter)
+            ]
+
+        @dataclass
+        class Metrics:
+            record_count: int = 0
+            label_count: int = 0
+            score_sum: float = 0
+            score_count: int = 0
+
+        summaries: defaultdict[str, defaultdict[Optional[str], Metrics]] = defaultdict(
+            lambda: defaultdict(Metrics)
+        )
+        for annotation in annotations:
+            metrics = summaries[annotation.name][annotation.label]
+            metrics.record_count += 1
+            metrics.label_count += int(annotation.label is not None)
+            metrics.score_sum += annotation.score or 0
+            metrics.score_count += int(annotation.score is not None)
+
+        result: list[AnnotationSummary] = []
+        for name, label_metrics in summaries.items():
+            rows = [{"label": label, **asdict(metrics)} for label, metrics in label_metrics.items()]
+            result.append(AnnotationSummary(name=name, df=pd.DataFrame(rows), simple_avg=True))
+        return result
+
 
 def to_gql_project_session(project_session: models.ProjectSession) -> ProjectSession:
     return ProjectSession(

phoenix/server/api/types/ProjectSessionAnnotation.py

@@ -0,0 +1,68 @@
+from typing import Optional
+
+import strawberry
+from strawberry import Private
+from strawberry.relay import GlobalID, Node, NodeID
+from strawberry.scalars import JSON
+from strawberry.types import Info
+
+from phoenix.db import models
+from phoenix.server.api.context import Context
+from phoenix.server.api.types.AnnotatorKind import AnnotatorKind
+
+from .AnnotationSource import AnnotationSource
+from .User import User, to_gql_user
+
+
+@strawberry.type
+class ProjectSessionAnnotation(Node):
+    id_attr: NodeID[int]
+    user_id: Private[Optional[int]]
+    name: str
+    annotator_kind: AnnotatorKind
+    label: Optional[str]
+    score: Optional[float]
+    explanation: Optional[str]
+    metadata: JSON
+    _project_session_id: Private[Optional[int]]
+    identifier: str
+    source: AnnotationSource
+
+    @strawberry.field
+    async def project_session_id(self) -> GlobalID:
+        from phoenix.server.api.types.ProjectSession import ProjectSession
+
+        return GlobalID(type_name=ProjectSession.__name__, node_id=str(self._project_session_id))
+
+    @strawberry.field
+    async def user(
+        self,
+        info: Info[Context, None],
+    ) -> Optional[User]:
+        if self.user_id is None:
+            return None
+        user = await info.context.data_loaders.users.load(self.user_id)
+        if user is None:
+            return None
+        return to_gql_user(user)
+
+
+def to_gql_project_session_annotation(
+    annotation: models.ProjectSessionAnnotation,
+) -> ProjectSessionAnnotation:
+    """
+    Converts an ORM projectSession annotation to a GraphQL ProjectSessionAnnotation.
+    """
+    return ProjectSessionAnnotation(
+        id_attr=annotation.id,
+        user_id=annotation.user_id,
+        _project_session_id=annotation.project_session_id,
+        name=annotation.name,
+        annotator_kind=AnnotatorKind(annotation.annotator_kind),
+        label=annotation.label,
+        score=annotation.score,
+        explanation=annotation.explanation,
+        metadata=JSON(annotation.metadata_),
+        identifier=annotation.identifier,
+        source=AnnotationSource(annotation.source),
+    )

phoenix/server/api/types/Span.py

@@ -23,11 +23,11 @@ from phoenix.server.api.helpers.dataset_helpers import (
     get_dataset_example_input,
     get_dataset_example_output,
 )
-from phoenix.server.api.input_types.InvocationParameters import InvocationParameter
-from phoenix.server.api.input_types.SpanAnnotationFilter import (
-    SpanAnnotationFilter,
+from phoenix.server.api.input_types.AnnotationFilter import (
+    AnnotationFilter,
     satisfies_filter,
 )
+from phoenix.server.api.input_types.InvocationParameters import InvocationParameter
 from phoenix.server.api.input_types.SpanAnnotationSort import (
     SpanAnnotationColumn,
     SpanAnnotationSort,
@@ -547,7 +547,7 @@ class Span(Node):
         self,
         info: Info[Context, None],
         sort: Optional[SpanAnnotationSort] = UNSET,
-        filter: Optional[SpanAnnotationFilter] = None,
+        filter: Optional[AnnotationFilter] = None,
     ) -> list[SpanAnnotation]:
         span_id = self.span_rowid
         annotations = await info.context.data_loaders.span_annotations.load(span_id)
@@ -580,7 +580,7 @@ class Span(Node):
     async def span_annotation_summaries(
         self,
         info: Info[Context, None],
-        filter: Optional[SpanAnnotationFilter] = None,
+        filter: Optional[AnnotationFilter] = None,
     ) -> list[AnnotationSummary]:
         """
         Retrieves and summarizes annotations associated with this span.

phoenix/server/api/types/Trace.py

@@ -1,8 +1,11 @@
 from __future__ import annotations
 
+from collections import defaultdict
+from dataclasses import asdict, dataclass
 from datetime import datetime
 from typing import TYPE_CHECKING, Annotated, Optional, Union
 
+import pandas as pd
 import strawberry
 from openinference.semconv.trace import SpanAttributes
 from sqlalchemy import desc, select
@@ -13,7 +16,9 @@ from typing_extensions import TypeAlias
 
 from phoenix.db import models
 from phoenix.server.api.context import Context
+from phoenix.server.api.input_types.AnnotationFilter import AnnotationFilter, satisfies_filter
 from phoenix.server.api.input_types.TraceAnnotationSort import TraceAnnotationSort
+from phoenix.server.api.types.AnnotationSummary import AnnotationSummary
 from phoenix.server.api.types.CostBreakdown import CostBreakdown
 from phoenix.server.api.types.pagination import (
     ConnectionArgs,
@@ -229,6 +234,62 @@ class Trace(Node):
             annotations = await session.scalars(stmt)
         return [to_gql_trace_annotation(annotation) for annotation in annotations]
 
+    @strawberry.field(description="Summarizes each annotation (by name) associated with the trace")  # type: ignore
+    async def trace_annotation_summaries(
+        self,
+        info: Info[Context, None],
+        filter: Optional[AnnotationFilter] = None,
+    ) -> list[AnnotationSummary]:
+        """
+        Retrieves and summarizes annotations associated with this span.
+
+        This method aggregates annotation data by name and label, calculating metrics
+        such as count of occurrences and sum of scores. The results are organized
+        into a structured format that can be easily converted to a DataFrame.
+
+        Args:
+            info: GraphQL context information
+            filter: Optional filter to apply to annotations before processing
+
+        Returns:
+            A list of AnnotationSummary objects, each containing:
+            - name: The name of the annotation
+            - data: A list of dictionaries with label statistics
+        """
+        # Load all annotations for this span from the data loader
+        annotations = await info.context.data_loaders.trace_annotations_by_trace.load(
+            self.trace_rowid
+        )
+
+        # Apply filter if provided to narrow down the annotations
+        if filter:
+            annotations = [
+                annotation for annotation in annotations if satisfies_filter(annotation, filter)
+            ]
+
+        @dataclass
+        class Metrics:
+            record_count: int = 0
+            label_count: int = 0
+            score_sum: float = 0
+            score_count: int = 0
+
+        summaries: defaultdict[str, defaultdict[Optional[str], Metrics]] = defaultdict(
+            lambda: defaultdict(Metrics)
+        )
+        for annotation in annotations:
+            metrics = summaries[annotation.name][annotation.label]
+            metrics.record_count += 1
+            metrics.label_count += int(annotation.label is not None)
+            metrics.score_sum += annotation.score or 0
+            metrics.score_count += int(annotation.score is not None)
+
+        result: list[AnnotationSummary] = []
+        for name, label_metrics in summaries.items():
+            rows = [{"label": label, **asdict(metrics)} for label, metrics in label_metrics.items()]
+            result.append(AnnotationSummary(name=name, df=pd.DataFrame(rows), simple_avg=True))
+        return result
+
     @strawberry.field
     async def cost_summary(
         self,