arize-phoenix 3.16.1__py3-none-any.whl → 7.7.0__py3-none-any.whl

phoenix/server/api/types/Evaluation.py

@@ -1,52 +1,38 @@
-from typing import Optional
-
 import strawberry
 
 import phoenix.trace.v1 as pb
-from phoenix.trace.schemas import SpanID
+from phoenix.db.models import DocumentAnnotation, TraceAnnotation
 
-
-@strawberry.interface
-class Evaluation:
-    name: str = strawberry.field(
-        description="Name of the evaluation, e.g. 'helpfulness' or 'relevance'."
-    )
-    score: Optional[float] = strawberry.field(
-        description="Result of the evaluation in the form of a numeric score."
-    )
-    label: Optional[str] = strawberry.field(
-        description="Result of the evaluation in the form of a string, e.g. "
-        "'helpful' or 'not helpful'. Note that the label is not necessarily binary."
-    )
-    explanation: Optional[str] = strawberry.field(
-        description="The evaluator's explanation for the evaluation result (i.e. "
-        "score or label, or both) given to the subject."
-    )
+from .Annotation import Annotation
 
 
 @strawberry.type
-class SpanEvaluation(Evaluation):
-    span_id: strawberry.Private[SpanID]
-
+class TraceEvaluation(Annotation):
     @staticmethod
-    def from_pb_evaluation(evaluation: pb.Evaluation) -> "SpanEvaluation":
+    def from_pb_evaluation(evaluation: pb.Evaluation) -> "TraceEvaluation":
         result = evaluation.result
         score = result.score.value if result.HasField("score") else None
         label = result.label.value if result.HasField("label") else None
         explanation = result.explanation.value if result.HasField("explanation") else None
-        span_id = SpanID(evaluation.subject_id.span_id)
-        return SpanEvaluation(
+        return TraceEvaluation(
             name=evaluation.name,
             score=score,
             label=label,
             explanation=explanation,
-            span_id=span_id,
+        )
+
+    @staticmethod
+    def from_sql_trace_annotation(annotation: TraceAnnotation) -> "TraceEvaluation":
+        return TraceEvaluation(
+            name=annotation.name,
+            score=annotation.score,
+            label=annotation.label,
+            explanation=annotation.explanation,
         )
 
 
 @strawberry.type
-class DocumentEvaluation(Evaluation):
-    span_id: strawberry.Private[SpanID]
+class DocumentEvaluation(Annotation):
     document_position: int = strawberry.field(
         description="The zero-based index among retrieved documents, which "
         "is collected as a list (even when ordering is not inherently meaningful)."
@@ -60,12 +46,20 @@ class DocumentEvaluation(Evaluation):
         explanation = result.explanation.value if result.HasField("explanation") else None
         document_retrieval_id = evaluation.subject_id.document_retrieval_id
         document_position = document_retrieval_id.document_position
-        span_id = SpanID(document_retrieval_id.span_id)
         return DocumentEvaluation(
             name=evaluation.name,
             score=score,
             label=label,
             explanation=explanation,
             document_position=document_position,
-            span_id=span_id,
+        )
+
+    @staticmethod
+    def from_sql_document_annotation(annotation: DocumentAnnotation) -> "DocumentEvaluation":
+        return DocumentEvaluation(
+            name=annotation.name,
+            score=annotation.score,
+            label=annotation.label,
+            explanation=annotation.explanation,
+            document_position=annotation.document_position,
         )

phoenix/server/api/types/EvaluationSummary.py

@@ -1,75 +1,55 @@
-import math
-from functools import cached_property
-from typing import List, Optional, Tuple, cast
+from typing import Optional, Union, cast
 
 import pandas as pd
 import strawberry
-from pandas.api.types import CategoricalDtype
 from strawberry import Private
 
-import phoenix.trace.v1 as pb
+from phoenix.db import models
+from phoenix.server.api.types.LabelFraction import LabelFraction
 
-
-@strawberry.type
-class LabelFraction:
-    label: str
-    fraction: float
+AnnotationType = Union[models.SpanAnnotation, models.TraceAnnotation]
 
 
 @strawberry.type
 class EvaluationSummary:
-    count: int
-    labels: Tuple[str, ...]
-    evaluations: Private[Tuple[pb.Evaluation, ...]]
+    df: Private[pd.DataFrame]
+
+    def __init__(self, dataframe: pd.DataFrame) -> None:
+        self.df = dataframe
+
+    @strawberry.field
+    def count(self) -> int:
+        return cast(int, self.df.record_count.sum())
 
-    def __init__(
-        self,
-        evaluations: Tuple[pb.Evaluation, ...],
-        labels: Tuple[str, ...],
-    ) -> None:
-        self.evaluations = evaluations
-        self.labels = labels
-        self.count = len(evaluations)
+    @strawberry.field
+    def labels(self) -> list[str]:
+        return self.df.label.dropna().tolist()
 
     @strawberry.field
-    def label_fractions(self) -> List[LabelFraction]:
-        if not self.labels or not (n := len(self._eval_labels)):
+    def label_fractions(self) -> list[LabelFraction]:
+        if not (n := self.df.label_count.sum()):
             return []
-        counts = self._eval_labels.value_counts(dropna=True)
         return [
-            LabelFraction(label=cast(str, label), fraction=count / n)
-            for label, count in counts.items()
+            LabelFraction(
+                label=cast(str, row.label),
+                fraction=row.label_count / n,
+            )
+            for row in self.df.loc[
+                self.df.label.notna(),
+                ["label", "label_count"],
+            ].itertuples()
         ]
 
     @strawberry.field
     def mean_score(self) -> Optional[float]:
-        value = self._eval_scores.mean()
-        return None if math.isnan(value) else value
+        if not (n := self.df.score_count.sum()):
+            return None
+        return cast(float, self.df.score_sum.sum() / n)
 
     @strawberry.field
     def score_count(self) -> int:
-        return self._eval_scores.count()
+        return cast(int, self.df.score_count.sum())
 
     @strawberry.field
     def label_count(self) -> int:
-        return self._eval_labels.count()
-
-    @cached_property
-    def _eval_scores(self) -> "pd.Series[float]":
-        return pd.Series(
-            (
-                evaluation.result.score.value if evaluation.result.HasField("score") else None
-                for evaluation in self.evaluations
-            ),
-            dtype=float,
-        )
-
-    @cached_property
-    def _eval_labels(self) -> "pd.Series[CategoricalDtype]":
-        return pd.Series(
-            (
-                evaluation.result.label.value if evaluation.result.HasField("label") else None
-                for evaluation in self.evaluations
-            ),
-            dtype=CategoricalDtype(categories=self.labels),  # type: ignore
-        )
+        return cast(int, self.df.label_count.sum())

phoenix/server/api/types/Event.py

@@ -1,6 +1,6 @@
 import math
 from collections import defaultdict
-from typing import Dict, List, Optional, Tuple, Union, cast
+from typing import Optional, Union, cast
 
 import strawberry
 from strawberry import ID
@@ -17,10 +17,10 @@ from phoenix.core.model_schema import (
 )
 
 from ..interceptor import GqlValueMediator
-from .DatasetRole import STR_TO_DATASET_ROLE, AncillaryDatasetRole, DatasetRole
 from .Dimension import Dimension
 from .DimensionWithValue import DimensionWithValue
 from .EventMetadata import EventMetadata
+from .InferencesRole import STR_TO_INFEREENCES_ROLE, AncillaryInferencesRole, InferencesRole
 from .PromptResponse import PromptResponse
 
 
@@ -28,7 +28,7 @@ from .PromptResponse import PromptResponse
 class Event:
     id: strawberry.ID
     eventMetadata: EventMetadata
-    dimensions: List[DimensionWithValue]
+    dimensions: list[DimensionWithValue]
     prompt_and_response: Optional[PromptResponse] = strawberry.field(
         description="The prompt and response pair associated with the event",
         default=GqlValueMediator(),
@@ -41,42 +41,42 @@ class Event:
 
 def create_event_id(
     row_id: int,
-    dataset_role: Union[DatasetRole, AncillaryDatasetRole, ms.DatasetRole],
+    inferences_role: Union[InferencesRole, AncillaryInferencesRole, ms.InferencesRole],
 ) -> ID:
-    dataset_role_str = (
-        dataset_role.value
-        if isinstance(dataset_role, (DatasetRole, AncillaryDatasetRole))
-        else dataset_role
+    inferences_role_str = (
+        inferences_role.value
+        if isinstance(inferences_role, (InferencesRole, AncillaryInferencesRole))
+        else inferences_role
     )
-    return ID(f"{row_id}:{dataset_role_str}")
+    return ID(f"{row_id}:{inferences_role_str}")
 
 
 def unpack_event_id(
     event_id: ID,
-) -> Tuple[int, Union[DatasetRole, AncillaryDatasetRole]]:
-    row_id_str, dataset_role_str = str(event_id).split(":")
+) -> tuple[int, Union[InferencesRole, AncillaryInferencesRole]]:
+    row_id_str, inferences_role_str = str(event_id).split(":")
     row_id = int(row_id_str)
-    dataset_role = STR_TO_DATASET_ROLE[dataset_role_str]
-    return row_id, dataset_role
+    inferences_role = STR_TO_INFEREENCES_ROLE[inferences_role_str]
+    return row_id, inferences_role
 
 
-def parse_event_ids_by_dataset_role(
-    event_ids: List[ID],
-) -> Dict[Union[DatasetRole, AncillaryDatasetRole], List[int]]:
+def parse_event_ids_by_inferences_role(
+    event_ids: list[ID],
+) -> dict[Union[InferencesRole, AncillaryInferencesRole], list[int]]:
     """
     Parses event IDs and returns the corresponding row indexes.
     """
-    row_indexes: Dict[Union[DatasetRole, AncillaryDatasetRole], List[int]] = defaultdict(list)
+    row_indexes: dict[Union[InferencesRole, AncillaryInferencesRole], list[int]] = defaultdict(list)
     for event_id in event_ids:
-        row_id, dataset_role = unpack_event_id(event_id)
-        row_indexes[dataset_role].append(row_id)
+        row_id, inferences_role = unpack_event_id(event_id)
+        row_indexes[inferences_role].append(row_id)
     return row_indexes
 
 
 def create_event(
     event_id: ID,
     event: ms.Event,
-    dimensions: List[Dimension],
+    dimensions: list[Dimension],
     is_document_record: bool = False,
 ) -> Event:
     """

phoenix/server/api/types/ExampleRevisionInterface.py

@@ -0,0 +1,14 @@
+import strawberry
+from strawberry.scalars import JSON
+
+
+@strawberry.interface
+class ExampleRevision:
+    """
+    Represents an example revision for generative tasks.
+    For example, you might have text -> text, text -> labels, etc.
+    """
+
+    input: JSON
+    output: JSON
+    metadata: JSON

phoenix/server/api/types/Experiment.py

@@ -0,0 +1,152 @@
+from datetime import datetime
+from typing import ClassVar, Optional
+
+import strawberry
+from sqlalchemy import select
+from sqlalchemy.orm import joinedload
+from strawberry import UNSET, Private
+from strawberry.relay import Connection, Node, NodeID
+from strawberry.scalars import JSON
+from strawberry.types import Info
+
+from phoenix.db import models
+from phoenix.server.api.context import Context
+from phoenix.server.api.types.ExperimentAnnotationSummary import ExperimentAnnotationSummary
+from phoenix.server.api.types.ExperimentRun import ExperimentRun, to_gql_experiment_run
+from phoenix.server.api.types.pagination import (
+    ConnectionArgs,
+    CursorString,
+    connection_from_list,
+)
+from phoenix.server.api.types.Project import Project
+
+
+@strawberry.type
+class Experiment(Node):
+    _table: ClassVar[type[models.Base]] = models.Experiment
+    cached_sequence_number: Private[Optional[int]] = None
+    id_attr: NodeID[int]
+    name: str
+    project_name: Optional[str]
+    description: Optional[str]
+    metadata: JSON
+    created_at: datetime
+    updated_at: datetime
+
+    @strawberry.field(
+        description="Sequence number (1-based) of experiments belonging to the same dataset"
+    )  # type: ignore
+    async def sequence_number(
+        self,
+        info: Info[Context, None],
+    ) -> int:
+        if self.cached_sequence_number is None:
+            seq_num = await info.context.data_loaders.experiment_sequence_number.load(self.id_attr)
+            if seq_num is None:
+                raise ValueError(f"invalid experiment: id={self.id_attr}")
+            self.cached_sequence_number = seq_num
+        return self.cached_sequence_number
+
+    @strawberry.field
+    async def runs(
+        self,
+        info: Info[Context, None],
+        first: Optional[int] = 50,
+        last: Optional[int] = UNSET,
+        after: Optional[CursorString] = UNSET,
+        before: Optional[CursorString] = UNSET,
+    ) -> Connection[ExperimentRun]:
+        args = ConnectionArgs(
+            first=first,
+            after=after if isinstance(after, CursorString) else None,
+            last=last,
+            before=before if isinstance(before, CursorString) else None,
+        )
+        experiment_id = self.id_attr
+        async with info.context.db() as session:
+            runs = (
+                await session.scalars(
+                    select(models.ExperimentRun)
+                    .where(models.ExperimentRun.experiment_id == experiment_id)
+                    .order_by(models.ExperimentRun.id.desc())
+                    .options(
+                        joinedload(models.ExperimentRun.trace).load_only(models.Trace.trace_id)
+                    )
+                )
+            ).all()
+        return connection_from_list([to_gql_experiment_run(run) for run in runs], args)
+
+    @strawberry.field
+    async def run_count(self, info: Info[Context, None]) -> int:
+        experiment_id = self.id_attr
+        return await info.context.data_loaders.experiment_run_counts.load(experiment_id)
+
+    @strawberry.field
+    async def annotation_summaries(
+        self, info: Info[Context, None]
+    ) -> list[ExperimentAnnotationSummary]:
+        experiment_id = self.id_attr
+        return [
+            ExperimentAnnotationSummary(
+                annotation_name=summary.annotation_name,
+                min_score=summary.min_score,
+                max_score=summary.max_score,
+                mean_score=summary.mean_score,
+                count=summary.count,
+                error_count=summary.error_count,
+            )
+            for summary in await info.context.data_loaders.experiment_annotation_summaries.load(
+                experiment_id
+            )
+        ]
+
+    @strawberry.field
+    async def error_rate(self, info: Info[Context, None]) -> Optional[float]:
+        return await info.context.data_loaders.experiment_error_rates.load(self.id_attr)
+
+    @strawberry.field
+    async def average_run_latency_ms(self, info: Info[Context, None]) -> Optional[float]:
+        latency_seconds = await info.context.data_loaders.average_experiment_run_latency.load(
+            self.id_attr
+        )
+        return latency_seconds * 1000 if latency_seconds is not None else None
+
+    @strawberry.field
+    async def project(self, info: Info[Context, None]) -> Optional[Project]:
+        if self.project_name is None:
+            return None
+
+        db_project = await info.context.data_loaders.project_by_name.load(self.project_name)
+
+        if db_project is None:
+            return None
+
+        return Project(
+            id_attr=db_project.id,
+            name=db_project.name,
+            gradient_start_color=db_project.gradient_start_color,
+            gradient_end_color=db_project.gradient_end_color,
+        )
+
+    @strawberry.field
+    def last_updated_at(self, info: Info[Context, None]) -> Optional[datetime]:
+        return info.context.last_updated_at.get(self._table, self.id_attr)
+
+
+def to_gql_experiment(
+    experiment: models.Experiment,
+    sequence_number: Optional[int] = None,
+) -> Experiment:
+    """
+    Converts an ORM experiment to a GraphQL Experiment.
+    """
+    return Experiment(
+        cached_sequence_number=sequence_number,
+        id_attr=experiment.id,
+        name=experiment.name,
+        project_name=experiment.project_name,
+        description=experiment.description,
+        metadata=experiment.metadata_,
+        created_at=experiment.created_at,
+        updated_at=experiment.updated_at,
+    )

phoenix/server/api/types/ExperimentAnnotationSummary.py

@@ -0,0 +1,13 @@
+from typing import Optional
+
+import strawberry
+
+
+@strawberry.type
+class ExperimentAnnotationSummary:
+    annotation_name: str
+    min_score: Optional[float]
+    max_score: Optional[float]
+    mean_score: Optional[float]
+    count: int
+    error_count: int

phoenix/server/api/types/ExperimentComparison.py

@@ -0,0 +1,17 @@
+import strawberry
+from strawberry.relay import GlobalID
+
+from phoenix.server.api.types.DatasetExample import DatasetExample
+from phoenix.server.api.types.ExperimentRun import ExperimentRun
+
+
+@strawberry.type
+class RunComparisonItem:
+    experiment_id: GlobalID
+    runs: list[ExperimentRun]
+
+
+@strawberry.type
+class ExperimentComparison:
+    example: DatasetExample
+    run_comparison_items: list[RunComparisonItem]

phoenix/server/api/types/ExperimentRun.py

@@ -0,0 +1,119 @@
+from datetime import datetime
+from typing import TYPE_CHECKING, Annotated, Optional
+
+import strawberry
+from sqlalchemy import select
+from sqlalchemy.orm import load_only
+from strawberry import UNSET
+from strawberry.relay import Connection, GlobalID, Node, NodeID
+from strawberry.scalars import JSON
+from strawberry.types import Info
+
+from phoenix.db import models
+from phoenix.server.api.context import Context
+from phoenix.server.api.types.ExperimentRunAnnotation import (
+    ExperimentRunAnnotation,
+    to_gql_experiment_run_annotation,
+)
+from phoenix.server.api.types.pagination import (
+    ConnectionArgs,
+    CursorString,
+    connection_from_list,
+)
+from phoenix.server.api.types.Trace import Trace, to_gql_trace
+
+if TYPE_CHECKING:
+    from phoenix.server.api.types.DatasetExample import DatasetExample
+
+
+@strawberry.type
+class ExperimentRun(Node):
+    id_attr: NodeID[int]
+    experiment_id: GlobalID
+    trace_id: Optional[str]
+    output: Optional[JSON]
+    start_time: datetime
+    end_time: datetime
+    error: Optional[str]
+
+    @strawberry.field
+    async def annotations(
+        self,
+        info: Info[Context, None],
+        first: Optional[int] = 50,
+        last: Optional[int] = UNSET,
+        after: Optional[CursorString] = UNSET,
+        before: Optional[CursorString] = UNSET,
+    ) -> Connection[ExperimentRunAnnotation]:
+        args = ConnectionArgs(
+            first=first,
+            after=after if isinstance(after, CursorString) else None,
+            last=last,
+            before=before if isinstance(before, CursorString) else None,
+        )
+        run_id = self.id_attr
+        annotations = await info.context.data_loaders.experiment_run_annotations.load(run_id)
+        return connection_from_list(
+            [to_gql_experiment_run_annotation(annotation) for annotation in annotations], args
+        )
+
+    @strawberry.field
+    async def trace(self, info: Info) -> Optional[Trace]:
+        if not self.trace_id:
+            return None
+        dataloader = info.context.data_loaders.trace_by_trace_ids
+        if (trace := await dataloader.load(self.trace_id)) is None:
+            return None
+        return to_gql_trace(trace)
+
+    @strawberry.field
+    async def example(
+        self, info: Info
+    ) -> Annotated[
+        "DatasetExample", strawberry.lazy("phoenix.server.api.types.DatasetExample")
+    ]:  # use lazy types to avoid circular import: https://strawberry.rocks/docs/types/lazy
+        from phoenix.server.api.types.DatasetExample import DatasetExample
+
+        async with info.context.db() as session:
+            assert (
+                result := await session.execute(
+                    select(models.DatasetExample, models.Experiment.dataset_version_id)
+                    .select_from(models.ExperimentRun)
+                    .join(
+                        models.DatasetExample,
+                        models.DatasetExample.id == models.ExperimentRun.dataset_example_id,
+                    )
+                    .join(
+                        models.Experiment,
+                        models.Experiment.id == models.ExperimentRun.experiment_id,
+                    )
+                    .where(models.ExperimentRun.id == self.id_attr)
+                    .options(load_only(models.DatasetExample.id, models.DatasetExample.created_at))
+                )
+            ) is not None
+            example, version_id = result.first()
+        return DatasetExample(
+            id_attr=example.id,
+            created_at=example.created_at,
+            version_id=version_id,
+        )
+
+
+def to_gql_experiment_run(run: models.ExperimentRun) -> ExperimentRun:
+    """
+    Converts an ORM experiment run to a GraphQL ExperimentRun.
+    """
+
+    from phoenix.server.api.types.Experiment import Experiment
+
+    return ExperimentRun(
+        id_attr=run.id,
+        experiment_id=GlobalID(Experiment.__name__, str(run.experiment_id)),
+        trace_id=trace_id
+        if (trace := run.trace) and (trace_id := trace.trace_id) is not None
+        else None,
+        output=run.output.get("task_output"),
+        start_time=run.start_time,
+        end_time=run.end_time,
+        error=run.error,
+    )

phoenix/server/api/types/ExperimentRunAnnotation.py

@@ -0,0 +1,56 @@
+from datetime import datetime
+from typing import Optional
+
+import strawberry
+from strawberry import Info
+from strawberry.relay import Node, NodeID
+from strawberry.scalars import JSON
+
+from phoenix.db import models
+from phoenix.server.api.types.AnnotatorKind import ExperimentRunAnnotatorKind
+from phoenix.server.api.types.Trace import Trace, to_gql_trace
+
+
+@strawberry.type
+class ExperimentRunAnnotation(Node):
+    id_attr: NodeID[int]
+    name: str
+    annotator_kind: ExperimentRunAnnotatorKind
+    label: Optional[str]
+    score: Optional[float]
+    explanation: Optional[str]
+    error: Optional[str]
+    metadata: JSON
+    start_time: datetime
+    end_time: datetime
+    trace_id: Optional[str]
+
+    @strawberry.field
+    async def trace(self, info: Info) -> Optional[Trace]:
+        if not self.trace_id:
+            return None
+        dataloader = info.context.data_loaders.trace_by_trace_ids
+        if (trace := await dataloader.load(self.trace_id)) is None:
+            return None
+        return to_gql_trace(trace)
+
+
+def to_gql_experiment_run_annotation(
+    annotation: models.ExperimentRunAnnotation,
+) -> ExperimentRunAnnotation:
+    """
+    Converts an ORM experiment run annotation to a GraphQL ExperimentRunAnnotation.
+    """
+    return ExperimentRunAnnotation(
+        id_attr=annotation.id,
+        name=annotation.name,
+        annotator_kind=ExperimentRunAnnotatorKind(annotation.annotator_kind),
+        label=annotation.label,
+        score=annotation.score,
+        explanation=annotation.explanation,
+        error=annotation.error,
+        metadata=annotation.metadata_,
+        start_time=annotation.start_time,
+        end_time=annotation.end_time,
+        trace_id=annotation.trace_id,
+    )

phoenix/server/api/types/GenerativeModel.py

@@ -0,0 +1,9 @@
+import strawberry
+
+from phoenix.server.api.types.GenerativeProvider import GenerativeProviderKey
+
+
+@strawberry.type
+class GenerativeModel:
+    name: str
+    provider_key: GenerativeProviderKey