arize-phoenix 4.12.1rc1__py3-none-any.whl → 4.14.1__py3-none-any.whl

phoenix/experiments/evaluators/utils.py

@@ -1,6 +1,5 @@
 import functools
 import inspect
-from itertools import chain, islice, repeat
 from typing import TYPE_CHECKING, Any, Callable, Optional, Union
 
 from phoenix.experiments.types import (
@@ -75,6 +74,72 @@ def create_evaluator(
     name: Optional[str] = None,
     scorer: Optional[Callable[[Any], EvaluationResult]] = None,
 ) -> Callable[[Callable[..., Any]], "Evaluator"]:
+    """
+    A decorator that configures a sync or async function to be used as an experiment evaluator.
+
+    If the `evaluator` is a function of one argument then that argument will be
+    bound to the `output` of an experiment task. Alternatively, the `evaluator` can be a function
+    of any combination of specific argument names that will be bound to special values:
+        `input`: The input field of the dataset example
+        `output`: The output of an experiment task
+        `expected`: The expected or reference output of the dataset example
+        `reference`: An alias for `expected`
+        `metadata`: Metadata associated with the dataset example
+
+    Args:
+        kind (str | AnnotatorKind): Broadly indicates how the evaluator scores an experiment run.
+            Valid kinds are: "CODE", "LLM". Defaults to "CODE".
+        name (str, optional): The name of the evaluator. If not provided, the name of the function
+            will be used.
+        scorer (callable, optional): An optional function that converts the output of the wrapped
+            function into an `EvaluationResult`. This allows configuring the evaluation
+            payload by setting a label, score and explanation. By default, numeric outputs will
+            be recorded as scores, boolean outputs will be recorded as scores and labels, and
+            string outputs will be recorded as labels. If the output is a 2-tuple, the first item
+            will be recorded as the score and the second item will recorded as the explanation.
+
+    Examples:
+        Configuring an evaluator that returns a boolean
+
+        .. code-block:: python
+            @create_evaluator(kind="CODE", name="exact-match)
+            def match(output: str, expected: str) -> bool:
+                return output == expected
+
+        Configuring an evaluator that returns a label
+
+        .. code-block:: python
+            client = openai.Client()
+
+            @create_evaluator(kind="LLM")
+            def label(output: str) -> str:
+                res = client.chat.completions.create(
+                    model = "gpt-4",
+                    messages = [
+                        {
+                            "role": "user",
+                            "content": (
+                                "in one word, characterize the sentiment of the following customer "
+                                f"request: {output}"
+                            )
+                        },
+                    ],
+                )
+                label = res.choices[0].message.content
+                return label
+
+        Configuring an evaluator that returns a score and explanation
+
+        .. code-block:: python
+            from textdistance import levenshtein
+
+            @create_evaluator(kind="CODE", name="levenshtein-distance")
+            def ld(output: str, expected: str) -> Tuple[float, str]:
+                return (
+                    levenshtein(output, expected),
+                    f"Levenshtein distance between {output} and {expected}"
+                )
+    """
     if scorer is None:
         scorer = _default_eval_scorer
 
@@ -163,24 +228,8 @@ def _default_eval_scorer(result: Any) -> EvaluationResult:
         return EvaluationResult(score=float(result))
     if isinstance(result, str):
         return EvaluationResult(label=result)
-    if isinstance(result, (tuple, list)) and 0 < len(result) <= 3:
-        # Possible interpretations are:
-        # - 3-tuple: (Score, Label, Explanation)
-        # - 2-tuple: (Score, Explanation) or (Label, Explanation)
-        # - 1-tuple: (Score, ) or (Label, )
-        # Note that (Score, Label) conflicts with (Score, Explanation) and we
-        # pick the latter because it's probably more prevalent. To get
-        # (Score, Label), use a 3-tuple instead, i.e. (Score, Label, None).
-        a, b, c = islice(chain(result, repeat(None)), 3)
-        score, label, explanation = None, a, b
-        if hasattr(a, "__float__"):
-            try:
-                score = float(a)
-            except ValueError:
-                pass
-            else:
-                label, explanation = (None, b) if len(result) < 3 else (b, c)
-        return EvaluationResult(score=score, label=label, explanation=explanation)
-    if result is None:
-        return EvaluationResult(score=0)
+    if isinstance(result, (tuple, list)) and len(result) == 2:
+        # If the result is a 2-tuple, the first item will be recorded as the score
+        # and the second item will recorded as the explanation.
+        return EvaluationResult(score=float(result[0]), explanation=str(result[1]))
     raise ValueError(f"Unsupported evaluation result type: {type(result)}")

phoenix/experiments/functions.py

@@ -120,21 +120,23 @@ def run_experiment(
     output. If the `task` is a function of one argument then that argument will be bound to the
     `input` field of the dataset example. Alternatively, the `task` can be a function of any
     combination of specific argument names that will be bound to special values:
-        `input`: The input field of the dataset example
-        `expected`: The expected or reference output of the dataset example
-        `reference`: An alias for `expected`
-        `metadata`: Metadata associated with the dataset example
-        `example`: The dataset `Example` object with all associated fields
+
+    - `input`: The input field of the dataset example
+    - `expected`: The expected or reference output of the dataset example
+    - `reference`: An alias for `expected`
+    - `metadata`: Metadata associated with the dataset example
+    - `example`: The dataset `Example` object with all associated fields
 
     An `evaluator` is either a synchronous or asynchronous function that returns either a boolean
     or numeric "score". If the `evaluator` is a function of one argument then that argument will be
     bound to the `output` of the task. Alternatively, the `evaluator` can be a function of any
     combination of specific argument names that will be bound to special values:
-        `input`: The input field of the dataset example
-        `output`: The output of the task
-        `expected`: The expected or reference output of the dataset example
-        `reference`: An alias for `expected`
-        `metadata`: Metadata associated with the dataset example
+
+    - `input`: The input field of the dataset example
+    - `output`: The output of the task
+    - `expected`: The expected or reference output of the dataset example
+    - `reference`: An alias for `expected`
+    - `metadata`: Metadata associated with the dataset example
 
     Phoenix also provides pre-built evaluators in the `phoenix.experiments.evaluators` module.
 
@@ -366,10 +368,9 @@ def run_experiment(
         return exp_run
 
     _errors: Tuple[Type[BaseException], ...]
-    if not hasattr(rate_limit_errors, "__iter__"):
+    if not isinstance(rate_limit_errors, Sequence):
         _errors = (rate_limit_errors,) if rate_limit_errors is not None else ()
     else:
-        rate_limit_errors = cast(Sequence[Type[BaseException]], rate_limit_errors)
         _errors = tuple(filter(None, rate_limit_errors))
     rate_limiters = [RateLimiter(rate_limit_error=rate_limit_error) for rate_limit_error in _errors]
 
@@ -606,10 +607,9 @@ def evaluate_experiment(
         return eval_run
 
     _errors: Tuple[Type[BaseException], ...]
-    if not hasattr(rate_limit_errors, "__iter__"):
+    if not isinstance(rate_limit_errors, Sequence):
         _errors = (rate_limit_errors,) if rate_limit_errors is not None else ()
     else:
-        rate_limit_errors = cast(Sequence[Type[BaseException]], rate_limit_errors)
         _errors = tuple(filter(None, rate_limit_errors))
     rate_limiters = [RateLimiter(rate_limit_error=rate_limit_error) for rate_limit_error in _errors]
 

phoenix/server/api/context.py

@@ -1,10 +1,12 @@
 from dataclasses import dataclass
 from datetime import datetime
 from pathlib import Path
-from typing import AsyncContextManager, Callable, Optional
+from typing import AsyncContextManager, Callable, Optional, Union
 
 from sqlalchemy.ext.asyncio import AsyncSession
-from strawberry.fastapi import BaseContext
+from starlette.requests import Request
+from starlette.responses import Response
+from starlette.websockets import WebSocket
 from typing_extensions import TypeAlias
 
 from phoenix.core.model_schema import Model
@@ -65,7 +67,9 @@ ProjectRowId: TypeAlias = int
 
 
 @dataclass
-class Context(BaseContext):
+class Context:
+    request: Union[Request, WebSocket]
+    response: Optional[Response]
     db: Callable[[], AsyncContextManager[AsyncSession]]
     data_loaders: DataLoaders
     cache_for_dataloaders: Optional[CacheForDataLoaders]

phoenix/server/api/dataloaders/average_experiment_run_latency.py

@@ -1,8 +1,4 @@
-from typing import (
-    AsyncContextManager,
-    Callable,
-    List,
-)
+from typing import AsyncContextManager, Callable, List, Optional
 
 from sqlalchemy import func, select
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -12,7 +8,7 @@ from typing_extensions import TypeAlias
 from phoenix.db import models
 
 ExperimentID: TypeAlias = int
-RunLatency: TypeAlias = float
+RunLatency: TypeAlias = Optional[float]
 Key: TypeAlias = ExperimentID
 Result: TypeAlias = RunLatency
 
@@ -27,26 +23,30 @@ class AverageExperimentRunLatencyDataLoader(DataLoader[Key, Result]):
 
     async def _load_fn(self, keys: List[Key]) -> List[Result]:
         experiment_ids = keys
+        resolved_experiment_ids = (
+            select(models.Experiment.id)
+            .where(models.Experiment.id.in_(set(experiment_ids)))
+            .subquery()
+        )
+        query = (
+            select(
+                resolved_experiment_ids.c.id,
+                func.avg(
+                    func.extract("epoch", models.ExperimentRun.end_time)
+                    - func.extract("epoch", models.ExperimentRun.start_time)
+                ),
+            )
+            .outerjoin_from(
+                from_=resolved_experiment_ids,
+                target=models.ExperimentRun,
+                onclause=resolved_experiment_ids.c.id == models.ExperimentRun.experiment_id,
+            )
+            .group_by(resolved_experiment_ids.c.id)
+        )
         async with self._db() as session:
             avg_latencies = {
                 experiment_id: avg_latency
-                async for experiment_id, avg_latency in await session.stream(
-                    select(
-                        models.ExperimentRun.experiment_id,
-                        func.avg(
-                            func.extract(
-                                "epoch",
-                                models.ExperimentRun.end_time,
-                            )
-                            - func.extract(
-                                "epoch",
-                                models.ExperimentRun.start_time,
-                            )
-                        ),
-                    )
-                    .where(models.ExperimentRun.experiment_id.in_(set(experiment_ids)))
-                    .group_by(models.ExperimentRun.experiment_id)
-                )
+                async for experiment_id, avg_latency in await session.stream(query)
             }
         return [
             avg_latencies.get(experiment_id, ValueError(f"Unknown experiment: {experiment_id}"))

phoenix/server/api/dataloaders/experiment_error_rates.py

@@ -5,7 +5,7 @@ from typing import (
     Optional,
 )
 
-from sqlalchemy import func, select
+from sqlalchemy import case, func, select
 from sqlalchemy.ext.asyncio import AsyncSession
 from strawberry.dataloader import DataLoader
 from typing_extensions import TypeAlias
@@ -28,16 +28,36 @@ class ExperimentErrorRatesDataLoader(DataLoader[Key, Result]):
 
     async def _load_fn(self, keys: List[Key]) -> List[Result]:
         experiment_ids = keys
+        resolved_experiment_ids = (
+            select(models.Experiment.id)
+            .where(models.Experiment.id.in_(set(experiment_ids)))
+            .subquery()
+        )
+        query = (
+            select(
+                resolved_experiment_ids.c.id,
+                case(
+                    (
+                        func.count(models.ExperimentRun.id) != 0,
+                        func.count(models.ExperimentRun.error)
+                        / func.count(models.ExperimentRun.id),
+                    ),
+                    else_=None,
+                ),
+            )
+            .outerjoin_from(
+                from_=resolved_experiment_ids,
+                target=models.ExperimentRun,
+                onclause=resolved_experiment_ids.c.id == models.ExperimentRun.experiment_id,
+            )
+            .group_by(resolved_experiment_ids.c.id)
+        )
         async with self._db() as session:
             error_rates = {
                 experiment_id: error_rate
-                async for experiment_id, error_rate in await session.stream(
-                    select(
-                        models.ExperimentRun.experiment_id,
-                        func.count(models.ExperimentRun.error) / func.count(),
-                    )
-                    .group_by(models.ExperimentRun.experiment_id)
-                    .where(models.ExperimentRun.experiment_id.in_(experiment_ids))
-                )
+                async for experiment_id, error_rate in await session.stream(query)
             }
-        return [error_rates.get(experiment_id) for experiment_id in experiment_ids]
+        return [
+            error_rates.get(experiment_id, ValueError(f"Unknown experiment ID: {experiment_id}"))
+            for experiment_id in experiment_ids
+        ]

phoenix/server/api/dataloaders/experiment_run_counts.py

@@ -27,14 +27,27 @@ class ExperimentRunCountsDataLoader(DataLoader[Key, Result]):
 
     async def _load_fn(self, keys: List[Key]) -> List[Result]:
         experiment_ids = keys
+        resolved_experiment_ids = (
+            select(models.Experiment.id)
+            .where(models.Experiment.id.in_(set(experiment_ids)))
+            .subquery()
+        )
+        query = (
+            select(
+                resolved_experiment_ids.c.id,
+                func.count(models.ExperimentRun.experiment_id),
+            )
+            .outerjoin_from(
+                from_=resolved_experiment_ids,
+                target=models.ExperimentRun,
+                onclause=resolved_experiment_ids.c.id == models.ExperimentRun.experiment_id,
+            )
+            .group_by(resolved_experiment_ids.c.id)
+        )
         async with self._db() as session:
             run_counts = {
                 experiment_id: run_count
-                async for experiment_id, run_count in await session.stream(
-                    select(models.ExperimentRun.experiment_id, func.count())
-                    .where(models.ExperimentRun.experiment_id.in_(set(experiment_ids)))
-                    .group_by(models.ExperimentRun.experiment_id)
-                )
+                async for experiment_id, run_count in await session.stream(query)
             }
         return [
             run_counts.get(experiment_id, ValueError(f"Unknown experiment: {experiment_id}"))

phoenix/server/api/input_types/{CreateSpanAnnotationsInput.py → CreateSpanAnnotationInput.py}

@@ -4,12 +4,14 @@ import strawberry
 from strawberry.relay import GlobalID
 from strawberry.scalars import JSON
 
+from phoenix.server.api.types.AnnotatorKind import AnnotatorKind
+
 
 @strawberry.input
-class CreateSpanAnnotationsInput:
+class CreateSpanAnnotationInput:
     span_id: GlobalID
     name: str
-    annotator_kind: str
+    annotator_kind: AnnotatorKind
     label: Optional[str] = None
     score: Optional[float] = None
     explanation: Optional[str] = None

phoenix/server/api/input_types/{CreateTraceAnnotationsInput.py → CreateTraceAnnotationInput.py}

@@ -4,12 +4,14 @@ import strawberry
 from strawberry.relay import GlobalID
 from strawberry.scalars import JSON
 
+from phoenix.server.api.types.AnnotatorKind import AnnotatorKind
+
 
 @strawberry.input
-class CreateTraceAnnotationsInput:
+class CreateTraceAnnotationInput:
     trace_id: GlobalID
     name: str
-    annotator_kind: str
+    annotator_kind: AnnotatorKind
     label: Optional[str] = None
     score: Optional[float] = None
     explanation: Optional[str] = None

phoenix/server/api/input_types/{PatchAnnotationsInput.py → PatchAnnotationInput.py}

@@ -5,12 +5,14 @@ from strawberry import UNSET
 from strawberry.relay import GlobalID
 from strawberry.scalars import JSON
 
+from phoenix.server.api.types.AnnotatorKind import AnnotatorKind
+
 
 @strawberry.input
-class PatchAnnotationsInput:
+class PatchAnnotationInput:
     annotation_id: GlobalID
     name: Optional[str] = UNSET
-    annotator_kind: Optional[str] = UNSET
+    annotator_kind: Optional[AnnotatorKind] = UNSET
     label: Optional[str] = UNSET
     score: Optional[float] = UNSET
     explanation: Optional[str] = UNSET

phoenix/server/api/mutations/span_annotations_mutations.py

@@ -7,9 +7,9 @@ from strawberry.types import Info
 
 from phoenix.db import models
 from phoenix.server.api.context import Context
-from phoenix.server.api.input_types.CreateSpanAnnotationsInput import CreateSpanAnnotationsInput
+from phoenix.server.api.input_types.CreateSpanAnnotationInput import CreateSpanAnnotationInput
 from phoenix.server.api.input_types.DeleteAnnotationsInput import DeleteAnnotationsInput
-from phoenix.server.api.input_types.PatchAnnotationsInput import PatchAnnotationsInput
+from phoenix.server.api.input_types.PatchAnnotationInput import PatchAnnotationInput
 from phoenix.server.api.mutations.auth import IsAuthenticated
 from phoenix.server.api.types.node import from_global_id_with_expected_type
 from phoenix.server.api.types.SpanAnnotation import SpanAnnotation, to_gql_span_annotation
@@ -24,7 +24,7 @@ class SpanAnnotationMutationPayload:
 class SpanAnnotationMutationMixin:
     @strawberry.mutation(permission_classes=[IsAuthenticated])  # type: ignore
     async def create_span_annotations(
-        self, info: Info[Context, None], input: List[CreateSpanAnnotationsInput]
+        self, info: Info[Context, None], input: List[CreateSpanAnnotationInput]
     ) -> SpanAnnotationMutationPayload:
         inserted_annotations: Sequence[models.SpanAnnotation] = []
         async with info.context.db() as session:
@@ -35,7 +35,7 @@ class SpanAnnotationMutationMixin:
                     label=annotation.label,
                     score=annotation.score,
                     explanation=annotation.explanation,
-                    annotator_kind=annotation.annotator_kind,
+                    annotator_kind=annotation.annotator_kind.value,
                     metadata_=annotation.metadata,
                 )
                 for annotation in input
@@ -54,7 +54,7 @@ class SpanAnnotationMutationMixin:
 
     @strawberry.mutation(permission_classes=[IsAuthenticated])  # type: ignore
     async def patch_span_annotations(
-        self, info: Info[Context, None], input: List[PatchAnnotationsInput]
+        self, info: Info[Context, None], input: List[PatchAnnotationInput]
     ) -> SpanAnnotationMutationPayload:
         patched_annotations = []
         async with info.context.db() as session:
@@ -66,7 +66,13 @@ class SpanAnnotationMutationMixin:
                     column.key: patch_value
                     for column, patch_value, column_is_nullable in (
                         (models.SpanAnnotation.name, annotation.name, False),
-                        (models.SpanAnnotation.annotator_kind, annotation.annotator_kind, False),
+                        (
+                            models.SpanAnnotation.annotator_kind,
+                            annotation.annotator_kind.value
+                            if annotation.annotator_kind is not None
+                            else None,
+                            False,
+                        ),
                         (models.SpanAnnotation.label, annotation.label, True),
                         (models.SpanAnnotation.score, annotation.score, True),
                         (models.SpanAnnotation.explanation, annotation.explanation, True),

phoenix/server/api/mutations/trace_annotations_mutations.py

@@ -7,9 +7,9 @@ from strawberry.types import Info
 
 from phoenix.db import models
 from phoenix.server.api.context import Context
-from phoenix.server.api.input_types.CreateTraceAnnotationsInput import CreateTraceAnnotationsInput
+from phoenix.server.api.input_types.CreateTraceAnnotationInput import CreateTraceAnnotationInput
 from phoenix.server.api.input_types.DeleteAnnotationsInput import DeleteAnnotationsInput
-from phoenix.server.api.input_types.PatchAnnotationsInput import PatchAnnotationsInput
+from phoenix.server.api.input_types.PatchAnnotationInput import PatchAnnotationInput
 from phoenix.server.api.mutations.auth import IsAuthenticated
 from phoenix.server.api.types.node import from_global_id_with_expected_type
 from phoenix.server.api.types.TraceAnnotation import TraceAnnotation, to_gql_trace_annotation
@@ -24,7 +24,7 @@ class TraceAnnotationMutationPayload:
 class TraceAnnotationMutationMixin:
     @strawberry.mutation(permission_classes=[IsAuthenticated])  # type: ignore
     async def create_trace_annotations(
-        self, info: Info[Context, None], input: List[CreateTraceAnnotationsInput]
+        self, info: Info[Context, None], input: List[CreateTraceAnnotationInput]
     ) -> TraceAnnotationMutationPayload:
         inserted_annotations: Sequence[models.TraceAnnotation] = []
         async with info.context.db() as session:
@@ -35,7 +35,7 @@ class TraceAnnotationMutationMixin:
                     label=annotation.label,
                     score=annotation.score,
                     explanation=annotation.explanation,
-                    annotator_kind=annotation.annotator_kind,
+                    annotator_kind=annotation.annotator_kind.value,
                     metadata_=annotation.metadata,
                 )
                 for annotation in input
@@ -54,7 +54,7 @@ class TraceAnnotationMutationMixin:
 
     @strawberry.mutation(permission_classes=[IsAuthenticated])  # type: ignore
     async def patch_trace_annotations(
-        self, info: Info[Context, None], input: List[PatchAnnotationsInput]
+        self, info: Info[Context, None], input: List[PatchAnnotationInput]
     ) -> TraceAnnotationMutationPayload:
         patched_annotations = []
         async with info.context.db() as session:
@@ -66,7 +66,13 @@ class TraceAnnotationMutationMixin:
                     column.key: patch_value
                     for column, patch_value, column_is_nullable in (
                         (models.TraceAnnotation.name, annotation.name, False),
-                        (models.TraceAnnotation.annotator_kind, annotation.annotator_kind, False),
+                        (
+                            models.TraceAnnotation.annotator_kind,
+                            annotation.annotator_kind.value
+                            if annotation.annotator_kind is not None
+                            else None,
+                            False,
+                        ),
                         (models.TraceAnnotation.label, annotation.label, True),
                         (models.TraceAnnotation.score, annotation.score, True),
                         (models.TraceAnnotation.explanation, annotation.explanation, True),

phoenix/server/api/openapi/main.py

@@ -1,22 +1,6 @@
-import json
-from argparse import ArgumentParser
-from typing import Optional, Tuple
-
 from .schema import get_openapi_schema
 
 if __name__ == "__main__":
-    parser = ArgumentParser()
-    parser.add_argument(
-        "--compress",
-        action="store_true",
-        help="Whether to output a compressed version of the OpenAPI schema",
-    )
-    args = parser.parse_args()
+    import yaml  # type: ignore
 
-    indent: Optional[int] = None
-    separator: Optional[Tuple[str, str]] = None
-    if args.compress:
-        separator = (",", ":")
-    else:
-        indent = 2
-    print(json.dumps(get_openapi_schema(), indent=indent, separators=separator))
+    print(yaml.dump(get_openapi_schema(), indent=2))

phoenix/server/api/openapi/schema.py

@@ -1,16 +1,16 @@
-from typing import Any, Dict
+from typing import Any
 
-from fastapi.openapi.utils import get_openapi
+from starlette.schemas import SchemaGenerator
 
-from phoenix.server.api.routers.v1 import REST_API_VERSION
-from phoenix.server.api.routers.v1 import router as v1_router
+from phoenix.server.api.routers.v1 import V1_ROUTES
 
+OPENAPI_SCHEMA_GENERATOR = SchemaGenerator(
+    {"openapi": "3.0.0", "info": {"title": "Arize-Phoenix API", "version": "1.0"}}
+)
 
-def get_openapi_schema() -> Dict[str, Any]:
-    return get_openapi(
-        title="Arize-Phoenix REST API",
-        version=REST_API_VERSION,
-        openapi_version="3.1.0",
-        description="Schema for Arize-Phoenix REST API",
-        routes=v1_router.routes,
-    )
+
+def get_openapi_schema() -> Any:
+    """
+    Exports an OpenAPI schema for the Phoenix REST API as a JSON object.
+    """
+    return OPENAPI_SCHEMA_GENERATOR.get_schema(V1_ROUTES)  # type: ignore