arize-phoenix 3.16.0__py3-none-any.whl → 7.7.0__py3-none-any.whl

phoenix/server/api/routers/v1/evaluations.py

@@ -0,0 +1,362 @@
+import gzip
+from collections.abc import Callable
+from itertools import chain
+from typing import Any, Iterator, Optional, Union, cast
+
+import pandas as pd
+import pyarrow as pa
+from fastapi import APIRouter, Header, HTTPException, Query
+from google.protobuf.message import DecodeError
+from pandas import DataFrame
+from sqlalchemy import select
+from sqlalchemy.engine import Connectable
+from starlette.background import BackgroundTask
+from starlette.datastructures import State
+from starlette.requests import Request
+from starlette.responses import Response, StreamingResponse
+from starlette.status import (
+    HTTP_204_NO_CONTENT,
+    HTTP_404_NOT_FOUND,
+    HTTP_415_UNSUPPORTED_MEDIA_TYPE,
+    HTTP_422_UNPROCESSABLE_ENTITY,
+)
+from typing_extensions import TypeAlias
+
+import phoenix.trace.v1 as pb
+from phoenix.config import DEFAULT_PROJECT_NAME
+from phoenix.db import models
+from phoenix.db.insertion.types import Precursors
+from phoenix.exceptions import PhoenixEvaluationNameIsMissing
+from phoenix.server.api.routers.utils import table_to_bytes
+from phoenix.server.types import DbSessionFactory
+from phoenix.trace.span_evaluations import (
+    DocumentEvaluations,
+    Evaluations,
+    SpanEvaluations,
+    TraceEvaluations,
+)
+
+from .utils import add_errors_to_responses
+
+EvaluationName: TypeAlias = str
+
+router = APIRouter(tags=["traces"], include_in_schema=True)
+
+
+@router.post(
+    "/evaluations",
+    operation_id="addEvaluations",
+    summary="Add span, trace, or document evaluations",
+    status_code=HTTP_204_NO_CONTENT,
+    responses=add_errors_to_responses(
+        [
+            {
+                "status_code": HTTP_415_UNSUPPORTED_MEDIA_TYPE,
+                "description": (
+                    "Unsupported content type, "
+                    "only gzipped protobuf and pandas-arrow are supported"
+                ),
+            },
+            HTTP_422_UNPROCESSABLE_ENTITY,
+        ]
+    ),
+    openapi_extra={
+        "requestBody": {
+            "required": True,
+            "content": {
+                "application/x-protobuf": {"schema": {"type": "string", "format": "binary"}},
+                "application/x-pandas-arrow": {"schema": {"type": "string", "format": "binary"}},
+            },
+        },
+    },
+)
+async def post_evaluations(
+    request: Request,
+    content_type: Optional[str] = Header(default=None),
+    content_encoding: Optional[str] = Header(default=None),
+) -> Response:
+    if content_type == "application/x-pandas-arrow":
+        return await _process_pyarrow(request)
+    if content_type != "application/x-protobuf":
+        raise HTTPException(
+            detail="Unsupported content type", status_code=HTTP_415_UNSUPPORTED_MEDIA_TYPE
+        )
+    body = await request.body()
+    if content_encoding == "gzip":
+        body = gzip.decompress(body)
+    elif content_encoding:
+        raise HTTPException(
+            detail="Unsupported content encoding", status_code=HTTP_415_UNSUPPORTED_MEDIA_TYPE
+        )
+    evaluation = pb.Evaluation()
+    try:
+        evaluation.ParseFromString(body)
+    except DecodeError:
+        raise HTTPException(
+            detail="Request body is invalid", status_code=HTTP_422_UNPROCESSABLE_ENTITY
+        )
+    if not evaluation.name.strip():
+        raise HTTPException(
+            detail="Evaluation name must not be blank/empty",
+            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+        )
+    await request.state.queue_evaluation_for_bulk_insert(evaluation)
+    return Response()
+
+
+@router.get(
+    "/evaluations",
+    operation_id="getEvaluations",
+    summary="Get span, trace, or document evaluations from a project",
+    responses=add_errors_to_responses([HTTP_404_NOT_FOUND]),
+)
+async def get_evaluations(
+    request: Request,
+    project_name: Optional[str] = Query(
+        default=None,
+        description=(
+            "The name of the project to get evaluations from (if omitted, "
+            f"evaluations will be drawn from the `{DEFAULT_PROJECT_NAME}` project)"
+        ),
+    ),
+) -> Response:
+    project_name = (
+        project_name
+        or request.query_params.get("project-name")  # for backward compatibility
+        or request.headers.get("project-name")  # read from headers for backwards compatibility
+        or DEFAULT_PROJECT_NAME
+    )
+
+    db: DbSessionFactory = request.app.state.db
+    async with db() as session:
+        connection = await session.connection()
+        trace_evals_dataframe = await connection.run_sync(
+            _read_sql_trace_evaluations_into_dataframe,
+            project_name,
+        )
+        span_evals_dataframe = await connection.run_sync(
+            _read_sql_span_evaluations_into_dataframe,
+            project_name,
+        )
+        document_evals_dataframe = await connection.run_sync(
+            _read_sql_document_evaluations_into_dataframe,
+            project_name,
+        )
+    if (
+        trace_evals_dataframe.empty
+        and span_evals_dataframe.empty
+        and document_evals_dataframe.empty
+    ):
+        return Response(status_code=HTTP_404_NOT_FOUND)
+
+    evals = chain(
+        map(
+            lambda args: TraceEvaluations(*args),
+            _groupby_eval_name(trace_evals_dataframe),
+        ),
+        map(
+            lambda args: SpanEvaluations(*args),
+            _groupby_eval_name(span_evals_dataframe),
+        ),
+        map(
+            lambda args: DocumentEvaluations(*args),
+            _groupby_eval_name(document_evals_dataframe),
+        ),
+    )
+    bytestream = map(lambda evals: table_to_bytes(evals.to_pyarrow_table()), evals)
+    return StreamingResponse(
+        content=bytestream,
+        media_type="application/x-pandas-arrow",
+    )
+
+
+async def _process_pyarrow(request: Request) -> Response:
+    body = await request.body()
+    try:
+        reader = pa.ipc.open_stream(body)
+    except pa.ArrowInvalid:
+        raise HTTPException(
+            detail="Request body is not valid pyarrow",
+            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+        )
+    try:
+        evaluations = Evaluations.from_pyarrow_reader(reader)
+    except Exception as e:
+        if isinstance(e, PhoenixEvaluationNameIsMissing):
+            raise HTTPException(
+                detail="Evaluation name must not be blank/empty",
+                status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+            )
+        raise HTTPException(
+            detail="Invalid data in request body",
+            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+        )
+    return Response(background=BackgroundTask(_add_evaluations, request.state, evaluations))
+
+
+async def _add_evaluations(state: State, evaluations: Evaluations) -> None:
+    dataframe = evaluations.dataframe
+    eval_name = evaluations.eval_name
+    names = dataframe.index.names
+    if (
+        len(names) == 2
+        and "document_position" in names
+        and ("context.span_id" in names or "span_id" in names)
+    ):
+        cls = _document_annotation_factory(
+            names.index("span_id") if "span_id" in names else names.index("context.span_id"),
+            names.index("document_position"),
+        )
+        for index, row in dataframe.iterrows():
+            score, label, explanation = _get_annotation_result(row)
+            document_annotation = cls(cast(Union[tuple[str, int], tuple[int, str]], index))(
+                name=eval_name,
+                annotator_kind="LLM",
+                score=score,
+                label=label,
+                explanation=explanation,
+                metadata_={},
+            )
+            await state.enqueue(document_annotation)
+    elif len(names) == 1 and names[0] in ("context.span_id", "span_id"):
+        for index, row in dataframe.iterrows():
+            score, label, explanation = _get_annotation_result(row)
+            span_annotation = _span_annotation_factory(cast(str, index))(
+                name=eval_name,
+                annotator_kind="LLM",
+                score=score,
+                label=label,
+                explanation=explanation,
+                metadata_={},
+            )
+            await state.enqueue(span_annotation)
+    elif len(names) == 1 and names[0] in ("context.trace_id", "trace_id"):
+        for index, row in dataframe.iterrows():
+            score, label, explanation = _get_annotation_result(row)
+            trace_annotation = _trace_annotation_factory(cast(str, index))(
+                name=eval_name,
+                annotator_kind="LLM",
+                score=score,
+                label=label,
+                explanation=explanation,
+                metadata_={},
+            )
+            await state.enqueue(trace_annotation)
+
+
+def _get_annotation_result(
+    row: "pd.Series[Any]",
+) -> tuple[Optional[float], Optional[str], Optional[str]]:
+    return (
+        cast(Optional[float], row.get("score")),
+        cast(Optional[str], row.get("label")),
+        cast(Optional[str], row.get("explanation")),
+    )
+
+
+def _document_annotation_factory(
+    span_id_idx: int,
+    document_position_idx: int,
+) -> Callable[
+    [Union[tuple[str, int], tuple[int, str]]],
+    Callable[..., Precursors.DocumentAnnotation],
+]:
+    return lambda index: lambda **kwargs: Precursors.DocumentAnnotation(
+        span_id=str(index[span_id_idx]),
+        document_position=int(index[document_position_idx]),
+        obj=models.DocumentAnnotation(
+            document_position=int(index[document_position_idx]),
+            **kwargs,
+        ),
+    )
+
+
+def _span_annotation_factory(span_id: str) -> Callable[..., Precursors.SpanAnnotation]:
+    return lambda **kwargs: Precursors.SpanAnnotation(
+        span_id=str(span_id),
+        obj=models.SpanAnnotation(**kwargs),
+    )
+
+
+def _trace_annotation_factory(trace_id: str) -> Callable[..., Precursors.TraceAnnotation]:
+    return lambda **kwargs: Precursors.TraceAnnotation(
+        trace_id=str(trace_id),
+        obj=models.TraceAnnotation(**kwargs),
+    )
+
+
+def _read_sql_trace_evaluations_into_dataframe(
+    connectable: Connectable,
+    project_name: str,
+) -> DataFrame:
+    """
+    Reads a project's trace evaluations into a pandas dataframe.
+
+    Inputs a synchronous connectable to pandas.read_sql since it does not
+    support async connectables. For more information, see:
+
+    https://stackoverflow.com/questions/70848256/how-can-i-use-pandas-read-sql-on-an-async-connection
+    """
+    return pd.read_sql(
+        select(models.TraceAnnotation, models.Trace.trace_id)
+        .join_from(models.TraceAnnotation, models.Trace)
+        .join_from(models.Trace, models.Project)
+        .where(models.Project.name == project_name)
+        .where(models.TraceAnnotation.annotator_kind == "LLM"),
+        connectable,
+        index_col="trace_id",
+    )
+
+
+def _read_sql_span_evaluations_into_dataframe(
+    connectable: Connectable,
+    project_name: str,
+) -> DataFrame:
+    """
+    Reads a project's span evaluations into a pandas dataframe.
+
+    Inputs a synchronous connectable to pandas.read_sql since it does not
+    support async connectables. For more information, see:
+
+    https://stackoverflow.com/questions/70848256/how-can-i-use-pandas-read-sql-on-an-async-connection
+    """
+    return pd.read_sql_query(
+        select(models.SpanAnnotation, models.Span.span_id)
+        .join_from(models.SpanAnnotation, models.Span)
+        .join_from(models.Span, models.Trace)
+        .join_from(models.Trace, models.Project)
+        .where(models.Project.name == project_name)
+        .where(models.SpanAnnotation.annotator_kind == "LLM"),
+        connectable,
+        index_col="span_id",
+    )
+
+
+def _read_sql_document_evaluations_into_dataframe(
+    connectable: Connectable,
+    project_name: str,
+) -> DataFrame:
+    """
+    Reads a project's document evaluations into a pandas dataframe.
+
+    Inputs a synchronous connectable to pandas.read_sql since it does not
+    support async connectables. For more information, see:
+
+    https://stackoverflow.com/questions/70848256/how-can-i-use-pandas-read-sql-on-an-async-connection
+    """
+    return pd.read_sql(
+        select(models.DocumentAnnotation, models.Span.span_id)
+        .join_from(models.DocumentAnnotation, models.Span)
+        .join_from(models.Span, models.Trace)
+        .join_from(models.Trace, models.Project)
+        .where(models.Project.name == project_name)
+        .where(models.DocumentAnnotation.annotator_kind == "LLM"),
+        connectable,
+    ).set_index(["span_id", "document_position"])
+
+
+def _groupby_eval_name(
+    evals_dataframe: DataFrame,
+) -> Iterator[tuple[EvaluationName, DataFrame]]:
+    for eval_name, evals_dataframe_for_name in evals_dataframe.groupby("name", as_index=False):
+        yield str(eval_name), evals_dataframe_for_name

phoenix/server/api/routers/v1/experiment_evaluations.py

@@ -0,0 +1,115 @@
+from datetime import datetime
+from typing import Any, Literal, Optional
+
+from fastapi import APIRouter, HTTPException
+from pydantic import Field
+from starlette.requests import Request
+from starlette.status import HTTP_404_NOT_FOUND
+from strawberry.relay import GlobalID
+
+from phoenix.db import models
+from phoenix.db.helpers import SupportedSQLDialect
+from phoenix.db.insertion.helpers import insert_on_conflict
+from phoenix.server.api.types.node import from_global_id_with_expected_type
+from phoenix.server.dml_event import ExperimentRunAnnotationInsertEvent
+
+from .pydantic_compat import V1RoutesBaseModel
+from .utils import ResponseBody, add_errors_to_responses
+
+router = APIRouter(tags=["experiments"], include_in_schema=False)
+
+
+class ExperimentEvaluationResult(V1RoutesBaseModel):
+    label: Optional[str] = Field(default=None, description="The label assigned by the evaluation")
+    score: Optional[float] = Field(default=None, description="The score assigned by the evaluation")
+    explanation: Optional[str] = Field(
+        default=None, description="Explanation of the evaluation result"
+    )
+
+
+class UpsertExperimentEvaluationRequestBody(V1RoutesBaseModel):
+    experiment_run_id: str = Field(description="The ID of the experiment run being evaluated")
+    name: str = Field(description="The name of the evaluation")
+    annotator_kind: Literal["LLM", "CODE", "HUMAN"] = Field(
+        description="The kind of annotator used for the evaluation"
+    )
+    start_time: datetime = Field(description="The start time of the evaluation in ISO format")
+    end_time: datetime = Field(description="The end time of the evaluation in ISO format")
+    result: ExperimentEvaluationResult = Field(description="The result of the evaluation")
+    error: Optional[str] = Field(
+        None, description="Optional error message if the evaluation encountered an error"
+    )
+    metadata: Optional[dict[str, Any]] = Field(
+        default=None, description="Metadata for the evaluation"
+    )
+    trace_id: Optional[str] = Field(default=None, description="Optional trace ID for tracking")
+
+
+class UpsertExperimentEvaluationResponseBodyData(V1RoutesBaseModel):
+    id: str = Field(description="The ID of the upserted experiment evaluation")
+
+
+class UpsertExperimentEvaluationResponseBody(
+    ResponseBody[UpsertExperimentEvaluationResponseBodyData]
+):
+    pass
+
+
+@router.post(
+    "/experiment_evaluations",
+    operation_id="upsertExperimentEvaluation",
+    summary="Create or update evaluation for an experiment run",
+    responses=add_errors_to_responses(
+        [{"status_code": HTTP_404_NOT_FOUND, "description": "Experiment run not found"}]
+    ),
+)
+async def upsert_experiment_evaluation(
+    request: Request, request_body: UpsertExperimentEvaluationRequestBody
+) -> UpsertExperimentEvaluationResponseBody:
+    payload = await request.json()
+    experiment_run_gid = GlobalID.from_id(payload["experiment_run_id"])
+    try:
+        experiment_run_id = from_global_id_with_expected_type(experiment_run_gid, "ExperimentRun")
+    except ValueError:
+        raise HTTPException(
+            detail=f"ExperimentRun with ID {experiment_run_gid} does not exist",
+            status_code=HTTP_404_NOT_FOUND,
+        )
+    name = request_body.name
+    annotator_kind = request_body.annotator_kind
+    result = request_body.result
+    label = result.label if result else None
+    score = result.score if result else None
+    explanation = result.explanation if result else None
+    error = request_body.error
+    metadata = request_body.metadata or {}
+    start_time = payload["start_time"]
+    end_time = payload["end_time"]
+    async with request.app.state.db() as session:
+        values = dict(
+            experiment_run_id=experiment_run_id,
+            name=name,
+            annotator_kind=annotator_kind,
+            label=label,
+            score=score,
+            explanation=explanation,
+            error=error,
+            metadata_=metadata,  # `metadata_` must match database
+            start_time=datetime.fromisoformat(start_time),
+            end_time=datetime.fromisoformat(end_time),
+            trace_id=payload.get("trace_id"),
+        )
+        dialect = SupportedSQLDialect(session.bind.dialect.name)
+        exp_eval_run = await session.scalar(
+            insert_on_conflict(
+                values,
+                dialect=dialect,
+                table=models.ExperimentRunAnnotation,
+                unique_by=("experiment_run_id", "name"),
+            ).returning(models.ExperimentRunAnnotation)
+        )
+    evaluation_gid = GlobalID("ExperimentEvaluation", str(exp_eval_run.id))
+    request.state.event_queue.put(ExperimentRunAnnotationInsertEvent((exp_eval_run.id,)))
+    return UpsertExperimentEvaluationResponseBody(
+        data=UpsertExperimentEvaluationResponseBodyData(id=str(evaluation_gid))
+    )

phoenix/server/api/routers/v1/experiment_runs.py

@@ -0,0 +1,167 @@
+from datetime import datetime
+from typing import Any, Optional
+
+from fastapi import APIRouter, HTTPException
+from pydantic import Field
+from sqlalchemy import select
+from starlette.requests import Request
+from starlette.status import HTTP_404_NOT_FOUND
+from strawberry.relay import GlobalID
+
+from phoenix.db import models
+from phoenix.db.models import ExperimentRunOutput
+from phoenix.server.api.types.node import from_global_id_with_expected_type
+from phoenix.server.dml_event import ExperimentRunInsertEvent
+
+from .pydantic_compat import V1RoutesBaseModel
+from .utils import ResponseBody, add_errors_to_responses
+
+router = APIRouter(tags=["experiments"], include_in_schema=False)
+
+
+class ExperimentRun(V1RoutesBaseModel):
+    dataset_example_id: str = Field(
+        description="The ID of the dataset example used in the experiment run"
+    )
+    output: Any = Field(description="The output of the experiment task")
+    repetition_number: int = Field(description="The repetition number of the experiment run")
+    start_time: datetime = Field(description="The start time of the experiment run")
+    end_time: datetime = Field(description="The end time of the experiment run")
+    trace_id: Optional[str] = Field(
+        default=None, description="The ID of the corresponding trace (if one exists)"
+    )
+    error: Optional[str] = Field(
+        default=None,
+        description="Optional error message if the experiment run encountered an error",
+    )
+
+
+class CreateExperimentRunRequestBody(ExperimentRun):
+    pass
+
+
+class CreateExperimentRunResponseBodyData(V1RoutesBaseModel):
+    id: str = Field(description="The ID of the newly created experiment run")
+
+
+class CreateExperimentResponseBody(ResponseBody[CreateExperimentRunResponseBodyData]):
+    pass
+
+
+@router.post(
+    "/experiments/{experiment_id}/runs",
+    operation_id="createExperimentRun",
+    summary="Create run for an experiment",
+    response_description="Experiment run created successfully",
+    responses=add_errors_to_responses(
+        [
+            {
+                "status_code": HTTP_404_NOT_FOUND,
+                "description": "Experiment or dataset example not found",
+            }
+        ]
+    ),
+)
+async def create_experiment_run(
+    request: Request, experiment_id: str, request_body: CreateExperimentRunRequestBody
+) -> CreateExperimentResponseBody:
+    experiment_gid = GlobalID.from_id(experiment_id)
+    try:
+        experiment_rowid = from_global_id_with_expected_type(experiment_gid, "Experiment")
+    except ValueError:
+        raise HTTPException(
+            detail=f"Experiment with ID {experiment_gid} does not exist",
+            status_code=HTTP_404_NOT_FOUND,
+        )
+
+    example_gid = GlobalID.from_id(request_body.dataset_example_id)
+    try:
+        dataset_example_id = from_global_id_with_expected_type(example_gid, "DatasetExample")
+    except ValueError:
+        raise HTTPException(
+            detail=f"DatasetExample with ID {example_gid} does not exist",
+            status_code=HTTP_404_NOT_FOUND,
+        )
+
+    trace_id = request_body.trace_id
+    task_output = request_body.output
+    repetition_number = request_body.repetition_number
+    start_time = request_body.start_time
+    end_time = request_body.end_time
+    error = request_body.error
+
+    async with request.app.state.db() as session:
+        exp_run = models.ExperimentRun(
+            experiment_id=experiment_rowid,
+            dataset_example_id=dataset_example_id,
+            trace_id=trace_id,
+            output=ExperimentRunOutput(task_output=task_output),
+            repetition_number=repetition_number,
+            start_time=start_time,
+            end_time=end_time,
+            error=error,
+        )
+        session.add(exp_run)
+        await session.flush()
+    request.state.event_queue.put(ExperimentRunInsertEvent((exp_run.id,)))
+    run_gid = GlobalID("ExperimentRun", str(exp_run.id))
+    return CreateExperimentResponseBody(data=CreateExperimentRunResponseBodyData(id=str(run_gid)))
+
+
+class ExperimentRunResponse(ExperimentRun):
+    id: str = Field(description="The ID of the experiment run")
+    experiment_id: str = Field(description="The ID of the experiment")
+
+
+class ListExperimentRunsResponseBody(ResponseBody[list[ExperimentRunResponse]]):
+    pass
+
+
+@router.get(
+    "/experiments/{experiment_id}/runs",
+    operation_id="listExperimentRuns",
+    summary="List runs for an experiment",
+    response_description="Experiment runs retrieved successfully",
+    responses=add_errors_to_responses(
+        [{"status_code": HTTP_404_NOT_FOUND, "description": "Experiment not found"}]
+    ),
+)
+async def list_experiment_runs(
+    request: Request, experiment_id: str
+) -> ListExperimentRunsResponseBody:
+    experiment_gid = GlobalID.from_id(experiment_id)
+    try:
+        experiment_rowid = from_global_id_with_expected_type(experiment_gid, "Experiment")
+    except ValueError:
+        raise HTTPException(
+            detail=f"Experiment with ID {experiment_gid} does not exist",
+            status_code=HTTP_404_NOT_FOUND,
+        )
+
+    async with request.app.state.db() as session:
+        experiment_runs = await session.execute(
+            select(models.ExperimentRun)
+            .where(models.ExperimentRun.experiment_id == experiment_rowid)
+            # order by dataset_example_id to be consistent with `list_dataset_examples`
+            .order_by(models.ExperimentRun.dataset_example_id.asc())
+        )
+        experiment_runs = experiment_runs.scalars().all()
+        runs = []
+        for exp_run in experiment_runs:
+            run_gid = GlobalID("ExperimentRun", str(exp_run.id))
+            experiment_gid = GlobalID("Experiment", str(exp_run.experiment_id))
+            example_gid = GlobalID("DatasetExample", str(exp_run.dataset_example_id))
+            runs.append(
+                ExperimentRunResponse(
+                    start_time=exp_run.start_time,
+                    end_time=exp_run.end_time,
+                    experiment_id=str(experiment_gid),
+                    dataset_example_id=str(example_gid),
+                    repetition_number=exp_run.repetition_number,
+                    output=exp_run.output.get("task_output"),
+                    error=exp_run.error,
+                    id=str(run_gid),
+                    trace_id=exp_run.trace_id,
+                )
+            )
+    return ListExperimentRunsResponseBody(data=runs)