arize-phoenix 4.5.0__py3-none-any.whl → 4.6.2__py3-none-any.whl

phoenix/server/api/routers/v1/evaluations.py

@@ -44,14 +44,7 @@ async def post_evaluations(request: Request) -> Response:
     summary: Add evaluations to a span, trace, or document
     operationId: addEvaluations
     tags:
-      - evaluations
-    parameters:
-      - name: project-name
-        in: query
-        schema:
-          type: string
-          default: default
-        description: The project name to add the evaluation to
+      - private
     requestBody:
       required: true
       content:
@@ -105,9 +98,9 @@ async def get_evaluations(request: Request) -> Response:
     summary: Get evaluations from Phoenix
     operationId: getEvaluation
     tags:
-      - evaluations
+      - private
     parameters:
-      - name: project-name
+      - name: project_name
         in: query
         schema:
           type: string
@@ -116,13 +109,15 @@ async def get_evaluations(request: Request) -> Response:
     responses:
       200:
         description: Success
+      403:
+        description: Forbidden
       404:
         description: Not found
     """
     project_name = (
-        request.query_params.get("project-name")
-        # read from headers for backwards compatibility
-        or request.headers.get("project-name")
+        request.query_params.get("project_name")
+        or request.query_params.get("project-name")  # for backward compatibility
+        or request.headers.get("project-name")  # read from headers for backwards compatibility
         or DEFAULT_PROJECT_NAME
     )
 

phoenix/server/api/routers/v1/experiment_evaluations.py

@@ -0,0 +1,143 @@
+from datetime import datetime
+
+from starlette.requests import Request
+from starlette.responses import JSONResponse, Response
+from starlette.status import HTTP_404_NOT_FOUND
+from strawberry.relay import GlobalID
+
+from phoenix.db import models
+from phoenix.db.helpers import SupportedSQLDialect
+from phoenix.db.insertion.helpers import OnConflict, insert_on_conflict
+from phoenix.server.api.types.node import from_global_id_with_expected_type
+
+
+async def upsert_experiment_evaluation(request: Request) -> Response:
+    """
+    summary: Create an evaluation for a specific experiment run
+    operationId: upsertExperimentEvaluation
+    tags:
+      - private
+    requestBody:
+      description: Details of the experiment evaluation to be upserted
+      required: true
+      content:
+        application/json:
+          schema:
+            type: object
+            properties:
+              experiment_run_id:
+                type: string
+                description: The ID of the experiment run being evaluated
+              name:
+                type: string
+                description: The name of the evaluation
+              annotator_kind:
+                type: string
+                description: The kind of annotator used for the evaluation
+              result:
+                type: object
+                description: The result of the evaluation
+                properties:
+                  label:
+                    type: string
+                    description: The label assigned by the evaluation
+                  score:
+                    type: number
+                    format: float
+                    description: The score assigned by the evaluation
+                  explanation:
+                    type: string
+                    description: Explanation of the evaluation result
+              error:
+                type: string
+                description: Optional error message if the evaluation encountered an error
+              metadata:
+                type: object
+                description: Metadata for the evaluation
+                additionalProperties:
+                  type: string
+              start_time:
+                type: string
+                format: date-time
+                description: The start time of the evaluation in ISO format
+              end_time:
+                type: string
+                format: date-time
+                description: The end time of the evaluation in ISO format
+              trace_id:
+                type: string
+                description: Optional trace ID for tracking
+            required:
+              - experiment_run_id
+              - name
+              - annotator_kind
+              - start_time
+              - end_time
+    responses:
+      200:
+        description: Experiment evaluation upserted successfully
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                data:
+                  type: object
+                  properties:
+                    id:
+                      type: string
+                      description: The ID of the upserted experiment evaluation
+      404:
+        description: ExperimentRun not found
+    """
+    payload = await request.json()
+    experiment_run_gid = GlobalID.from_id(payload["experiment_run_id"])
+    try:
+        experiment_run_id = from_global_id_with_expected_type(experiment_run_gid, "ExperimentRun")
+    except ValueError:
+        return Response(
+            content=f"ExperimentRun with ID {experiment_run_gid} does not exist",
+            status_code=HTTP_404_NOT_FOUND,
+        )
+    name = payload["name"]
+    annotator_kind = payload["annotator_kind"]
+    result = payload.get("result")
+    label = result.get("label") if result else None
+    score = result.get("score") if result else None
+    explanation = result.get("explanation") if result else None
+    error = payload.get("error")
+    metadata = payload.get("metadata") or {}
+    start_time = payload["start_time"]
+    end_time = payload["end_time"]
+    async with request.app.state.db() as session:
+        values = dict(
+            experiment_run_id=experiment_run_id,
+            name=name,
+            annotator_kind=annotator_kind,
+            label=label,
+            score=score,
+            explanation=explanation,
+            error=error,
+            metadata_=metadata,  # `metadata_` must match database
+            start_time=datetime.fromisoformat(start_time),
+            end_time=datetime.fromisoformat(end_time),
+            trace_id=payload.get("trace_id"),
+        )
+        set_ = {
+            **{k: v for k, v in values.items() if k != "metadata_"},
+            "metadata": values["metadata_"],  # `metadata` must match database
+        }
+        dialect = SupportedSQLDialect(session.bind.dialect.name)
+        exp_eval_run = await session.scalar(
+            insert_on_conflict(
+                dialect=dialect,
+                table=models.ExperimentRunAnnotation,
+                values=values,
+                constraint="uq_experiment_run_annotations_experiment_run_id_name",
+                column_names=("experiment_run_id", "name"),
+                on_conflict=OnConflict.DO_UPDATE,
+                set_=set_,
+            ).returning(models.ExperimentRunAnnotation)
+        )
+    evaluation_gid = GlobalID("ExperimentEvaluation", str(exp_eval_run.id))
+    return JSONResponse(content={"data": {"id": str(evaluation_gid)}})

phoenix/server/api/routers/v1/experiment_runs.py

@@ -0,0 +1,220 @@
+from datetime import datetime
+
+from sqlalchemy import select
+from starlette.requests import Request
+from starlette.responses import JSONResponse, Response
+from starlette.status import HTTP_404_NOT_FOUND
+from strawberry.relay import GlobalID
+
+from phoenix.db import models
+from phoenix.experiments.types import ExperimentRun, ExperimentRunOutput
+from phoenix.server.api.types.node import from_global_id_with_expected_type
+from phoenix.utilities.json import jsonify
+
+
+async def create_experiment_run(request: Request) -> Response:
+    """
+    summary: Create a new experiment run for a specific experiment
+    operationId: createExperimentRun
+    tags:
+      - private
+    parameters:
+      - in: path
+        name: experiment_id
+        required: true
+        description: The ID of the experiment for which the run is being created
+        schema:
+          type: string
+    requestBody:
+      description: Details of the experiment run to be created
+      required: true
+      content:
+        application/json:
+          schema:
+            type: object
+            properties:
+              dataset_example_id:
+                type: string
+                description: The ID of the dataset example used in the experiment run
+              trace_id:
+                type: string
+                description: Optional trace ID for tracking
+              experiment_run_output:
+                type: object
+                description: The output of the experiment run
+              repetition_number:
+                type: integer
+                description: The repetition number of the experiment run
+              start_time:
+                type: string
+                format: date-time
+                description: The start time of the experiment run in ISO format
+              end_time:
+                type: string
+                format: date-time
+                description: The end time of the experiment run in ISO format
+              error:
+                type: string
+                description: Optional error message if the experiment run encountered an error
+                nullable: true
+            required:
+              - dataset_example_id
+              - output
+              - repetition_number
+              - start_time
+              - end_time
+    responses:
+      200:
+        description: Experiment run created successfully
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                data:
+                  type: object
+                  properties:
+                    id:
+                      type: string
+                      description: The ID of the created experiment run
+      404:
+        description: Experiment or DatasetExample not found
+    """
+    experiment_gid = GlobalID.from_id(request.path_params["experiment_id"])
+    try:
+        experiment_id = from_global_id_with_expected_type(experiment_gid, "Experiment")
+    except ValueError:
+        return Response(
+            content=f"Experiment with ID {experiment_gid} does not exist",
+            status_code=HTTP_404_NOT_FOUND,
+        )
+
+    payload = await request.json()
+
+    example_gid = GlobalID.from_id(payload["dataset_example_id"])
+    try:
+        dataset_example_id = from_global_id_with_expected_type(example_gid, "DatasetExample")
+    except ValueError:
+        return Response(
+            content=f"DatasetExample with ID {example_gid} does not exist",
+            status_code=HTTP_404_NOT_FOUND,
+        )
+
+    trace_id = payload.get("trace_id", None)
+    output = payload["experiment_run_output"]
+    repetition_number = payload["repetition_number"]
+    start_time = payload["start_time"]
+    end_time = payload["end_time"]
+    error = payload.get("error")
+
+    async with request.app.state.db() as session:
+        exp_run = models.ExperimentRun(
+            experiment_id=experiment_id,
+            dataset_example_id=dataset_example_id,
+            trace_id=trace_id,
+            output=output,
+            repetition_number=repetition_number,
+            start_time=datetime.fromisoformat(start_time),
+            end_time=datetime.fromisoformat(end_time),
+            error=error,
+        )
+        session.add(exp_run)
+        await session.flush()
+    run_gid = GlobalID("ExperimentRun", str(exp_run.id))
+    return JSONResponse(content={"data": {"id": str(run_gid)}})
+
+
+async def list_experiment_runs(request: Request) -> Response:
+    """
+    summary: List all runs for a specific experiment
+    operationId: listExperimentRuns
+    tags:
+      - private
+    parameters:
+      - in: path
+        name: experiment_id
+        required: true
+        description: The ID of the experiment to list runs for
+        schema:
+          type: string
+    responses:
+      200:
+        description: Experiment runs retrieved successfully
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                data:
+                  type: array
+                  items:
+                    type: object
+                    properties:
+                      id:
+                        type: string
+                        description: The ID of the experiment run
+                      experiment_id:
+                        type: string
+                        description: The ID of the experiment
+                      dataset_example_id:
+                        type: string
+                        description: The ID of the dataset example
+                      repetition_number:
+                        type: integer
+                        description: The repetition number of the experiment run
+                      start_time:
+                        type: string
+                        format: date-time
+                        description: The start time of the experiment run in ISO format
+                      end_time:
+                        type: string
+                        format: date-time
+                        description: The end time of the experiment run in ISO format
+                      experiment_run_output:
+                        type: object
+                        description: The output of the experiment run
+                      error:
+                        type: string
+                        description: Error message if the experiment run encountered an error
+                      trace_id:
+                        type: string
+                        description: Optional trace ID for tracking
+      404:
+        description: Experiment not found
+    """
+    experiment_gid = GlobalID.from_id(request.path_params["experiment_id"])
+    try:
+        experiment_id = from_global_id_with_expected_type(experiment_gid, "Experiment")
+    except ValueError:
+        return Response(
+            content=f"Experiment with ID {experiment_gid} does not exist",
+            status_code=HTTP_404_NOT_FOUND,
+        )
+
+    async with request.app.state.db() as session:
+        experiment_runs = await session.execute(
+            select(models.ExperimentRun)
+            .where(models.ExperimentRun.experiment_id == experiment_id)
+            # order by dataset_example_id to be consistent with `list_dataset_examples`
+            .order_by(models.ExperimentRun.dataset_example_id.asc())
+        )
+        experiment_runs = experiment_runs.scalars().all()
+        runs = []
+        for exp_run in experiment_runs:
+            run_gid = GlobalID("ExperimentRun", str(exp_run.id))
+            experiment_gid = GlobalID("Experiment", str(exp_run.experiment_id))
+            example_gid = GlobalID("DatasetExample", str(exp_run.dataset_example_id))
+            runs.append(
+                ExperimentRun(
+                    start_time=exp_run.start_time,
+                    end_time=exp_run.end_time,
+                    experiment_id=str(experiment_gid),
+                    dataset_example_id=str(example_gid),
+                    repetition_number=exp_run.repetition_number,
+                    experiment_run_output=ExperimentRunOutput.from_dict(exp_run.output),
+                    error=exp_run.error,
+                    id=str(run_gid),
+                    trace_id=exp_run.trace_id,
+                )
+            )
+    return JSONResponse(content={"data": jsonify(runs)}, status_code=200)