arize-phoenix 4.4.2__py3-none-any.whl → 4.4.4rc0__py3-none-any.whl

phoenix/server/api/routers/v1/evaluations.py

@@ -44,7 +44,7 @@ async def post_evaluations(request: Request) -> Response:
     summary: Add evaluations to a span, trace, or document
     operationId: addEvaluations
     tags:
-      - evaluations
+      - private
     parameters:
       - name: project-name
         in: query
@@ -105,7 +105,7 @@ async def get_evaluations(request: Request) -> Response:
     summary: Get evaluations from Phoenix
     operationId: getEvaluation
     tags:
-      - evaluations
+      - private
     parameters:
       - name: project-name
         in: query
@@ -116,6 +116,8 @@ async def get_evaluations(request: Request) -> Response:
     responses:
       200:
         description: Success
+      403:
+        description: Forbidden
       404:
         description: Not found
     """

phoenix/server/api/routers/v1/experiment_evaluations.py

@@ -0,0 +1,65 @@
+from datetime import datetime
+
+from starlette.requests import Request
+from starlette.responses import JSONResponse, Response
+from starlette.status import HTTP_404_NOT_FOUND
+from strawberry.relay import GlobalID
+
+from phoenix.datasets.types import EvaluationResult, ExperimentEvaluationRun
+from phoenix.db import models
+from phoenix.server.api.types.node import from_global_id_with_expected_type
+from phoenix.utilities.json import jsonify
+
+
+async def create_experiment_evaluation(request: Request) -> Response:
+    payload = await request.json()
+    experiment_run_gid = GlobalID.from_id(payload["experiment_run_id"])
+    try:
+        experiment_run_id = from_global_id_with_expected_type(experiment_run_gid, "ExperimentRun")
+    except ValueError:
+        return Response(
+            content=f"ExperimentRun with ID {experiment_run_gid} does not exist",
+            status_code=HTTP_404_NOT_FOUND,
+        )
+    name = payload["name"]
+    annotator_kind = payload["annotator_kind"]
+    result = payload.get("result")
+    label = result.get("label") if result else None
+    score = result.get("score") if result else None
+    explanation = result.get("explanation") if result else None
+    error = payload.get("error")
+    metadata = payload.get("metadata") or {}
+    start_time = payload["start_time"]
+    end_time = payload["end_time"]
+    async with request.app.state.db() as session:
+        exp_eval_run = models.ExperimentRunAnnotation(
+            experiment_run_id=experiment_run_id,
+            name=name,
+            annotator_kind=annotator_kind,
+            label=label,
+            score=score,
+            explanation=explanation,
+            error=error,
+            metadata_=metadata,
+            start_time=datetime.fromisoformat(start_time),
+            end_time=datetime.fromisoformat(end_time),
+        )
+        session.add(exp_eval_run)
+        await session.flush()
+        evaluation_gid = GlobalID("ExperimentEvaluation", str(exp_eval_run.id))
+        eval_payload = ExperimentEvaluationRun(
+            id=str(evaluation_gid),
+            experiment_run_id=str(experiment_run_gid),
+            start_time=exp_eval_run.start_time,
+            end_time=exp_eval_run.end_time,
+            name=exp_eval_run.name,
+            annotator_kind=exp_eval_run.annotator_kind,
+            error=exp_eval_run.error,
+            result=EvaluationResult(
+                label=exp_eval_run.label,
+                score=exp_eval_run.score,
+                explanation=exp_eval_run.explanation,
+                metadata=exp_eval_run.metadata_,
+            ),
+        )
+        return JSONResponse(content=jsonify(eval_payload), status_code=200)

phoenix/server/api/routers/v1/experiment_runs.py

@@ -0,0 +1,108 @@
+from datetime import datetime
+
+from sqlalchemy import select
+from starlette.requests import Request
+from starlette.responses import JSONResponse, Response
+from starlette.status import HTTP_404_NOT_FOUND
+from strawberry.relay import GlobalID
+
+from phoenix.datasets.types import ExperimentResult, ExperimentRun
+from phoenix.db import models
+from phoenix.server.api.types.node import from_global_id_with_expected_type
+from phoenix.utilities.json import jsonify
+
+
+async def create_experiment_run(request: Request) -> Response:
+    experiment_gid = GlobalID.from_id(request.path_params["experiment_id"])
+    try:
+        experiment_id = from_global_id_with_expected_type(experiment_gid, "Experiment")
+    except ValueError:
+        return Response(
+            content=f"Experiment with ID {experiment_gid} does not exist",
+            status_code=HTTP_404_NOT_FOUND,
+        )
+
+    payload = await request.json()
+
+    example_gid = GlobalID.from_id(payload["dataset_example_id"])
+    try:
+        dataset_example_id = from_global_id_with_expected_type(example_gid, "DatasetExample")
+    except ValueError:
+        return Response(
+            content=f"DatasetExample with ID {example_gid} does not exist",
+            status_code=HTTP_404_NOT_FOUND,
+        )
+
+    trace_id = payload.get("trace_id", None)
+    output = payload["output"]
+    repetition_number = payload["repetition_number"]
+    start_time = payload["start_time"]
+    end_time = payload["end_time"]
+    error = payload.get("error")
+
+    async with request.app.state.db() as session:
+        exp_run = models.ExperimentRun(
+            experiment_id=experiment_id,
+            dataset_example_id=dataset_example_id,
+            trace_id=trace_id,
+            output=output,
+            repetition_number=repetition_number,
+            start_time=datetime.fromisoformat(start_time),
+            end_time=datetime.fromisoformat(end_time),
+            error=error,
+        )
+        session.add(exp_run)
+        await session.flush()
+
+        run_gid = GlobalID("ExperimentRun", str(exp_run.id))
+        run_payload = ExperimentRun(
+            start_time=exp_run.start_time,
+            end_time=exp_run.end_time,
+            experiment_id=str(experiment_gid),
+            dataset_example_id=str(example_gid),
+            repetition_number=exp_run.repetition_number,
+            output=ExperimentResult(result=exp_run.output),
+            error=exp_run.error,
+            id=str(run_gid),
+            trace_id=exp_run.trace_id,
+        )
+        return JSONResponse(content=jsonify(run_payload), status_code=200)
+
+
+async def list_experiment_runs(request: Request) -> Response:
+    experiment_gid = GlobalID.from_id(request.path_params["experiment_id"])
+    try:
+        experiment_id = from_global_id_with_expected_type(experiment_gid, "Experiment")
+    except ValueError:
+        return Response(
+            content=f"Experiment with ID {experiment_gid} does not exist",
+            status_code=HTTP_404_NOT_FOUND,
+        )
+
+    async with request.app.state.db() as session:
+        experiment_runs = await session.execute(
+            select(models.ExperimentRun)
+            .where(models.ExperimentRun.experiment_id == experiment_id)
+            # order by dataset_example_id to be consistent with `list_dataset_examples`
+            .order_by(models.ExperimentRun.dataset_example_id.asc())
+        )
+        experiment_runs = experiment_runs.scalars().all()
+        runs = []
+        for exp_run in experiment_runs:
+            run_gid = GlobalID("ExperimentRun", str(exp_run.id))
+            experiment_gid = GlobalID("Experiment", str(exp_run.experiment_id))
+            example_gid = GlobalID("DatasetExample", str(exp_run.dataset_example_id))
+            runs.append(
+                ExperimentRun(
+                    start_time=exp_run.start_time,
+                    end_time=exp_run.end_time,
+                    experiment_id=str(experiment_gid),
+                    dataset_example_id=str(example_gid),
+                    repetition_number=exp_run.repetition_number,
+                    output=ExperimentResult(result=exp_run.output),
+                    error=exp_run.error,
+                    id=str(run_gid),
+                    trace_id=exp_run.trace_id,
+                )
+            )
+        return JSONResponse(content=jsonify(runs), status_code=200)

phoenix/server/api/routers/v1/experiments.py

@@ -0,0 +1,174 @@
+from random import getrandbits
+
+from sqlalchemy import select
+from starlette.requests import Request
+from starlette.responses import JSONResponse, Response
+from starlette.status import HTTP_404_NOT_FOUND
+from strawberry.relay import GlobalID
+
+from phoenix.db import models
+from phoenix.db.helpers import SupportedSQLDialect
+from phoenix.db.insertion.helpers import insert_stmt
+from phoenix.server.api.types.node import from_global_id_with_expected_type
+
+
+def _short_uuid() -> str:
+    return str(getrandbits(32).to_bytes(4, "big").hex())
+
+
+def _generate_experiment_name(dataset_name: str) -> str:
+    """
+    Generate a semi-unique name for the experiment.
+    """
+    short_ds_name = dataset_name[:8].replace(" ", "-")
+    return f"{short_ds_name}-{_short_uuid()}"
+
+
+async def create_experiment(request: Request) -> Response:
+    dataset_globalid = GlobalID.from_id(request.path_params["dataset_id"])
+    try:
+        dataset_id = from_global_id_with_expected_type(dataset_globalid, "Dataset")
+    except ValueError:
+        return Response(
+            content="Dataset with ID {dataset_globalid} does not exist",
+            status_code=HTTP_404_NOT_FOUND,
+        )
+
+    payload = await request.json()
+    repetitions = payload.get("repetitions", 1)
+    metadata = payload.get("metadata") or {}
+    dataset_version_globalid_str = payload.get("version-id")
+    if dataset_version_globalid_str is not None:
+        try:
+            dataset_version_globalid = GlobalID.from_id(dataset_version_globalid_str)
+            dataset_version_id = from_global_id_with_expected_type(
+                dataset_version_globalid, "DatasetVersion"
+            )
+        except ValueError:
+            return Response(
+                content="DatasetVersion with ID {dataset_version_globalid} does not exist",
+                status_code=HTTP_404_NOT_FOUND,
+            )
+
+    async with request.app.state.db() as session:
+        result = (
+            await session.execute(select(models.Dataset).where(models.Dataset.id == dataset_id))
+        ).scalar()
+        if result is None:
+            return Response(
+                content=f"Dataset with ID {dataset_globalid} does not exist",
+                status_code=HTTP_404_NOT_FOUND,
+            )
+        dataset_name = result.name
+        if dataset_version_globalid_str is None:
+            dataset_version_result = await session.execute(
+                select(models.DatasetVersion)
+                .where(models.DatasetVersion.dataset_id == dataset_id)
+                .order_by(models.DatasetVersion.id.desc())
+            )
+            dataset_version = dataset_version_result.scalar()
+            if not dataset_version:
+                return Response(
+                    content=f"Dataset {dataset_globalid} does not have any versions",
+                    status_code=HTTP_404_NOT_FOUND,
+                )
+            dataset_version_id = dataset_version.id
+            dataset_version_globalid = GlobalID("DatasetVersion", str(dataset_version_id))
+        else:
+            dataset_version = await session.execute(
+                select(models.DatasetVersion).where(models.DatasetVersion.id == dataset_version_id)
+            )
+            dataset_version = dataset_version.scalar()
+            if not dataset_version:
+                return Response(
+                    content=f"DatasetVersion with ID {dataset_version_globalid} does not exist",
+                    status_code=HTTP_404_NOT_FOUND,
+                )
+
+        # generate a semi-unique name for the experiment
+        experiment_name = payload.get("name") or _generate_experiment_name(dataset_name)
+        project_name = f"Experiment-{getrandbits(96).to_bytes(12, 'big').hex()}"
+        project_description = (
+            f"dataset_id: {dataset_globalid}\ndataset_version_id: {dataset_version_globalid}"
+        )
+        experiment = models.Experiment(
+            dataset_id=int(dataset_id),
+            dataset_version_id=int(dataset_version_id),
+            name=experiment_name,
+            description=payload.get("description"),
+            repetitions=repetitions,
+            metadata_=metadata,
+            project_name=project_name,
+        )
+        session.add(experiment)
+        await session.flush()
+
+        dialect = SupportedSQLDialect(session.bind.dialect.name)
+        project_rowid = await session.scalar(
+            insert_stmt(
+                dialect=dialect,
+                table=models.Project,
+                constraint="uq_projects_name",
+                column_names=("name",),
+                values=dict(
+                    name=project_name,
+                    description=project_description,
+                    created_at=experiment.created_at,
+                    updated_at=experiment.updated_at,
+                ),
+            ).returning(models.Project.id)
+        )
+        assert project_rowid is not None
+
+        experiment_globalid = GlobalID("Experiment", str(experiment.id))
+        if dataset_version_globalid_str is None:
+            dataset_version_globalid = GlobalID(
+                "DatasetVersion", str(experiment.dataset_version_id)
+            )
+        experiment_payload = {
+            "id": str(experiment_globalid),
+            "dataset_id": str(dataset_globalid),
+            "dataset_version_id": str(dataset_version_globalid),
+            "repetitions": experiment.repetitions,
+            "metadata": experiment.metadata_,
+            "project_name": experiment.project_name,
+            "created_at": experiment.created_at.isoformat(),
+            "updated_at": experiment.updated_at.isoformat(),
+        }
+        return JSONResponse(content=experiment_payload, status_code=200)
+
+
+async def read_experiment(request: Request) -> Response:
+    experiment_globalid = GlobalID.from_id(request.path_params["experiment_id"])
+    try:
+        experiment_id = from_global_id_with_expected_type(experiment_globalid, "Experiment")
+    except ValueError:
+        return Response(
+            content="Experiment with ID {experiment_globalid} does not exist",
+            status_code=HTTP_404_NOT_FOUND,
+        )
+
+    async with request.app.state.db() as session:
+        experiment = await session.execute(
+            select(models.Experiment).where(models.Experiment.id == experiment_id)
+        )
+        experiment = experiment.scalar()
+        if not experiment:
+            return Response(
+                content=f"Experiment with ID {experiment_globalid} does not exist",
+                status_code=HTTP_404_NOT_FOUND,
+            )
+
+        dataset_globalid = GlobalID("Dataset", str(experiment.dataset_id))
+        dataset_version_globalid = GlobalID("DatasetVersion", str(experiment.dataset_version_id))
+        experiment_payload = {
+            "id": str(experiment_globalid),
+            "dataset_id": str(dataset_globalid),
+            "dataset_version_id": str(dataset_version_globalid),
+            "repetitions": experiment.repetitions,
+            "metadata": experiment.metadata_,
+            "project_name": experiment.project_name,
+            "created_at": experiment.created_at.isoformat(),
+            "updated_at": experiment.updated_at.isoformat(),
+        }
+        return JSONResponse(content=experiment_payload, status_code=200)

phoenix/server/api/routers/v1/spans.py

@@ -19,7 +19,7 @@ async def query_spans_handler(request: Request) -> Response:
     summary: Query spans using query DSL
     operationId: querySpans
     tags:
-      - spans
+      - private
     parameters:
       - name: project-name
         in: query
@@ -68,6 +68,8 @@ async def query_spans_handler(request: Request) -> Response:
     responses:
       200:
         description: Success
+      403:
+        description: Forbidden
       404:
         description: Not found
       422:

phoenix/server/api/routers/v1/traces.py

@@ -11,7 +11,6 @@ from starlette.datastructures import State
 from starlette.requests import Request
 from starlette.responses import Response
 from starlette.status import (
-    HTTP_403_FORBIDDEN,
     HTTP_415_UNSUPPORTED_MEDIA_TYPE,
     HTTP_422_UNPROCESSABLE_ENTITY,
 )
@@ -25,7 +24,7 @@ async def post_traces(request: Request) -> Response:
     summary: Send traces to Phoenix
     operationId: addTraces
     tags:
-      - traces
+      - private
     requestBody:
       required: true
       content:
@@ -43,8 +42,6 @@ async def post_traces(request: Request) -> Response:
       422:
         description: Request body is invalid
     """
-    if request.app.state.read_only:
-        return Response(status_code=HTTP_403_FORBIDDEN)
     content_type = request.headers.get("content-type")
     if content_type != "application/x-protobuf":
         return Response(