arize-phoenix 4.4.4rc5__py3-none-any.whl → 4.4.4rc6__py3-none-any.whl

phoenix/experiments/utils.py

@@ -0,0 +1,9 @@
+from phoenix.config import get_web_base_url
+
+
+def get_experiment_url(*, dataset_id: str, experiment_id: str) -> str:
+    return f"{get_web_base_url()}datasets/{dataset_id}/compare?experimentId={experiment_id}"
+
+
+def get_dataset_experiments_url(*, dataset_id: str) -> str:
+    return f"{get_web_base_url()}datasets/{dataset_id}/experiments"

phoenix/server/api/context.py

@@ -11,6 +11,7 @@ from typing_extensions import TypeAlias
 
 from phoenix.core.model_schema import Model
 from phoenix.server.api.dataloaders import (
+    AverageExperimentRunLatencyDataLoader,
     CacheForDataLoaders,
     DatasetExampleRevisionsDataLoader,
     DatasetExampleSpansDataLoader,
@@ -37,6 +38,7 @@ from phoenix.server.api.dataloaders import (
 
 @dataclass
 class DataLoaders:
+    average_experiment_run_latency: AverageExperimentRunLatencyDataLoader
     dataset_example_revisions: DatasetExampleRevisionsDataLoader
     dataset_example_spans: DatasetExampleSpansDataLoader
     document_evaluation_summaries: DocumentEvaluationSummaryDataLoader

phoenix/server/api/dataloaders/__init__.py

@@ -8,6 +8,7 @@ from phoenix.db.insertion.evaluation import (
 )
 from phoenix.db.insertion.span import ClearProjectSpansEvent, SpanInsertionEvent
 
+from .average_experiment_run_latency import AverageExperimentRunLatencyDataLoader
 from .dataset_example_revisions import DatasetExampleRevisionsDataLoader
 from .dataset_example_spans import DatasetExampleSpansDataLoader
 from .document_evaluation_summaries import (
@@ -34,6 +35,7 @@ from .trace_row_ids import TraceRowIdsDataLoader
 
 __all__ = [
     "CacheForDataLoaders",
+    "AverageExperimentRunLatencyDataLoader",
     "DatasetExampleRevisionsDataLoader",
     "DatasetExampleSpansDataLoader",
     "DocumentEvaluationSummaryDataLoader",

phoenix/server/api/dataloaders/average_experiment_run_latency.py

@@ -0,0 +1,54 @@
+from typing import (
+    AsyncContextManager,
+    Callable,
+    List,
+)
+
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+from strawberry.dataloader import DataLoader
+from typing_extensions import TypeAlias
+
+from phoenix.db import models
+
+ExperimentID: TypeAlias = int
+RunLatency: TypeAlias = float
+Key: TypeAlias = ExperimentID
+Result: TypeAlias = RunLatency
+
+
+class AverageExperimentRunLatencyDataLoader(DataLoader[Key, Result]):
+    def __init__(
+        self,
+        db: Callable[[], AsyncContextManager[AsyncSession]],
+    ) -> None:
+        super().__init__(load_fn=self._load_fn)
+        self._db = db
+
+    async def _load_fn(self, keys: List[Key]) -> List[Result]:
+        experiment_ids = keys
+        async with self._db() as session:
+            avg_latencies = {
+                experiment_id: avg_latency
+                async for experiment_id, avg_latency in await session.stream(
+                    select(
+                        models.ExperimentRun.experiment_id,
+                        func.avg(
+                            func.extract(
+                                "epoch",
+                                models.ExperimentRun.end_time,
+                            )
+                            - func.extract(
+                                "epoch",
+                                models.ExperimentRun.start_time,
+                            )
+                        ),
+                    )
+                    .where(models.ExperimentRun.experiment_id.in_(set(experiment_ids)))
+                    .group_by(models.ExperimentRun.experiment_id)
+                )
+            }
+        return [
+            avg_latencies.get(experiment_id, ValueError(f"Unknown experiment: {experiment_id}"))
+            for experiment_id in experiment_ids
+        ]

phoenix/server/api/routers/v1/__init__.py

@@ -80,7 +80,7 @@ V1_ROUTES = [
     ),
     Route(
         "/v1/experiment_evaluations",
-        experiment_evaluations.create_experiment_evaluation,
+        experiment_evaluations.upsert_experiment_evaluation,
         methods=["POST"],
     ),
 ]

phoenix/server/api/routers/v1/dataset_examples.py

@@ -21,7 +21,7 @@ async def list_dataset_examples(request: Request) -> Response:
           type: string
         description: Dataset ID
       - in: query
-        name: version-id
+        name: version_id
         schema:
           type: string
         description: Dataset version ID. If omitted, returns the latest version.
@@ -79,7 +79,7 @@ async def list_dataset_examples(request: Request) -> Response:
         description: Dataset does not exist.
     """
     dataset_id = GlobalID.from_id(request.path_params["id"])
-    raw_version_id = request.query_params.get("version-id")
+    raw_version_id = request.query_params.get("version_id")
     version_id = GlobalID.from_id(raw_version_id) if raw_version_id else None
 
     if (dataset_type := dataset_id.type_name) != "Dataset":
@@ -167,12 +167,12 @@ async def list_dataset_examples(request: Request) -> Response:
             }
             async for example, revision in await session.stream(query)
         ]
-        return JSONResponse(
-            {
-                "data": {
-                    "dataset_id": str(GlobalID("Dataset", str(resolved_dataset_id))),
-                    "version_id": str(GlobalID("DatasetVersion", str(resolved_version_id))),
-                    "examples": examples,
-                }
+    return JSONResponse(
+        {
+            "data": {
+                "dataset_id": str(GlobalID("Dataset", str(resolved_dataset_id))),
+                "version_id": str(GlobalID("DatasetVersion", str(resolved_version_id))),
+                "examples": examples,
             }
-        )
+        }
+    )

phoenix/server/api/routers/v1/datasets.py

@@ -233,7 +233,7 @@ async def get_dataset_by_id(request: Request) -> Response:
             "updated_at": dataset.updated_at.isoformat(),
             "example_count": example_count,
         }
-        return JSONResponse(content=output_dict)
+        return JSONResponse(content={"data": output_dict})
 
 
 async def get_dataset_versions(request: Request) -> Response:
@@ -713,7 +713,7 @@ async def get_dataset_csv(request: Request) -> Response:
           type: string
         description: Dataset ID
       - in: query
-        name: version
+        name: version_id
         schema:
           type: string
         description: Dataset version ID. If omitted, returns the latest version.
@@ -762,7 +762,7 @@ async def get_dataset_jsonl_openai_ft(request: Request) -> Response:
           type: string
         description: Dataset ID
       - in: query
-        name: version
+        name: version_id
         schema:
           type: string
         description: Dataset version ID. If omitted, returns the latest version.
@@ -811,7 +811,7 @@ async def get_dataset_jsonl_openai_evals(request: Request) -> Response:
           type: string
         description: Dataset ID
       - in: query
-        name: version
+        name: version_id
         schema:
           type: string
         description: Dataset version ID. If omitted, returns the latest version.
@@ -915,9 +915,9 @@ async def _get_db_examples(request: Request) -> Tuple[str, List[models.DatasetEx
         raise ValueError("Missing Dataset ID")
     dataset_id = from_global_id_with_expected_type(GlobalID.from_id(id_), Dataset.__name__)
     dataset_version_id: Optional[int] = None
-    if version := request.query_params.get("version"):
+    if version_id := request.query_params.get("version_id"):
         dataset_version_id = from_global_id_with_expected_type(
-            GlobalID.from_id(version),
+            GlobalID.from_id(version_id),
             DatasetVersion.__name__,
         )
     latest_version = (

phoenix/server/api/routers/v1/evaluations.py

@@ -45,13 +45,6 @@ async def post_evaluations(request: Request) -> Response:
     operationId: addEvaluations
     tags:
       - private
-    parameters:
-      - name: project-name
-        in: query
-        schema:
-          type: string
-          default: default
-        description: The project name to add the evaluation to
     requestBody:
       required: true
       content:
@@ -107,7 +100,7 @@ async def get_evaluations(request: Request) -> Response:
     tags:
       - private
     parameters:
-      - name: project-name
+      - name: project_name
         in: query
         schema:
           type: string
@@ -122,9 +115,9 @@ async def get_evaluations(request: Request) -> Response:
         description: Not found
     """
     project_name = (
-        request.query_params.get("project-name")
-        # read from headers for backwards compatibility
-        or request.headers.get("project-name")
+        request.query_params.get("project_name")
+        or request.query_params.get("project-name")  # for backward compatibility
+        or request.headers.get("project-name")  # read from headers for backwards compatibility
         or DEFAULT_PROJECT_NAME
     )
 

phoenix/server/api/routers/v1/experiment_evaluations.py

@@ -5,13 +5,13 @@ from starlette.responses import JSONResponse, Response
 from starlette.status import HTTP_404_NOT_FOUND
 from strawberry.relay import GlobalID
 
-from phoenix.datasets.types import EvaluationResult, ExperimentEvaluationRun
 from phoenix.db import models
+from phoenix.db.helpers import SupportedSQLDialect
+from phoenix.db.insertion.helpers import OnConflict, insert_on_conflict
 from phoenix.server.api.types.node import from_global_id_with_expected_type
-from phoenix.utilities.json import jsonify
 
 
-async def create_experiment_evaluation(request: Request) -> Response:
+async def upsert_experiment_evaluation(request: Request) -> Response:
     payload = await request.json()
     experiment_run_gid = GlobalID.from_id(payload["experiment_run_id"])
     try:
@@ -32,7 +32,7 @@ async def create_experiment_evaluation(request: Request) -> Response:
     start_time = payload["start_time"]
     end_time = payload["end_time"]
     async with request.app.state.db() as session:
-        exp_eval_run = models.ExperimentRunAnnotation(
+        values = dict(
             experiment_run_id=experiment_run_id,
             name=name,
             annotator_kind=annotator_kind,
@@ -40,27 +40,26 @@ async def create_experiment_evaluation(request: Request) -> Response:
             score=score,
             explanation=explanation,
             error=error,
-            metadata_=metadata,
+            metadata_=metadata,  # `metadata_` must match database
             start_time=datetime.fromisoformat(start_time),
             end_time=datetime.fromisoformat(end_time),
             trace_id=payload.get("trace_id"),
         )
-        session.add(exp_eval_run)
-        await session.flush()
-        evaluation_gid = GlobalID("ExperimentEvaluation", str(exp_eval_run.id))
-        eval_payload = ExperimentEvaluationRun(
-            id=str(evaluation_gid),
-            experiment_run_id=str(experiment_run_gid),
-            start_time=exp_eval_run.start_time,
-            end_time=exp_eval_run.end_time,
-            name=exp_eval_run.name,
-            annotator_kind=exp_eval_run.annotator_kind,
-            error=exp_eval_run.error,
-            result=EvaluationResult(
-                label=exp_eval_run.label,
-                score=exp_eval_run.score,
-                explanation=exp_eval_run.explanation,
-                metadata=exp_eval_run.metadata_,
-            ),
+        set_ = {
+            **{k: v for k, v in values.items() if k != "metadata_"},
+            "metadata": values["metadata_"],  # `metadata` must match database
+        }
+        dialect = SupportedSQLDialect(session.bind.dialect.name)
+        exp_eval_run = await session.scalar(
+            insert_on_conflict(
+                dialect=dialect,
+                table=models.ExperimentRunAnnotation,
+                values=values,
+                constraint="uq_experiment_run_annotations_experiment_run_id_name",
+                column_names=("experiment_run_id", "name"),
+                on_conflict=OnConflict.DO_UPDATE,
+                set_=set_,
+            ).returning(models.ExperimentRunAnnotation)
         )
-        return JSONResponse(content=jsonify(eval_payload), status_code=200)
+    evaluation_gid = GlobalID("ExperimentEvaluation", str(exp_eval_run.id))
+    return JSONResponse(content={"data": {"id": str(evaluation_gid)}})

phoenix/server/api/routers/v1/experiment_runs.py

@@ -6,8 +6,8 @@ from starlette.responses import JSONResponse, Response
 from starlette.status import HTTP_404_NOT_FOUND
 from strawberry.relay import GlobalID
 
-from phoenix.datasets.types import ExperimentResult, ExperimentRun
 from phoenix.db import models
+from phoenix.experiments.types import ExperimentResult, ExperimentRun
 from phoenix.server.api.types.node import from_global_id_with_expected_type
 from phoenix.utilities.json import jsonify
 
@@ -53,20 +53,8 @@ async def create_experiment_run(request: Request) -> Response:
         )
         session.add(exp_run)
         await session.flush()
-
-        run_gid = GlobalID("ExperimentRun", str(exp_run.id))
-        run_payload = ExperimentRun(
-            start_time=exp_run.start_time,
-            end_time=exp_run.end_time,
-            experiment_id=str(experiment_gid),
-            dataset_example_id=str(example_gid),
-            repetition_number=exp_run.repetition_number,
-            output=ExperimentResult.from_dict(exp_run.output) if exp_run.output else None,
-            error=exp_run.error,
-            id=str(run_gid),
-            trace_id=exp_run.trace_id,
-        )
-        return JSONResponse(content=jsonify(run_payload), status_code=200)
+    run_gid = GlobalID("ExperimentRun", str(exp_run.id))
+    return JSONResponse(content={"data": {"id": str(run_gid)}})
 
 
 async def list_experiment_runs(request: Request) -> Response:
@@ -105,4 +93,4 @@ async def list_experiment_runs(request: Request) -> Response:
                     trace_id=exp_run.trace_id,
                 )
             )
-        return JSONResponse(content=jsonify(runs), status_code=200)
+    return JSONResponse(content={"data": jsonify(runs)}, status_code=200)

phoenix/server/api/routers/v1/experiments.py

@@ -8,7 +8,7 @@ from strawberry.relay import GlobalID
 
 from phoenix.db import models
 from phoenix.db.helpers import SupportedSQLDialect
-from phoenix.db.insertion.helpers import insert_stmt
+from phoenix.db.insertion.helpers import insert_on_conflict
 from phoenix.server.api.types.node import from_global_id_with_expected_type
 
 
@@ -37,7 +37,7 @@ async def create_experiment(request: Request) -> Response:
     payload = await request.json()
     repetitions = payload.get("repetitions", 1)
     metadata = payload.get("metadata") or {}
-    dataset_version_globalid_str = payload.get("version-id")
+    dataset_version_globalid_str = payload.get("version_id")
     if dataset_version_globalid_str is not None:
         try:
             dataset_version_globalid = GlobalID.from_id(dataset_version_globalid_str)
@@ -105,7 +105,7 @@ async def create_experiment(request: Request) -> Response:
 
         dialect = SupportedSQLDialect(session.bind.dialect.name)
         project_rowid = await session.scalar(
-            insert_stmt(
+            insert_on_conflict(
                 dialect=dialect,
                 table=models.Project,
                 constraint="uq_projects_name",
@@ -135,7 +135,7 @@ async def create_experiment(request: Request) -> Response:
             "created_at": experiment.created_at.isoformat(),
             "updated_at": experiment.updated_at.isoformat(),
         }
-        return JSONResponse(content=experiment_payload, status_code=200)
+    return JSONResponse(content={"data": experiment_payload})
 
 
 async def read_experiment(request: Request) -> Response:
@@ -171,4 +171,4 @@ async def read_experiment(request: Request) -> Response:
             "created_at": experiment.created_at.isoformat(),
             "updated_at": experiment.updated_at.isoformat(),
         }
-        return JSONResponse(content=experiment_payload, status_code=200)
+    return JSONResponse(content={"data": experiment_payload})

phoenix/server/api/routers/v1/spans.py

@@ -21,7 +21,7 @@ async def query_spans_handler(request: Request) -> Response:
     tags:
       - private
     parameters:
-      - name: project-name
+      - name: project_name
         in: query
         schema:
           type: string
@@ -78,9 +78,11 @@ async def query_spans_handler(request: Request) -> Response:
     payload = await request.json()
     queries = payload.pop("queries", [])
     project_name = (
-        request.query_params.get("project-name")
-        # read from headers/payload for backward-compatibility
-        or request.headers.get("project-name")
+        request.query_params.get("project_name")
+        or request.query_params.get("project-name")  # for backward compatibility
+        or request.headers.get(
+            "project-name"
+        )  # read from headers/payload for backward-compatibility
         or payload.get("project_name")
         or DEFAULT_PROJECT_NAME
     )

phoenix/server/api/types/Experiment.py

@@ -103,6 +103,13 @@ class Experiment(Node):
     async def error_rate(self, info: Info[Context, None]) -> Optional[float]:
         return await info.context.data_loaders.experiment_error_rates.load(self.id_attr)
 
+    @strawberry.field
+    async def average_run_latency_ms(self, info: Info[Context, None]) -> float:
+        latency_seconds = await info.context.data_loaders.average_experiment_run_latency.load(
+            self.id_attr
+        )
+        return latency_seconds * 1000
+
     @strawberry.field
     async def project(self, info: Info[Context, None]) -> Optional[Project]:
         if self.project_name is None:

phoenix/server/app.py

@@ -56,6 +56,7 @@ from phoenix.exceptions import PhoenixMigrationError
 from phoenix.pointcloud.umap_parameters import UMAPParameters
 from phoenix.server.api.context import Context, DataLoaders
 from phoenix.server.api.dataloaders import (
+    AverageExperimentRunLatencyDataLoader,
     CacheForDataLoaders,
     DatasetExampleRevisionsDataLoader,
     DatasetExampleSpansDataLoader,
@@ -191,6 +192,7 @@ class GraphQLWithContext(GraphQL):  # type: ignore
             export_path=self.export_path,
             streaming_last_updated_at=self.streaming_last_updated_at,
             data_loaders=DataLoaders(
+                average_experiment_run_latency=AverageExperimentRunLatencyDataLoader(self.db),
                 dataset_example_revisions=DatasetExampleRevisionsDataLoader(self.db),
                 dataset_example_spans=DatasetExampleSpansDataLoader(self.db),
                 document_evaluation_summaries=DocumentEvaluationSummaryDataLoader(