arize-phoenix 4.4.4rc4__py3-none-any.whl → 4.4.4rc6__py3-none-any.whl

phoenix/server/api/routers/v1/experiment_evaluations.py

@@ -5,13 +5,13 @@ from starlette.responses import JSONResponse, Response
 from starlette.status import HTTP_404_NOT_FOUND
 from strawberry.relay import GlobalID
 
-from phoenix.datasets.types import EvaluationResult, ExperimentEvaluationRun
 from phoenix.db import models
+from phoenix.db.helpers import SupportedSQLDialect
+from phoenix.db.insertion.helpers import OnConflict, insert_on_conflict
 from phoenix.server.api.types.node import from_global_id_with_expected_type
-from phoenix.utilities.json import jsonify
 
 
-async def create_experiment_evaluation(request: Request) -> Response:
+async def upsert_experiment_evaluation(request: Request) -> Response:
     payload = await request.json()
     experiment_run_gid = GlobalID.from_id(payload["experiment_run_id"])
     try:
@@ -32,7 +32,7 @@ async def create_experiment_evaluation(request: Request) -> Response:
     start_time = payload["start_time"]
     end_time = payload["end_time"]
     async with request.app.state.db() as session:
-        exp_eval_run = models.ExperimentRunAnnotation(
+        values = dict(
             experiment_run_id=experiment_run_id,
             name=name,
             annotator_kind=annotator_kind,
@@ -40,26 +40,26 @@ async def create_experiment_evaluation(request: Request) -> Response:
             score=score,
             explanation=explanation,
             error=error,
-            metadata_=metadata,
+            metadata_=metadata,  # `metadata_` must match database
             start_time=datetime.fromisoformat(start_time),
             end_time=datetime.fromisoformat(end_time),
+            trace_id=payload.get("trace_id"),
         )
-        session.add(exp_eval_run)
-        await session.flush()
-        evaluation_gid = GlobalID("ExperimentEvaluation", str(exp_eval_run.id))
-        eval_payload = ExperimentEvaluationRun(
-            id=str(evaluation_gid),
-            experiment_run_id=str(experiment_run_gid),
-            start_time=exp_eval_run.start_time,
-            end_time=exp_eval_run.end_time,
-            name=exp_eval_run.name,
-            annotator_kind=exp_eval_run.annotator_kind,
-            error=exp_eval_run.error,
-            result=EvaluationResult(
-                label=exp_eval_run.label,
-                score=exp_eval_run.score,
-                explanation=exp_eval_run.explanation,
-                metadata=exp_eval_run.metadata_,
-            ),
+        set_ = {
+            **{k: v for k, v in values.items() if k != "metadata_"},
+            "metadata": values["metadata_"],  # `metadata` must match database
+        }
+        dialect = SupportedSQLDialect(session.bind.dialect.name)
+        exp_eval_run = await session.scalar(
+            insert_on_conflict(
+                dialect=dialect,
+                table=models.ExperimentRunAnnotation,
+                values=values,
+                constraint="uq_experiment_run_annotations_experiment_run_id_name",
+                column_names=("experiment_run_id", "name"),
+                on_conflict=OnConflict.DO_UPDATE,
+                set_=set_,
+            ).returning(models.ExperimentRunAnnotation)
         )
-        return JSONResponse(content=jsonify(eval_payload), status_code=200)
+    evaluation_gid = GlobalID("ExperimentEvaluation", str(exp_eval_run.id))
+    return JSONResponse(content={"data": {"id": str(evaluation_gid)}})

phoenix/server/api/routers/v1/experiment_runs.py

@@ -6,8 +6,8 @@ from starlette.responses import JSONResponse, Response
 from starlette.status import HTTP_404_NOT_FOUND
 from strawberry.relay import GlobalID
 
-from phoenix.datasets.types import ExperimentResult, ExperimentRun
 from phoenix.db import models
+from phoenix.experiments.types import ExperimentResult, ExperimentRun
 from phoenix.server.api.types.node import from_global_id_with_expected_type
 from phoenix.utilities.json import jsonify
 
@@ -53,20 +53,8 @@ async def create_experiment_run(request: Request) -> Response:
         )
         session.add(exp_run)
         await session.flush()
-
-        run_gid = GlobalID("ExperimentRun", str(exp_run.id))
-        run_payload = ExperimentRun(
-            start_time=exp_run.start_time,
-            end_time=exp_run.end_time,
-            experiment_id=str(experiment_gid),
-            dataset_example_id=str(example_gid),
-            repetition_number=exp_run.repetition_number,
-            output=ExperimentResult(result=exp_run.output),
-            error=exp_run.error,
-            id=str(run_gid),
-            trace_id=exp_run.trace_id,
-        )
-        return JSONResponse(content=jsonify(run_payload), status_code=200)
+    run_gid = GlobalID("ExperimentRun", str(exp_run.id))
+    return JSONResponse(content={"data": {"id": str(run_gid)}})
 
 
 async def list_experiment_runs(request: Request) -> Response:
@@ -99,10 +87,10 @@ async def list_experiment_runs(request: Request) -> Response:
                     experiment_id=str(experiment_gid),
                     dataset_example_id=str(example_gid),
                     repetition_number=exp_run.repetition_number,
-                    output=ExperimentResult(result=exp_run.output),
+                    output=ExperimentResult.from_dict(exp_run.output) if exp_run.output else None,
                     error=exp_run.error,
                     id=str(run_gid),
                     trace_id=exp_run.trace_id,
                 )
             )
-        return JSONResponse(content=jsonify(runs), status_code=200)
+    return JSONResponse(content={"data": jsonify(runs)}, status_code=200)

phoenix/server/api/routers/v1/experiments.py

@@ -8,7 +8,7 @@ from strawberry.relay import GlobalID
 
 from phoenix.db import models
 from phoenix.db.helpers import SupportedSQLDialect
-from phoenix.db.insertion.helpers import insert_stmt
+from phoenix.db.insertion.helpers import insert_on_conflict
 from phoenix.server.api.types.node import from_global_id_with_expected_type
 
 
@@ -37,7 +37,7 @@ async def create_experiment(request: Request) -> Response:
     payload = await request.json()
     repetitions = payload.get("repetitions", 1)
     metadata = payload.get("metadata") or {}
-    dataset_version_globalid_str = payload.get("version-id")
+    dataset_version_globalid_str = payload.get("version_id")
     if dataset_version_globalid_str is not None:
         try:
             dataset_version_globalid = GlobalID.from_id(dataset_version_globalid_str)
@@ -105,7 +105,7 @@ async def create_experiment(request: Request) -> Response:
 
         dialect = SupportedSQLDialect(session.bind.dialect.name)
         project_rowid = await session.scalar(
-            insert_stmt(
+            insert_on_conflict(
                 dialect=dialect,
                 table=models.Project,
                 constraint="uq_projects_name",
@@ -135,7 +135,7 @@ async def create_experiment(request: Request) -> Response:
             "created_at": experiment.created_at.isoformat(),
             "updated_at": experiment.updated_at.isoformat(),
         }
-        return JSONResponse(content=experiment_payload, status_code=200)
+    return JSONResponse(content={"data": experiment_payload})
 
 
 async def read_experiment(request: Request) -> Response:
@@ -171,4 +171,4 @@ async def read_experiment(request: Request) -> Response:
             "created_at": experiment.created_at.isoformat(),
             "updated_at": experiment.updated_at.isoformat(),
         }
-        return JSONResponse(content=experiment_payload, status_code=200)
+    return JSONResponse(content={"data": experiment_payload})

phoenix/server/api/routers/v1/spans.py

@@ -21,7 +21,7 @@ async def query_spans_handler(request: Request) -> Response:
     tags:
       - private
     parameters:
-      - name: project-name
+      - name: project_name
         in: query
         schema:
           type: string
@@ -78,9 +78,11 @@ async def query_spans_handler(request: Request) -> Response:
     payload = await request.json()
     queries = payload.pop("queries", [])
     project_name = (
-        request.query_params.get("project-name")
-        # read from headers/payload for backward-compatibility
-        or request.headers.get("project-name")
+        request.query_params.get("project_name")
+        or request.query_params.get("project-name")  # for backward compatibility
+        or request.headers.get(
+            "project-name"
+        )  # read from headers/payload for backward-compatibility
         or payload.get("project_name")
         or DEFAULT_PROJECT_NAME
     )

phoenix/server/api/types/Experiment.py

@@ -75,6 +75,11 @@ class Experiment(Node):
             ).all()
         return connection_from_list([to_gql_experiment_run(run) for run in runs], args)
 
+    @strawberry.field
+    async def run_count(self, info: Info[Context, None]) -> int:
+        experiment_id = self.id_attr
+        return await info.context.data_loaders.experiment_run_counts.load(experiment_id)
+
     @strawberry.field
     async def annotation_summaries(
         self, info: Info[Context, None]
@@ -98,6 +103,13 @@ class Experiment(Node):
     async def error_rate(self, info: Info[Context, None]) -> Optional[float]:
         return await info.context.data_loaders.experiment_error_rates.load(self.id_attr)
 
+    @strawberry.field
+    async def average_run_latency_ms(self, info: Info[Context, None]) -> float:
+        latency_seconds = await info.context.data_loaders.average_experiment_run_latency.load(
+            self.id_attr
+        )
+        return latency_seconds * 1000
+
     @strawberry.field
     async def project(self, info: Info[Context, None]) -> Optional[Project]:
         if self.project_name is None:

phoenix/server/api/types/ExperimentRun.py

@@ -84,7 +84,7 @@ def to_gql_experiment_run(run: models.ExperimentRun) -> ExperimentRun:
         trace_id=trace_id
         if (trace := run.trace) and (trace_id := trace.trace_id) is not None
         else None,
-        output=run.output,
+        output=run.output.get("result"),
         start_time=run.start_time,
         end_time=run.end_time,
         error=run.error,

phoenix/server/api/types/ExperimentRunAnnotation.py

@@ -33,7 +33,7 @@ class ExperimentRunAnnotation(Node):
         if (trace := await dataloader.load(self.trace_id)) is None:
             return None
         trace_row_id, project_row_id = trace
-        return Trace(id_attr=trace_row_id, trace_id=trace.trace_id, project_rowid=project_row_id)
+        return Trace(id_attr=trace_row_id, trace_id=self.trace_id, project_rowid=project_row_id)
 
 
 def to_gql_experiment_run_annotation(

phoenix/server/app.py

@@ -56,6 +56,7 @@ from phoenix.exceptions import PhoenixMigrationError
 from phoenix.pointcloud.umap_parameters import UMAPParameters
 from phoenix.server.api.context import Context, DataLoaders
 from phoenix.server.api.dataloaders import (
+    AverageExperimentRunLatencyDataLoader,
     CacheForDataLoaders,
     DatasetExampleRevisionsDataLoader,
     DatasetExampleSpansDataLoader,
@@ -65,6 +66,7 @@ from phoenix.server.api.dataloaders import (
     EvaluationSummaryDataLoader,
     ExperimentAnnotationSummaryDataLoader,
     ExperimentErrorRatesDataLoader,
+    ExperimentRunCountsDataLoader,
     ExperimentSequenceNumberDataLoader,
     LatencyMsQuantileDataLoader,
     MinStartOrMaxEndTimeDataLoader,
@@ -190,6 +192,7 @@ class GraphQLWithContext(GraphQL):  # type: ignore
             export_path=self.export_path,
             streaming_last_updated_at=self.streaming_last_updated_at,
             data_loaders=DataLoaders(
+                average_experiment_run_latency=AverageExperimentRunLatencyDataLoader(self.db),
                 dataset_example_revisions=DatasetExampleRevisionsDataLoader(self.db),
                 dataset_example_spans=DatasetExampleSpansDataLoader(self.db),
                 document_evaluation_summaries=DocumentEvaluationSummaryDataLoader(
@@ -208,6 +211,7 @@ class GraphQLWithContext(GraphQL):  # type: ignore
                 ),
                 experiment_annotation_summaries=ExperimentAnnotationSummaryDataLoader(self.db),
                 experiment_error_rates=ExperimentErrorRatesDataLoader(self.db),
+                experiment_run_counts=ExperimentRunCountsDataLoader(self.db),
                 experiment_sequence_number=ExperimentSequenceNumberDataLoader(self.db),
                 latency_ms_quantile=LatencyMsQuantileDataLoader(
                     self.db,