arize-phoenix 11.28.0__py3-none-any.whl → 11.30.0__py3-none-any.whl

{arize_phoenix-11.28.0.dist-info → arize_phoenix-11.30.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: arize-phoenix
-Version: 11.28.0
+Version: 11.30.0
 Summary: AI Observability and Evaluation
 Project-URL: Documentation, https://arize.com/docs/phoenix/
 Project-URL: Issues, https://github.com/Arize-ai/phoenix/issues
@@ -50,7 +50,8 @@ Requires-Dist: python-multipart
 Requires-Dist: scikit-learn
 Requires-Dist: scipy
 Requires-Dist: sqlalchemy[asyncio]<3,>=2.0.4
-Requires-Dist: sqlean-py>=3.45.1
+Requires-Dist: sqlean-py<3.50,>=3.45.1; platform_system == 'Windows'
+Requires-Dist: sqlean-py>=3.45.1; platform_system != 'Windows'
 Requires-Dist: starlette
 Requires-Dist: strawberry-graphql==0.270.1
 Requires-Dist: tqdm

{arize_phoenix-11.28.0.dist-info → arize_phoenix-11.30.0.dist-info}/RECORD

@@ -6,7 +6,7 @@ phoenix/exceptions.py,sha256=n2L2KKuecrdflB9MsCdAYCiSEvGJptIsfRkXMoJle7A,169
 phoenix/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 phoenix/services.py,sha256=ngkyKGVatX3cO2WJdo2hKdaVKP-xJCMvqthvga6kJss,5196
 phoenix/settings.py,sha256=2kHfT3BNOVd4dAO1bq-syEQbHSG8oX2-7NhOwK2QREk,896
-phoenix/version.py,sha256=9BbsyqXMqD5D5cPHbfNkmPeW-WTp3yZaVegN3LRUWSU,24
+phoenix/version.py,sha256=JHzYn5SQjWvqEZWixlgmU5X8oFvAzqJf2SdpbNxVVK4,24
 phoenix/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 phoenix/core/embedding_dimension.py,sha256=zKGbcvwOXgLf-yrJBpQyKtd-LEOPRKHnUToyAU8Owis,87
 phoenix/core/model.py,sha256=qBFraOtmwCCnWJltKNP18DDG0mULXigytlFsa6YOz6k,4837
@@ -120,14 +120,14 @@ phoenix/server/api/utils.py,sha256=quCBRcusc6PUq9tJq7M8PgwFZp7nXgVAxtbw8feribY,8
 phoenix/server/api/dataloaders/__init__.py,sha256=ddiX1BdbyGkPTzMZNo-hkF_2kqIquelBUFvQejnAJYk,6834
 phoenix/server/api/dataloaders/annotation_configs_by_project.py,sha256=_Nfiug9o01JimU3Z0LpZJ0uaMCjchXomyt_dYAxPFRY,1178
 phoenix/server/api/dataloaders/annotation_summaries.py,sha256=0b23-bucBKyL25RWb2QzCNJjQzrq403qMmHKUVD5W4M,14377
-phoenix/server/api/dataloaders/average_experiment_run_latency.py,sha256=GLFoFAbztOH-0FVzzZ77mATIO63UcjB50j3qXiNi-tE,1811
+phoenix/server/api/dataloaders/average_experiment_run_latency.py,sha256=_wEcC47zOtLFMYTfyaFWA93TlAVC3kAPZhMnRO-xB24,1809
 phoenix/server/api/dataloaders/dataset_example_revisions.py,sha256=xF7M2dg3UmjhdCrscnztCIBBI0cg3RF48IIqvilpc18,4623
 phoenix/server/api/dataloaders/dataset_example_spans.py,sha256=z_MFquqAcJ9wat7BBp7MVeJ9BYuu2EZEdaog52iWDno,1390
 phoenix/server/api/dataloaders/document_evaluation_summaries.py,sha256=9fdROnzp-mymggHwNvpRkCk93LUFxxLy55-j3HP_2HY,5565
-phoenix/server/api/dataloaders/document_evaluations.py,sha256=KCnCItJ2DQOCHvxFp1KK2AStPN1akGlAGOEzEAu6c6I,1246
+phoenix/server/api/dataloaders/document_evaluations.py,sha256=W1b7TIlmPG61vR7kEcLZ5hAQYfkSKZAgrJOXYkA9-Ko,1246
 phoenix/server/api/dataloaders/document_retrieval_metrics.py,sha256=37EcAW7oYQuWYHMDHb0wcqbWj9lhSskvzDO7NJbT5Js,4136
-phoenix/server/api/dataloaders/experiment_annotation_summaries.py,sha256=fFEjpJzUOhu_cJKQ-YnwEvtn8NDl6bPs-moERe-Bp04,2767
-phoenix/server/api/dataloaders/experiment_error_rates.py,sha256=V4U_y16LwfBDksTZ4QP1dDALGSsQAQ1KcttKRE6lXto,1920
+phoenix/server/api/dataloaders/experiment_annotation_summaries.py,sha256=CFVj7DwFYj330FLU5w3zEr12AGUX1e8ZX0X5buxMuEk,5643
+phoenix/server/api/dataloaders/experiment_error_rates.py,sha256=06IZF07qt2y167DBM49QkSNdnphPArhcsgYFcunaL-U,1992
 phoenix/server/api/dataloaders/experiment_run_annotations.py,sha256=uJ--9Ue4tnmZYH9Zy-Cj2Y2t3xzs2xUqN8Y6EGe1MCo,1296
 phoenix/server/api/dataloaders/experiment_run_counts.py,sha256=j_7229IL705p_TycxIMYylhSgLHsAn91TiGPYXtOuQ8,1617
 phoenix/server/api/dataloaders/experiment_sequence_number.py,sha256=zM_f78fnqhppLtevrx9iISQSN7w_BNeXT9CoX8jYgAI,1534
@@ -255,10 +255,11 @@ phoenix/server/api/routers/auth.py,sha256=PKGwWdw7O015KmjMY1mIxlvXeU7OrmqIPF5TWT
 phoenix/server/api/routers/embeddings.py,sha256=BpZGJee0pdL0W5Rp1L0b30dEtZTgJeVqXky8LgZ0ZXw,898
 phoenix/server/api/routers/oauth2.py,sha256=rPcKFvfijzBYLjfwbCNzCn0ihn4wGWh4xh6BRqg9Ay4,24524
 phoenix/server/api/routers/utils.py,sha256=M41BoH-fl37izhRuN2aX7lWm7jOC20A_3uClv9TVUUY,583
-phoenix/server/api/routers/v1/__init__.py,sha256=ngLMPjC7lgZxgKy_Is33KxTRnMzSqy25qTTChCVx_Mo,2696
+phoenix/server/api/routers/v1/__init__.py,sha256=_CxVCs26dPuC2KygV3VzYqmoAECeFwiSTava9mxIJTE,2790
 phoenix/server/api/routers/v1/annotation_configs.py,sha256=xp5lJmKYlRsINCUrRD9-lTAElw2v4hdFndS5BWrxICA,16048
 phoenix/server/api/routers/v1/annotations.py,sha256=fVl2qeh_ZbWXGvFBTZgeL7aGkkINIScdjuyxnOoSzNM,6817
 phoenix/server/api/routers/v1/datasets.py,sha256=9iPORLmbOrPKgUUcRDMs6ZczSIz7hvc6bngJy3IbdR0,38331
+phoenix/server/api/routers/v1/documents.py,sha256=D8Pg6lEBHzSuPEDVts__X0ArIKBdQs_3gtIgoZXk_eU,6930
 phoenix/server/api/routers/v1/evaluations.py,sha256=aBrPO-xCAWyTxydaHq7W2wQFm65k89uVR-H3VWsd6WQ,13062
 phoenix/server/api/routers/v1/experiment_evaluations.py,sha256=DZ3UK9OoYKElpRcEER7559-KiAqWr-1IXpZ27FbfP3k,5249
 phoenix/server/api/routers/v1/experiment_runs.py,sha256=LZeCQWQIEOZ9jK5Gp_C4JbiYY6AmnnWe85cVcvdkCLE,7107
@@ -266,8 +267,8 @@ phoenix/server/api/routers/v1/experiments.py,sha256=hIBecGACzGZEgl93ap_JV52pUv-I
 phoenix/server/api/routers/v1/models.py,sha256=p3gJN-9SWiUYTUTft4bZMsZVCBNTb4nN1Foy68eRZzQ,1997
 phoenix/server/api/routers/v1/projects.py,sha256=XR6uJxHXXtC1q8LNyS9W6iaj440sv1OKCu-OSBfxEys,12824
 phoenix/server/api/routers/v1/prompts.py,sha256=chRYcLkOYDJdJfVZVukVTUyIRnLPvsJCg41CuPxOIU8,26695
-phoenix/server/api/routers/v1/spans.py,sha256=roDDE0RDBGSCircPgzwYWhwwyK33IaW7YliWN-lwxWw,49385
-phoenix/server/api/routers/v1/traces.py,sha256=ho6SXJ0R3g1ROxgtd7hNPo8QSZnA85oG0dgWDokHM2w,11365
+phoenix/server/api/routers/v1/spans.py,sha256=6wu8nUQNp9ma_k5XGvcVx3fq5xPuaN5sbv15ouBWcVc,49438
+phoenix/server/api/routers/v1/traces.py,sha256=ur4qVh8NDHDfwXKUNlAQoZhe4xAWe1Dv2ODixR5qroE,11418
 phoenix/server/api/routers/v1/users.py,sha256=eO8zMtGU33Td2_G1l9D7Z0a4CG1CwBUCj_Z9z2uk7wg,12089
 phoenix/server/api/routers/v1/utils.py,sha256=oXIOGPzPTkE0ZWUTRCoRIQQ7wTzoSwtWFaUSjlGBqts,4960
 phoenix/server/api/types/Annotation.py,sha256=gsl8CwjIbDUbZRj4d9USwZ_w_Tkz4i7zuZh9ftV80jA,1132
@@ -284,7 +285,7 @@ phoenix/server/api/types/CostBreakdown.py,sha256=yw9dlb0blGIB_dWNP8yEvDHJztHjpiV
 phoenix/server/api/types/CreateDatasetPayload.py,sha256=R-6zCmuD0f76RU9Giu78xwTHlASQs6Aq8yzvX1Kxc3g,140
 phoenix/server/api/types/CronExpression.py,sha256=R7oxuSSX_eTUHQWaoaSueQqWDmkkHr5dBKRN6q-6ROk,331
 phoenix/server/api/types/DataQualityMetric.py,sha256=Aieg3bHeBFaAf4mqeRcH1zT04sXAtQD8ATSHJt7FaBQ,1538
-phoenix/server/api/types/Dataset.py,sha256=StVJmOE996Citau11JtFTmcgLqvN9IeZsHAbe-Y1gkg,12933
+phoenix/server/api/types/Dataset.py,sha256=23dst_glr7kFNC62-q6D9H2hJgrfZnGe7V-Bg72SJgg,15303
 phoenix/server/api/types/DatasetExample.py,sha256=_9byxGpXfYb-hmFMUJeG7Bw1wsRKSJaHwF2IPAbFpFw,3115
 phoenix/server/api/types/DatasetExampleRevision.py,sha256=c-jWR6dTguEZTm54IMlFr0Ic84I3nefyDnZb7nF5hnI,874
 phoenix/server/api/types/DatasetValues.py,sha256=7VbCOLlzOXpZN80-zYF2UGuafRcPsZF-8WQNc0YsKFc,1119
@@ -298,12 +299,12 @@ phoenix/server/api/types/DocumentEvaluationSummary.py,sha256=dx4Btlfw9_XsfmibjfW
 phoenix/server/api/types/DocumentRetrievalMetrics.py,sha256=amkpC3H5IU5-9GvO0telpbq00m6lIcv_2v446OpwFwc,1822
 phoenix/server/api/types/EmbeddingDimension.py,sha256=AYvpZ1nWINAgN4BAZsA_xI_2TNFK6h5jmqzvkPs651M,19428
 phoenix/server/api/types/EmbeddingMetadata.py,sha256=fJvNNYCbkf3SJalArLy9rcBq9Uj1SNac60zjqe1PFnM,461
-phoenix/server/api/types/Evaluation.py,sha256=jPUH4fUFTpQHmiHA4zS4o3zigYp2LNX7VhlT580HJ_I,1301
+phoenix/server/api/types/Evaluation.py,sha256=qkMmq5G9pgwUNOPSTRwR-nNAoqLT3LlXrq7O8cAgXQ8,1356
 phoenix/server/api/types/EvaluationSummary.py,sha256=vILYejnfPvMwWEXOwhQZsANvYe3AdO2OkMR2rcgp1H4,1512
 phoenix/server/api/types/Event.py,sha256=iYt_Jx1Roioo0vZ0iPeJTHcTu6NSm4ilVMJ-IMUHAKk,3970
 phoenix/server/api/types/EventMetadata.py,sha256=-J0tYF9eZTHwCjwxQHY7Gckr2_MNW5OoWT1mydweZNM,635
 phoenix/server/api/types/ExampleRevisionInterface.py,sha256=gV3Gt9-3Oi5wjaVtepC6nOt3FzTzZFD1KebNnqiw56E,294
-phoenix/server/api/types/Experiment.py,sha256=8Hd-8-4Rcym95sPHIBLyTfMuRdyl50YxlKO_fMvdNAA,7830
+phoenix/server/api/types/Experiment.py,sha256=ifymI5SGqotIo01yAx-AsUONLzvFvjAKEU8gKi2WOzA,7700
 phoenix/server/api/types/ExperimentAnnotationSummary.py,sha256=Uk3JtxIrsMoZT5tqc4nJdUOM3XegVzjUyoV3pkjNotE,256
 phoenix/server/api/types/ExperimentComparison.py,sha256=PXFcB0e8aaJ391yRsuRJr9_dvTZI1RAzF93oC_-HtxU,461
 phoenix/server/api/types/ExperimentRun.py,sha256=_fcwDLuURV0yviOlkjWAgJJwcCPdz-xGR6VX3UKf73s,6541
@@ -337,7 +338,7 @@ phoenix/server/api/types/ScalarDriftMetricEnum.py,sha256=IUAcRPpgL41WdoIgK6cNk2T
 phoenix/server/api/types/Segments.py,sha256=vT2v0efoa5cuBKxLtxTnsUP5YJJCZfTloM71Spu0tMI,2915
 phoenix/server/api/types/ServerStatus.py,sha256=t92OHuVhK9DXDk2vsBuHceQNKqYGpHwUp8DNGKz2wOk,88
 phoenix/server/api/types/SortDir.py,sha256=OUpXhlCzCxPoXSDkJJygEs9Rw9pMymfaZUG5zPTrw4Y,152
-phoenix/server/api/types/Span.py,sha256=6vurLa8yoBs7GH7Jpj5niPAX7JxKPKVUp2qEJbaZxCI,32000
+phoenix/server/api/types/Span.py,sha256=ag5qWOEbWDCNUAkSUd0K3biE_PuGT2oweL4gVKElNis,32000
 phoenix/server/api/types/SpanAnnotation.py,sha256=uPWu7Z8rmpfKhaaxbged4_o00pPCR3nkn7Gji9vB8jY,1959
 phoenix/server/api/types/SpanCostDetailSummaryEntry.py,sha256=RXAdOC6MFyR9mwaoj8lMMdI3_9r3z6mR2izJvlsj12U,252
 phoenix/server/api/types/SpanCostSummary.py,sha256=wo03FCMcFzB5m4P5kvA5jzi9ACLbht38ozQbDJUh94g,357
@@ -392,10 +393,10 @@ phoenix/server/static/apple-touch-icon-76x76.png,sha256=CT_xT12I0u2i0WU8JzBZBuOQ
 phoenix/server/static/apple-touch-icon.png,sha256=fOfpjqGpWYbJ0eAurKsyoZP1EAs6ZVooBJ_SGk2ZkDs,3801
 phoenix/server/static/favicon.ico,sha256=bY0vvCKRftemZfPShwZtE93DiiQdaYaozkPGwNFr6H8,34494
 phoenix/server/static/modernizr.js,sha256=mvK-XtkNqjOral-QvzoqsyOMECXIMu5BQwSVN_wcU9c,2564
-phoenix/server/static/.vite/manifest.json,sha256=YLmkhNaP8CIb58FZA_WbstLTHmeLxO7Bf95-j2o1iwg,2328
-phoenix/server/static/assets/components-C3HQDu_r.js,sha256=KTCwdIg8fU5Xo3nRecbeh-LQqPr3s_T9LIEzYVbz8LE,664499
-phoenix/server/static/assets/index-C-sjZRYC.js,sha256=Y51REgta4DEkMYGcas1FLL1as0OyUOyEM1fFtRTxazc,63396
-phoenix/server/static/assets/pages-DvrxSPg3.js,sha256=LNxKvwIv55HqiVUdtSyiFcnS3qMSHUf0bBXOR0whd0k,1266943
+phoenix/server/static/.vite/manifest.json,sha256=podIkdilPnaQGOQjbnF7zpOylgP4MXrSImpB5UGvNBc,2328
+phoenix/server/static/assets/components-BBwXqJXQ.js,sha256=U8GMAX0TEjXRzhmqCMXjnVaQ9trJDICWTq9IcjqcNhE,664533
+phoenix/server/static/assets/index-C_gU3x10.js,sha256=ouZyJslt5vOJGBHCHiO0ziM9Y5lTwAdB86lGFUbr-Ec,63396
+phoenix/server/static/assets/pages-YmQb55Uo.js,sha256=tPGWJ9VTxJJSVoyQj_7VutaInS_SuhuXSal0nuJQTKQ,1269456
 phoenix/server/static/assets/vendor-CqDb5u4o.css,sha256=zIyFiNJKxMaQk8AvtLgt1rR01oO10d1MFndSDKH9Clw,5517
 phoenix/server/static/assets/vendor-RdRDaQiR.js,sha256=oTxLetZZXJ20yoKNAYExto9V73y8X5zjddWV46K9CWM,2595492
 phoenix/server/static/assets/vendor-arizeai-DsYDNOqt.js,sha256=0HIkPJXbKTh85nqphdAXYeStRzdaim0IQxRXiXxa21U,121514
@@ -442,9 +443,9 @@ phoenix/utilities/project.py,sha256=auVpARXkDb-JgeX5f2aStyFIkeKvGwN9l7qrFeJMVxI,
 phoenix/utilities/re.py,sha256=6YyUWIkv0zc2SigsxfOWIHzdpjKA_TZo2iqKq7zJKvw,2081
 phoenix/utilities/span_store.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 phoenix/utilities/template_formatters.py,sha256=gh9PJD6WEGw7TEYXfSst1UR4pWWwmjxMLrDVQ_CkpkQ,2779
-arize_phoenix-11.28.0.dist-info/METADATA,sha256=13C-DWFlaybHe2koaiC7DFxTwuJ93qLXMo9pxK8nkXM,31634
-arize_phoenix-11.28.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-arize_phoenix-11.28.0.dist-info/entry_points.txt,sha256=Pgpn8Upxx9P8z8joPXZWl2LlnAlGc3gcQoVchb06X1Q,94
-arize_phoenix-11.28.0.dist-info/licenses/IP_NOTICE,sha256=JBqyyCYYxGDfzQ0TtsQgjts41IJoa-hiwDrBjCb9gHM,469
-arize_phoenix-11.28.0.dist-info/licenses/LICENSE,sha256=HFkW9REuMOkvKRACuwLPT0hRydHb3zNg-fdFt94td18,3794
-arize_phoenix-11.28.0.dist-info/RECORD,,
+arize_phoenix-11.30.0.dist-info/METADATA,sha256=p2LDUgcsaZ4l5qRVqP5FVFjrpVpZTvzTN60j913wTtM,31733
+arize_phoenix-11.30.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+arize_phoenix-11.30.0.dist-info/entry_points.txt,sha256=Pgpn8Upxx9P8z8joPXZWl2LlnAlGc3gcQoVchb06X1Q,94
+arize_phoenix-11.30.0.dist-info/licenses/IP_NOTICE,sha256=JBqyyCYYxGDfzQ0TtsQgjts41IJoa-hiwDrBjCb9gHM,469
+arize_phoenix-11.30.0.dist-info/licenses/LICENSE,sha256=HFkW9REuMOkvKRACuwLPT0hRydHb3zNg-fdFt94td18,3794
+arize_phoenix-11.30.0.dist-info/RECORD,,

phoenix/server/api/dataloaders/average_experiment_run_latency.py

@@ -23,32 +23,25 @@ class AverageExperimentRunLatencyDataLoader(DataLoader[Key, Result]):
 
     async def _load_fn(self, keys: list[Key]) -> list[Result]:
         experiment_ids = keys
-        resolved_experiment_ids = (
-            select(models.Experiment.id)
-            .where(models.Experiment.id.in_(set(experiment_ids)))
-            .subquery()
-        )
-        query = (
+        average_repetition_latency_ms = (
             select(
-                resolved_experiment_ids.c.id,
-                func.avg(
-                    func.extract("epoch", models.ExperimentRun.end_time)
-                    - func.extract("epoch", models.ExperimentRun.start_time)
-                ),
+                models.ExperimentRun.experiment_id.label("experiment_id"),
+                func.avg(models.ExperimentRun.latency_ms).label("average_repetition_latency_ms"),
             )
-            .outerjoin_from(
-                from_=resolved_experiment_ids,
-                target=models.ExperimentRun,
-                onclause=resolved_experiment_ids.c.id == models.ExperimentRun.experiment_id,
-            )
-            .group_by(resolved_experiment_ids.c.id)
+            .select_from(models.ExperimentRun)
+            .where(models.ExperimentRun.experiment_id.in_(experiment_ids))
+            .group_by(models.ExperimentRun.dataset_example_id, models.ExperimentRun.experiment_id)
+            .subquery()
         )
+        query = select(
+            average_repetition_latency_ms.c.experiment_id,
+            func.avg(average_repetition_latency_ms.c.average_repetition_latency_ms).label(
+                "average_run_latency_ms"
+            ),
+        ).group_by(average_repetition_latency_ms.c.experiment_id)
         async with self._db() as session:
-            avg_latencies = {
-                experiment_id: avg_latency
-                async for experiment_id, avg_latency in await session.stream(query)
+            average_run_latencies_ms = {
+                experiment_id: average_run_latency_ms
+                async for experiment_id, average_run_latency_ms in await session.stream(query)
             }
-        return [
-            avg_latencies.get(experiment_id, ValueError(f"Unknown experiment: {experiment_id}"))
-            for experiment_id in keys
-        ]
+        return [average_run_latencies_ms.get(experiment_id) for experiment_id in keys]

phoenix/server/api/dataloaders/document_evaluations.py

@@ -5,11 +5,11 @@ from strawberry.dataloader import DataLoader
 from typing_extensions import TypeAlias
 
 from phoenix.db import models
-from phoenix.server.api.types.Evaluation import DocumentEvaluation
+from phoenix.server.api.types.Evaluation import DocumentAnnotation
 from phoenix.server.types import DbSessionFactory
 
 Key: TypeAlias = int
-Result: TypeAlias = list[DocumentEvaluation]
+Result: TypeAlias = list[DocumentAnnotation]
 
 
 class DocumentEvaluationsDataLoader(DataLoader[Key, Result]):
@@ -26,6 +26,6 @@ class DocumentEvaluationsDataLoader(DataLoader[Key, Result]):
             )
             async for document_evaluation in data:
                 document_evaluations_by_id[document_evaluation.span_rowid].append(
-                    DocumentEvaluation.from_sql_document_annotation(document_evaluation)
+                    DocumentAnnotation.from_sql_document_annotation(document_evaluation)
                 )
         return [document_evaluations_by_id[key] for key in keys]

phoenix/server/api/dataloaders/experiment_annotation_summaries.py

@@ -2,7 +2,7 @@ from collections import defaultdict
 from dataclasses import dataclass
 from typing import Optional
 
-from sqlalchemy import func, select
+from sqlalchemy import and_, func, select
 from strawberry.dataloader import AbstractCache, DataLoader
 from typing_extensions import TypeAlias
 
@@ -37,43 +37,97 @@ class ExperimentAnnotationSummaryDataLoader(DataLoader[Key, Result]):
     async def _load_fn(self, keys: list[Key]) -> list[Result]:
         experiment_ids = keys
         summaries: defaultdict[ExperimentID, Result] = defaultdict(list)
+        repetition_mean_scores_by_example_subquery = (
+            select(
+                models.ExperimentRun.experiment_id.label("experiment_id"),
+                models.ExperimentRunAnnotation.name.label("annotation_name"),
+                func.avg(models.ExperimentRunAnnotation.score).label("mean_repetition_score"),
+            )
+            .select_from(models.ExperimentRunAnnotation)
+            .join(
+                models.ExperimentRun,
+                models.ExperimentRunAnnotation.experiment_run_id == models.ExperimentRun.id,
+            )
+            .where(models.ExperimentRun.experiment_id.in_(experiment_ids))
+            .group_by(
+                models.ExperimentRun.experiment_id,
+                models.ExperimentRun.dataset_example_id,
+                models.ExperimentRunAnnotation.name,
+            )
+            .subquery()
+            .alias("repetition_mean_scores_by_example")
+        )
+        repetition_mean_scores_subquery = (
+            select(
+                repetition_mean_scores_by_example_subquery.c.experiment_id.label("experiment_id"),
+                repetition_mean_scores_by_example_subquery.c.annotation_name.label(
+                    "annotation_name"
+                ),
+                func.avg(repetition_mean_scores_by_example_subquery.c.mean_repetition_score).label(
+                    "mean_score"
+                ),
+            )
+            .select_from(repetition_mean_scores_by_example_subquery)
+            .group_by(
+                repetition_mean_scores_by_example_subquery.c.experiment_id,
+                repetition_mean_scores_by_example_subquery.c.annotation_name,
+            )
+            .subquery()
+            .alias("repetition_mean_scores")
+        )
+        repetitions_subquery = (
+            select(
+                models.ExperimentRun.experiment_id.label("experiment_id"),
+                models.ExperimentRunAnnotation.name.label("annotation_name"),
+                func.min(models.ExperimentRunAnnotation.score).label("min_score"),
+                func.max(models.ExperimentRunAnnotation.score).label("max_score"),
+                func.count().label("count"),
+                func.count(models.ExperimentRunAnnotation.error).label("error_count"),
+            )
+            .select_from(models.ExperimentRunAnnotation)
+            .join(
+                models.ExperimentRun,
+                models.ExperimentRunAnnotation.experiment_run_id == models.ExperimentRun.id,
+            )
+            .where(models.ExperimentRun.experiment_id.in_(experiment_ids))
+            .group_by(models.ExperimentRun.experiment_id, models.ExperimentRunAnnotation.name)
+            .subquery()
+        )
+        run_scores_query = (
+            select(
+                repetition_mean_scores_subquery.c.experiment_id.label("experiment_id"),
+                repetition_mean_scores_subquery.c.annotation_name.label("annotation_name"),
+                repetition_mean_scores_subquery.c.mean_score.label("mean_score"),
+                repetitions_subquery.c.min_score.label("min_score"),
+                repetitions_subquery.c.max_score.label("max_score"),
+                repetitions_subquery.c.count.label("count_"),
+                repetitions_subquery.c.error_count.label("error_count"),
+            )
+            .select_from(repetition_mean_scores_subquery)
+            .join(
+                repetitions_subquery,
+                and_(
+                    repetitions_subquery.c.experiment_id
+                    == repetition_mean_scores_subquery.c.experiment_id,
+                    repetitions_subquery.c.annotation_name
+                    == repetition_mean_scores_subquery.c.annotation_name,
+                ),
+            )
+            .order_by(repetition_mean_scores_subquery.c.annotation_name)
+        )
         async with self._db() as session:
-            async for (
-                experiment_id,
-                annotation_name,
-                min_score,
-                max_score,
-                mean_score,
-                count,
-                error_count,
-            ) in await session.stream(
-                select(
-                    models.ExperimentRun.experiment_id,
-                    models.ExperimentRunAnnotation.name,
-                    func.min(models.ExperimentRunAnnotation.score),
-                    func.max(models.ExperimentRunAnnotation.score),
-                    func.avg(models.ExperimentRunAnnotation.score),
-                    func.count(),
-                    func.count(models.ExperimentRunAnnotation.error),
-                )
-                .join(
-                    models.ExperimentRun,
-                    models.ExperimentRunAnnotation.experiment_run_id == models.ExperimentRun.id,
-                )
-                .where(models.ExperimentRun.experiment_id.in_(experiment_ids))
-                .group_by(models.ExperimentRun.experiment_id, models.ExperimentRunAnnotation.name)
-            ):
-                summaries[experiment_id].append(
+            async for scores_tuple in await session.stream(run_scores_query):
+                summaries[scores_tuple.experiment_id].append(
                     ExperimentAnnotationSummary(
-                        annotation_name=annotation_name,
-                        min_score=min_score,
-                        max_score=max_score,
-                        mean_score=mean_score,
-                        count=count,
-                        error_count=error_count,
+                        annotation_name=scores_tuple.annotation_name,
+                        min_score=scores_tuple.min_score,
+                        max_score=scores_tuple.max_score,
+                        mean_score=scores_tuple.mean_score,
+                        count=scores_tuple.count_,
+                        error_count=scores_tuple.error_count,
                     )
                 )
         return [
             sorted(summaries[experiment_id], key=lambda summary: summary.annotation_name)
-            for experiment_id in keys
+            for experiment_id in experiment_ids
         ]

phoenix/server/api/dataloaders/experiment_error_rates.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from sqlalchemy import case, func, select
+from sqlalchemy import func, select
 from strawberry.dataloader import DataLoader
 from typing_extensions import TypeAlias
 
@@ -23,36 +23,29 @@ class ExperimentErrorRatesDataLoader(DataLoader[Key, Result]):
 
     async def _load_fn(self, keys: list[Key]) -> list[Result]:
         experiment_ids = keys
-        resolved_experiment_ids = (
-            select(models.Experiment.id)
-            .where(models.Experiment.id.in_(set(experiment_ids)))
-            .subquery()
-        )
-        query = (
+        average_repetition_error_rates_subquery = (
             select(
-                resolved_experiment_ids.c.id,
-                case(
-                    (
-                        func.count(models.ExperimentRun.id) != 0,
-                        func.count(models.ExperimentRun.error)
-                        / func.count(models.ExperimentRun.id),
-                    ),
-                    else_=None,
-                ),
+                models.ExperimentRun.experiment_id.label("experiment_id"),
+                (
+                    func.count(models.ExperimentRun.error) / func.count(models.ExperimentRun.id)
+                ).label("average_repetition_error_rate"),
             )
-            .outerjoin_from(
-                from_=resolved_experiment_ids,
-                target=models.ExperimentRun,
-                onclause=resolved_experiment_ids.c.id == models.ExperimentRun.experiment_id,
-            )
-            .group_by(resolved_experiment_ids.c.id)
+            .where(models.ExperimentRun.experiment_id.in_(experiment_ids))
+            .group_by(models.ExperimentRun.dataset_example_id, models.ExperimentRun.experiment_id)
+            .subquery()
+            .alias("average_repetition_error_rates")
         )
+        average_run_error_rates_query = select(
+            average_repetition_error_rates_subquery.c.experiment_id,
+            func.avg(average_repetition_error_rates_subquery.c.average_repetition_error_rate).label(
+                "average_run_error_rates"
+            ),
+        ).group_by(average_repetition_error_rates_subquery.c.experiment_id)
         async with self._db() as session:
-            error_rates = {
+            average_run_error_rates = {
                 experiment_id: error_rate
-                async for experiment_id, error_rate in await session.stream(query)
+                async for experiment_id, error_rate in await session.stream(
+                    average_run_error_rates_query
+                )
             }
-        return [
-            error_rates.get(experiment_id, ValueError(f"Unknown experiment ID: {experiment_id}"))
-            for experiment_id in keys
-        ]
+        return [average_run_error_rates.get(experiment_id) for experiment_id in experiment_ids]

phoenix/server/api/routers/v1/__init__.py

@@ -7,6 +7,7 @@ from phoenix.server.bearer_auth import is_authenticated
 from .annotation_configs import router as annotation_configs_router
 from .annotations import router as annotations_router
 from .datasets import router as datasets_router
+from .documents import router as documents_router
 from .evaluations import router as evaluations_router
 from .experiment_evaluations import router as experiment_evaluations_router
 from .experiment_runs import router as experiment_runs_router
@@ -70,5 +71,6 @@ def create_v1_router(authentication_enabled: bool) -> APIRouter:
     router.include_router(evaluations_router)
     router.include_router(prompts_router)
     router.include_router(projects_router)
+    router.include_router(documents_router)
     router.include_router(users_router)
     return router

phoenix/server/api/routers/v1/documents.py

@@ -0,0 +1,175 @@
+from datetime import datetime, timezone
+from typing import Any, Literal, Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from pydantic import Field
+from sqlalchemy import select
+from starlette.requests import Request
+from starlette.status import HTTP_404_NOT_FOUND
+from strawberry.relay import GlobalID
+
+from phoenix.db import models
+from phoenix.db.helpers import SupportedSQLDialect
+from phoenix.db.insertion.helpers import as_kv, insert_on_conflict
+from phoenix.db.insertion.types import Precursors
+from phoenix.server.api.types.Evaluation import DocumentAnnotation
+from phoenix.server.authorization import is_not_locked
+from phoenix.server.bearer_auth import PhoenixUser
+from phoenix.server.dml_event import DocumentAnnotationInsertEvent
+
+from .models import V1RoutesBaseModel
+from .spans import SpanAnnotationResult
+from .utils import RequestBody, ResponseBody, add_errors_to_responses
+
+# Since the document annotations are spans related, we place it under spans
+router = APIRouter(tags=["spans"])
+
+
+class SpanDocumentAnnotationData(V1RoutesBaseModel):
+    span_id: str = Field(description="OpenTelemetry Span ID (hex format w/o 0x prefix)")
+    name: str = Field(description="The name of the document annotation. E.x. relevance")
+    annotator_kind: Literal["LLM", "CODE", "HUMAN"] = Field(
+        description="The kind of annotator. E.g. llm judge, a heuristic piece of code, or a human"
+    )
+    document_position: int = Field(
+        description="A 0 based index of the document. E.x. the first document during retrieval is 0"
+    )
+    result: Optional[SpanAnnotationResult] = Field(
+        default=None, description="The score and or label of the annotation"
+    )
+    metadata: Optional[dict[str, Any]] = Field(
+        default=None, description="Metadata for custom values of the annotation"
+    )
+    identifier: str = Field(
+        default="",
+        description=(
+            "An custom ID for the annotation. "
+            "If provided, the annotation will be updated if it already exists."
+        ),
+    )
+
+    # Precursor here means a value to add to a queue for processing async
+    def as_precursor(self, *, user_id: Optional[int] = None) -> Precursors.DocumentAnnotation:
+        return Precursors.DocumentAnnotation(
+            datetime.now(timezone.utc),
+            self.span_id,
+            self.document_position,
+            models.DocumentAnnotation(
+                name=self.name,
+                annotator_kind=self.annotator_kind,
+                document_position=self.document_position,
+                score=self.result.score if self.result else None,
+                label=self.result.label if self.result else None,
+                explanation=self.result.explanation if self.result else None,
+                metadata_=self.metadata or {},
+                identifier=self.identifier,
+                source="API",
+                user_id=user_id,
+            ),
+        )
+
+
+class AnnotateSpanDocumentsRequestBody(RequestBody[list[SpanDocumentAnnotationData]]):
+    pass
+
+
+class InsertedSpanDocumentAnnotation(V1RoutesBaseModel):
+    id: str = Field(description="The ID of the inserted span document annotation")
+
+
+class AnnotateSpanDocumentsResponseBody(ResponseBody[list[InsertedSpanDocumentAnnotation]]):
+    pass
+
+
+@router.post(
+    "/document_annotations",
+    dependencies=[Depends(is_not_locked)],
+    operation_id="annotateSpanDocuments",
+    responses=add_errors_to_responses(
+        [
+            {
+                "status_code": HTTP_404_NOT_FOUND,
+                "description": "Span not found",
+            }
+        ]
+    ),
+    response_description="Span document annotation inserted successfully",
+    include_in_schema=True,
+)
+async def annotate_span_documents(
+    request: Request,
+    request_body: AnnotateSpanDocumentsRequestBody,
+    sync: bool = Query(
+        default=False, description="If set to true, the annotations are inserted synchronously."
+    ),
+) -> AnnotateSpanDocumentsResponseBody:
+    if not request_body.data:
+        return AnnotateSpanDocumentsResponseBody(data=[])
+
+    user_id: Optional[int] = None
+    if request.app.state.authentication_enabled and isinstance(request.user, PhoenixUser):
+        user_id = int(request.user.identity)
+
+    span_document_annotations = request_body.data
+
+    precursors = [
+        annotation.as_precursor(user_id=user_id) for annotation in span_document_annotations
+    ]
+    if not sync:
+        await request.state.enqueue(*precursors)
+
+    span_ids = {p.span_id for p in precursors}
+    # Account for the fact that the spans could arrive after the annotation
+    async with request.app.state.db() as session:
+        existing_spans = {
+            span_id: (id_, num_docs)
+            async for span_id, id_, num_docs in await session.stream(
+                select(models.Span.span_id, models.Span.id, models.Span.num_documents).filter(
+                    models.Span.span_id.in_(span_ids)
+                )
+            )
+        }
+
+    missing_span_ids = span_ids - set(existing_spans.keys())
+    # We prefer to fail the entire operation if there are missing spans in sync mode
+    if missing_span_ids:
+        raise HTTPException(
+            detail=f"Spans with IDs {', '.join(missing_span_ids)} do not exist.",
+            status_code=HTTP_404_NOT_FOUND,
+        )
+
+    # Validate that document positions are within bounds
+    for annotation in span_document_annotations:
+        _, num_docs = existing_spans[annotation.span_id]
+        if annotation.document_position not in range(num_docs):
+            raise HTTPException(
+                detail=f"Document position {annotation.document_position} is out of bounds for "
+                f"span {annotation.span_id} (max: {num_docs - 1})",
+                status_code=422,  # Unprocessable Entity
+            )
+
+    inserted_document_annotation_ids = []
+    dialect = SupportedSQLDialect(session.bind.dialect.name)
+    for anno in precursors:
+        span_rowid, _ = existing_spans[anno.span_id]
+        values = dict(as_kv(anno.as_insertable(span_rowid).row))
+        span_document_annotation_id = await session.scalar(
+            insert_on_conflict(
+                values,
+                dialect=dialect,
+                table=models.DocumentAnnotation,
+                unique_by=("name", "span_rowid", "identifier", "document_position"),
+            ).returning(models.DocumentAnnotation.id)
+        )
+        inserted_document_annotation_ids.append(span_document_annotation_id)
+
+    # We queue an event to let the application know that annotations have changed
+    request.state.event_queue.put(
+        DocumentAnnotationInsertEvent(tuple(inserted_document_annotation_ids))
+    )
+    return AnnotateSpanDocumentsResponseBody(
+        data=[
+            InsertedSpanDocumentAnnotation(id=str(GlobalID(DocumentAnnotation.__name__, str(id_))))
+            for id_ in inserted_document_annotation_ids
+        ]
+    )

phoenix/server/api/routers/v1/spans.py

@@ -949,9 +949,11 @@ async def annotate_spans(
     span_ids = {p.span_id for p in precursors}
     async with request.app.state.db() as session:
         existing_spans = {
-            span.span_id: span.id
-            async for span in await session.stream_scalars(
-                select(models.Span).filter(models.Span.span_id.in_(span_ids))
+            span_id: id_
+            async for span_id, id_ in await session.stream(
+                select(models.Span.span_id, models.Span.id).filter(
+                    models.Span.span_id.in_(span_ids)
+                )
             )
         }
 

phoenix/server/api/routers/v1/traces.py

@@ -193,9 +193,11 @@ async def annotate_traces(
     trace_ids = {p.trace_id for p in precursors}
     async with request.app.state.db() as session:
         existing_traces = {
-            trace.trace_id: trace.id
-            async for trace in await session.stream_scalars(
-                select(models.Trace).filter(models.Trace.trace_id.in_(trace_ids))
+            trace_id: id_
+            async for trace_id, id_ in await session.stream(
+                select(models.Trace.trace_id, models.Trace.id).filter(
+                    models.Trace.trace_id.in_(trace_ids)
+                )
             )
         }