arize-phoenix 4.16.1__py3-none-any.whl → 4.18.0__py3-none-any.whl

{arize_phoenix-4.16.1.dist-info → arize_phoenix-4.18.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: arize-phoenix
-Version: 4.16.1
+Version: 4.18.0
 Summary: AI Observability and Evaluation
 Project-URL: Documentation, https://docs.arize.com/phoenix/
 Project-URL: Issues, https://github.com/Arize-ai/phoenix/issues
@@ -28,10 +28,10 @@ Requires-Dist: hdbscan>=0.8.33
 Requires-Dist: httpx
 Requires-Dist: jinja2
 Requires-Dist: numpy<2
-Requires-Dist: openinference-instrumentation
-Requires-Dist: openinference-instrumentation-langchain>=0.1.12
-Requires-Dist: openinference-instrumentation-llama-index>=1.2.0
-Requires-Dist: openinference-instrumentation-openai>=0.1.4
+Requires-Dist: openinference-instrumentation-langchain>=0.1.26
+Requires-Dist: openinference-instrumentation-llama-index>=2.2.1
+Requires-Dist: openinference-instrumentation-openai>=0.1.11
+Requires-Dist: openinference-instrumentation>=0.1.12
 Requires-Dist: openinference-semantic-conventions>=0.1.9
 Requires-Dist: opentelemetry-exporter-otlp
 Requires-Dist: opentelemetry-proto>=1.12.0

{arize_phoenix-4.16.1.dist-info → arize_phoenix-4.18.0.dist-info}/RECORD

@@ -5,7 +5,7 @@ phoenix/exceptions.py,sha256=n2L2KKuecrdflB9MsCdAYCiSEvGJptIsfRkXMoJle7A,169
 phoenix/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 phoenix/services.py,sha256=aTxhcOA1pZHB6U-B3TEcp6fqDF5oT0xCUvEUNMZVTUQ,5175
 phoenix/settings.py,sha256=cO-qgis_S27nHirTobYI9hHPfZH18R--WMmxNdsVUwc,273
-phoenix/version.py,sha256=JsY9dR1EpUWL0aoebyfhTna4xxHCsYTHWzp8Gm8fUa8,23
+phoenix/version.py,sha256=g7MWtE7e6MvbE-FD9TSDyMr8ejjsU5QFtDnHc8sT424,23
 phoenix/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 phoenix/core/embedding_dimension.py,sha256=zKGbcvwOXgLf-yrJBpQyKtd-LEOPRKHnUToyAU8Owis,87
 phoenix/core/model.py,sha256=km_a--PBHOuA337ClRw9xqhOHhrUT6Rl9pz_zV0JYkQ,4843
@@ -65,7 +65,7 @@ phoenix/pointcloud/pointcloud.py,sha256=4zAIkKs2xOUbchpj4XDAV-iPMXrfAJ15TG6rlIYG
 phoenix/pointcloud/projectors.py,sha256=zO_RrtDYSv2rqVOfIP2_9Cv11Dc8EmcZR94xhFcBYPU,1057
 phoenix/pointcloud/umap_parameters.py,sha256=3UQSjrysVOvq2V4KNpTMqNqNiK0BsTZnPBHWZ4fyJtQ,1708
 phoenix/server/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-phoenix/server/app.py,sha256=gAmGzOAhGO05IBEnBNfOc8Suw15_0_h0OCp-9RP0udQ,19098
+phoenix/server/app.py,sha256=FqsV0CqFwzyPDmVeTSf3uwBC6DkyZXbiikSJkhS0Og8,19387
 phoenix/server/grpc_server.py,sha256=jllxDNkpLQxDkvej4RhTokobowbvydF-SU8gSw1MTCc,3378
 phoenix/server/main.py,sha256=dvjv3g8ANpkvSGCUN02S2Yse643Nlwrp_bj4iXBSVTE,11082
 phoenix/server/prometheus.py,sha256=j9DHB2fERuq_ZKmwVaqR-9wx5WcPPuU1Cm5Bhg5241Y,2996
@@ -73,12 +73,13 @@ phoenix/server/telemetry.py,sha256=T_2OKrxNViAeaANlNspEekg_Y5uZIFWvKAnpz8Aoqvk,2
 phoenix/server/thread_server.py,sha256=RwXQGP_QhGD7le6WB7xEygEEuwBl5Ck_Zo8xGIYGi9M,2135
 phoenix/server/types.py,sha256=o3uwy8NvfLO-j86v2Dl3irko6er0hvB_x4Df3S1DveM,460
 phoenix/server/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-phoenix/server/api/context.py,sha256=IZNZB8VMOL_mKzTkvyS5RyIcZcI0gwUt2qZ6jppBDWc,2853
+phoenix/server/api/context.py,sha256=70fJ9_O2yD-JEiCUM6WuKr6Byw9Sq00z_q0gBFW0Acg,2940
 phoenix/server/api/interceptor.py,sha256=ykDnoC_apUd-llVli3m1CW18kNSIgjz2qZ6m5JmPDu8,1294
 phoenix/server/api/queries.py,sha256=eq2xHaQF-x4k6AGSY6b6mU2pie9bj-AJML6P2Mr0_DM,19886
 phoenix/server/api/schema.py,sha256=BcxdqO5CSGqpKd-AAJHMjFlzaK9oJA8GJuxmMfcdjn4,434
 phoenix/server/api/utils.py,sha256=Kl47G-1A7QKTDrc75BU2QK6HupsG6MWuXxy351FOfKQ,858
-phoenix/server/api/dataloaders/__init__.py,sha256=C48-E6wkXSfAweypgH3BE_m0bAOLBVbIVzJxkva6t4g,5038
+phoenix/server/api/dataloaders/__init__.py,sha256=aWL1--L5fOh6ouBS8VeJRJBtJogKTtRHySPpkr_AUGk,5506
+phoenix/server/api/dataloaders/annotation_summaries.py,sha256=HO5PWAmFr_hmN6vYbui5jbAV3s6YPfOx6I94wr0_pgw,5500
 phoenix/server/api/dataloaders/average_experiment_run_latency.py,sha256=q091UmkXx37OBKh7L-GJ5LXHyRXfX2w4XTk1NMHtPpw,1827
 phoenix/server/api/dataloaders/dataset_example_revisions.py,sha256=i0g8F4akEf3kQOzAvBjO27QwXNsq-kJEM8dtzduxQgY,3720
 phoenix/server/api/dataloaders/dataset_example_spans.py,sha256=-TjdyyJv2c2JiN1OXu6MMmQ-BEKlHXucEDcuObeRVsU,1416
@@ -140,7 +141,7 @@ phoenix/server/api/mutations/dataset_mutations.py,sha256=CuKhxsYfvwVcdN_9EXhKxB6
 phoenix/server/api/mutations/experiment_mutations.py,sha256=vV2lbJ7ccXZqe-LY7nXx6QxWqhKQE4UNZAFcML-KQ8I,3011
 phoenix/server/api/mutations/export_events_mutations.py,sha256=t_wYBxaqvBJYRoHslh3Bmoxmwlzoy0u8SsBKWIKN5hE,4028
 phoenix/server/api/mutations/project_mutations.py,sha256=d_xtYkYfZ5flpVgEkGknKB8rsEux-zZraczzqAs4e8A,2255
-phoenix/server/api/mutations/span_annotations_mutations.py,sha256=DJ1wLlvx7zZJZclw9cFHzrBFtSf98a7grHBb0rd-Hyc,5339
+phoenix/server/api/mutations/span_annotations_mutations.py,sha256=C6sBOe0EbClj6pYSODBsmJ_mrrYtMFaMZkPO0Be-5I0,5410
 phoenix/server/api/mutations/trace_annotations_mutations.py,sha256=GtI_kDpzLDBFjh0CkYjCgVfdBCog7aKHlIVsEM2x92k,5379
 phoenix/server/api/openapi/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 phoenix/server/api/openapi/main.py,sha256=KNutA_7AvV_WlGX8cOkvvDujcJKQ7AD1HT6rTpCpR8A,616
@@ -158,6 +159,7 @@ phoenix/server/api/routers/v1/spans.py,sha256=-nDcr0AjynS-0U6A2nxEQiUTLoNyfzE-3N
 phoenix/server/api/routers/v1/traces.py,sha256=4k_57UICPfRcyQsdPBK-Xo_n6oZ1p41XUwXhZEQpXwE,7568
 phoenix/server/api/routers/v1/utils.py,sha256=xvl2v-BKUkqmFVMmgmmWGFKuRBTrUdoiAeT3mCYEE68,3086
 phoenix/server/api/types/Annotation.py,sha256=7Ym7iuVcbwHlw2yIRylz4nATAF_Cm-Z17qcjiooj1cc,751
+phoenix/server/api/types/AnnotationSummary.py,sha256=8B2LIROqcrPOi8hvYygsblKvSEBfSrysnKOV7F36hgA,1518
 phoenix/server/api/types/AnnotatorKind.py,sha256=rPgGdbN1Gvc109sGQ_ZH-gfJbp93V9wlarzTEJNtUwI,236
 phoenix/server/api/types/Cluster.py,sha256=ac4YfT1OH3xLVmex7EUmB6b9IpULnhLTt554LR0jglE,5689
 phoenix/server/api/types/CreateDatasetPayload.py,sha256=R-6zCmuD0f76RU9Giu78xwTHlASQs6Aq8yzvX1Kxc3g,140
@@ -177,7 +179,7 @@ phoenix/server/api/types/DocumentRetrievalMetrics.py,sha256=amkpC3H5IU5-9GvO0tel
 phoenix/server/api/types/EmbeddingDimension.py,sha256=2OTVzAcfY-2xnJLl7122EAaqJyp0sob1v9-ryaNWknw,18961
 phoenix/server/api/types/EmbeddingMetadata.py,sha256=fJvNNYCbkf3SJalArLy9rcBq9Uj1SNac60zjqe1PFnM,461
 phoenix/server/api/types/Evaluation.py,sha256=oW9UfUCs6Td60kb5s6_sJNvUkxk5TQXjG0pmQ0-gT4M,3298
-phoenix/server/api/types/EvaluationSummary.py,sha256=EFucuzAhcxR9sdEn6WNAtmAGJk-QR_3j3y7Mn2A9_bU,1528
+phoenix/server/api/types/EvaluationSummary.py,sha256=N1DhPXGhBbOOQakF43OGuZ5fl4vye7Uc-HnW5M262F8,1518
 phoenix/server/api/types/Event.py,sha256=XdYgaIxcVIW-YFViCkxj5l9OaVNepyIrCtm5Iqg2le8,3989
 phoenix/server/api/types/EventMetadata.py,sha256=-J0tYF9eZTHwCjwxQHY7Gckr2_MNW5OoWT1mydweZNM,635
 phoenix/server/api/types/ExampleRevisionInterface.py,sha256=gV3Gt9-3Oi5wjaVtepC6nOt3FzTzZFD1KebNnqiw56E,294
@@ -190,11 +192,12 @@ phoenix/server/api/types/ExportedFile.py,sha256=e3GTn7B5LgsTbqiwjhMCQH7VsiqXitrB
 phoenix/server/api/types/Functionality.py,sha256=tzV9xdhB8zqfsjWxP66NDC7EZsplYkYO7jRbLWJIeeg,382
 phoenix/server/api/types/Inferences.py,sha256=BOMlOSsRtUV9XQxpnjmZxdBcJ1w-t9PiFfVOSTS160E,3367
 phoenix/server/api/types/InferencesRole.py,sha256=Kj9aiXOpGhpeg9PHd9MDU7aXVIT28EjJxr4P6xybfzc,601
+phoenix/server/api/types/LabelFraction.py,sha256=zsDxdFALrNiGA1eNykeP8o65gbA0HOhRp54MPH_iRAM,93
 phoenix/server/api/types/MimeType.py,sha256=Zpi6zCalkSFgsvhzvOs-O1gYA04usAi9H__QZUmFlO0,365
 phoenix/server/api/types/Model.py,sha256=BRIzH5xSGiDrAUYvhwDpwxT6--ddS3Xr3vCvP8_vzdo,8051
 phoenix/server/api/types/NumericRange.py,sha256=afEjgF97Go_OvmjMggbPBt-zGM8IONewAyEiKEHRds0,192
 phoenix/server/api/types/PerformanceMetric.py,sha256=W92B7OghEOgzFvmY0LCqpgavHaQggTGshdgfD0yqHX4,350
-phoenix/server/api/types/Project.py,sha256=R2_nart3H4m8QYRbBe-SRnYvIjH4BCipcj_tKe6GaC8,14516
+phoenix/server/api/types/Project.py,sha256=5Tlz06QSGth9atu2gs0l0GrdoLYrH-lWR4sI-RmLoM8,15418
 phoenix/server/api/types/PromptResponse.py,sha256=Q8HKtpp8GpUOcxPCzZpkkokidDd6u0aZOv_SuPZZd5Q,630
 phoenix/server/api/types/Retrieval.py,sha256=OhMK2ncjoyp5h1yjKhjlKpoTbQrMHuxmgSFw-AO1rWw,285
 phoenix/server/api/types/ScalarDriftMetricEnum.py,sha256=IUAcRPpgL41WdoIgK6cNk2Te38SspXGyEs-S1fY23_A,232
@@ -222,15 +225,15 @@ phoenix/server/static/apple-touch-icon-76x76.png,sha256=CT_xT12I0u2i0WU8JzBZBuOQ
 phoenix/server/static/apple-touch-icon.png,sha256=fOfpjqGpWYbJ0eAurKsyoZP1EAs6ZVooBJ_SGk2ZkDs,3801
 phoenix/server/static/favicon.ico,sha256=bY0vvCKRftemZfPShwZtE93DiiQdaYaozkPGwNFr6H8,34494
 phoenix/server/static/modernizr.js,sha256=mvK-XtkNqjOral-QvzoqsyOMECXIMu5BQwSVN_wcU9c,2564
-phoenix/server/static/.vite/manifest.json,sha256=Y5Vxm1VUdmt2c5ANmtD1Z0r2Q1mnApb-UhzLnyJxtU0,1929
-phoenix/server/static/assets/components-Ci5kMOk5.js,sha256=WxnZ4voqivS6S9USAF9zhC5Q2AdvhaJTGFbkTwDLiUI,165385
-phoenix/server/static/assets/index-BQG5WVX7.js,sha256=7RjUXpSAeTK-Q_mLrs4cWscpGyhtSIMI0RhsNn6SwMg,6347
-phoenix/server/static/assets/pages-BrevprVW.js,sha256=GZiR--twaLN-wYV1jvIM-pJiuPbae-HReCjkXNZ5O_U,441239
-phoenix/server/static/assets/vendor-CP0b0YG0.js,sha256=hoxXvVgcHofQrVahK5Q4hAMDaQiH6GygbQvdiXlhQAo,1355423
+phoenix/server/static/.vite/manifest.json,sha256=CeeRBcW2EJ5cdrH84djoZC1WUuNz0fALYFQ384p20Ws,1929
+phoenix/server/static/assets/components-B2sHrnz0.js,sha256=oOzJsP1Fj37O6lXH-hUAnGFWczNcggn7PXzfiyyBPnk,184229
+phoenix/server/static/assets/index-BIFmgnip.js,sha256=GkgpPIvpXXzWVE-1nCSfOovlCRA1EGdKv0pUs-6-AYA,7362
+phoenix/server/static/assets/pages-BTHjBGHq.js,sha256=l0r6E3vq2RzZ__rtPdJ6Edoq3hKHQYXRcy1_M3eCQzc,445413
+phoenix/server/static/assets/vendor-BMWfu6zp.js,sha256=AAVTM5SjGUI_CmAWFUFmhpp5VDhvCD-MrEoh-pXXADY,1355423
 phoenix/server/static/assets/vendor-DxkFTwjz.css,sha256=nZrkr0u6NNElFGvpWHk9GTHeGoibCXCli1bE7mXZGZg,1816
-phoenix/server/static/assets/vendor-arizeai-DTbiPGp6.js,sha256=-o-2f0ByoAbbFXyNrOogVNfdqmXKVMsuxhnuwTJqVLc,296355
-phoenix/server/static/assets/vendor-codemirror-DtdPDzrv.js,sha256=ngcX7xxWUANIvplBuEqkDowqRkknIoX3gXUeQk-jtMQ,503031
-phoenix/server/static/assets/vendor-recharts-A0DA1O99.js,sha256=Zp6oIbsFZPDN-M7VNovxGfJkg9XUISg1fM7eRMtHCaw,282859
+phoenix/server/static/assets/vendor-arizeai-CkyzG9Wl.js,sha256=m1xvoung59IMplKeV-0XmsSPRwu1AJDRxwHbq6BN-oc,296355
+phoenix/server/static/assets/vendor-codemirror-DO3VqEcD.js,sha256=M7t6xd6WpgKes25OOeGyxT1MU1dDrEKdmUBHgy5zslw,503031
+phoenix/server/static/assets/vendor-recharts-BGN0SxgJ.js,sha256=L9LAYSjuf0GHh1_PQh9bF4l9euWCDVQcnQN1RgMDMBw,282859
 phoenix/server/static/assets/vendor-three-DwGkEfCM.js,sha256=0D12ZgKzfKCTSdSTKJBFR2RZO_xxeMXrqDp0AszZqHY,620972
 phoenix/server/templates/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 phoenix/server/templates/index.html,sha256=gVpjB8pCMiubdMh2DA9mTCtV5AVTXJH_9u5PmG2t7Vk,4238
@@ -238,7 +241,7 @@ phoenix/session/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 phoenix/session/client.py,sha256=RSaEALbFu-En5oJLv7K4BmwdkrsIC5ZbjMX7Mlny8_Y,32665
 phoenix/session/data_extractor.py,sha256=gkEM3WWZAlWGMfRgQopAQlid4cSi6GNco-sdrGir0qc,2788
 phoenix/session/evaluation.py,sha256=aKeV8UVOyq3b7CYOwt3cWuLz0xzvMjX7vlEPILJ_fcs,5311
-phoenix/session/session.py,sha256=RstuZpkdZcuyZXhWdY59YOr46wjGsRJObIC5Rr5QRO0,26733
+phoenix/session/session.py,sha256=l10fiotlO1b-SFehcm8N3aVupdFUYYLiLSBrOCJO9as,26911
 phoenix/trace/__init__.py,sha256=ujk_uYjM8gmm-YqnyXxF-kekfwid0bcaPMTtNNcaw6U,407
 phoenix/trace/attributes.py,sha256=B_OrzVaxZwFkrAFXZyicYoIti1UdUysURsvUS2GyW1U,12488
 phoenix/trace/errors.py,sha256=wB1z8qdPckngdfU-TORToekvg3344oNFAA83_hC2yFY,180
@@ -276,8 +279,8 @@ phoenix/utilities/logging.py,sha256=lDXd6EGaamBNcQxL4vP1au9-i_SXe0OraUDiJOcszSw,
 phoenix/utilities/project.py,sha256=8IJuMM4yUMoooPi37sictGj8Etu9rGmq6RFtc9848cQ,436
 phoenix/utilities/re.py,sha256=PDve_OLjRTM8yQQJHC8-n3HdIONi7aNils3ZKRZ5uBM,2045
 phoenix/utilities/span_store.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arize_phoenix-4.16.1.dist-info/METADATA,sha256=4Qd29PvWIh7HEa1xcYXQO_nFiXhuXOPe3f1LZ0qkJiM,11820
-arize_phoenix-4.16.1.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-arize_phoenix-4.16.1.dist-info/licenses/IP_NOTICE,sha256=JBqyyCYYxGDfzQ0TtsQgjts41IJoa-hiwDrBjCb9gHM,469
-arize_phoenix-4.16.1.dist-info/licenses/LICENSE,sha256=HFkW9REuMOkvKRACuwLPT0hRydHb3zNg-fdFt94td18,3794
-arize_phoenix-4.16.1.dist-info/RECORD,,
+arize_phoenix-4.18.0.dist-info/METADATA,sha256=lFZ7DWyKpcAieBqYPFhQ9wmXTGgXajU_pEHvo7msrGc,11829
+arize_phoenix-4.18.0.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+arize_phoenix-4.18.0.dist-info/licenses/IP_NOTICE,sha256=JBqyyCYYxGDfzQ0TtsQgjts41IJoa-hiwDrBjCb9gHM,469
+arize_phoenix-4.18.0.dist-info/licenses/LICENSE,sha256=HFkW9REuMOkvKRACuwLPT0hRydHb3zNg-fdFt94td18,3794
+arize_phoenix-4.18.0.dist-info/RECORD,,

phoenix/server/api/context.py

@@ -8,6 +8,7 @@ from typing_extensions import TypeAlias
 
 from phoenix.core.model_schema import Model
 from phoenix.server.api.dataloaders import (
+    AnnotationSummaryDataLoader,
     AverageExperimentRunLatencyDataLoader,
     CacheForDataLoaders,
     DatasetExampleRevisionsDataLoader,
@@ -44,6 +45,7 @@ class DataLoaders:
     document_evaluation_summaries: DocumentEvaluationSummaryDataLoader
     document_evaluations: DocumentEvaluationsDataLoader
     document_retrieval_metrics: DocumentRetrievalMetricsDataLoader
+    annotation_summaries: AnnotationSummaryDataLoader
     evaluation_summaries: EvaluationSummaryDataLoader
     experiment_annotation_summaries: ExperimentAnnotationSummaryDataLoader
     experiment_error_rates: ExperimentErrorRatesDataLoader

phoenix/server/api/dataloaders/__init__.py

@@ -8,6 +8,7 @@ from phoenix.db.insertion.evaluation import (
 )
 from phoenix.db.insertion.span import ClearProjectSpansEvent, SpanInsertionEvent
 
+from .annotation_summaries import AnnotationSummaryCache, AnnotationSummaryDataLoader
 from .average_experiment_run_latency import AverageExperimentRunLatencyDataLoader
 from .dataset_example_revisions import DatasetExampleRevisionsDataLoader
 from .dataset_example_spans import DatasetExampleSpansDataLoader
@@ -43,6 +44,7 @@ __all__ = [
     "DocumentEvaluationSummaryDataLoader",
     "DocumentEvaluationsDataLoader",
     "DocumentRetrievalMetricsDataLoader",
+    "AnnotationSummaryDataLoader",
     "EvaluationSummaryDataLoader",
     "ExperimentAnnotationSummaryDataLoader",
     "ExperimentErrorRatesDataLoader",
@@ -68,6 +70,9 @@ class CacheForDataLoaders:
     document_evaluation_summary: DocumentEvaluationSummaryCache = field(
         default_factory=DocumentEvaluationSummaryCache,
     )
+    annotation_summary: AnnotationSummaryCache = field(
+        default_factory=AnnotationSummaryCache,
+    )
     evaluation_summary: EvaluationSummaryCache = field(
         default_factory=EvaluationSummaryCache,
     )
@@ -92,6 +97,7 @@ class CacheForDataLoaders:
 
     def _clear_spans(self, project_rowid: int) -> None:
         self._update_spans(project_rowid)
+        self.annotation_summary.invalidate_project(project_rowid)
         self.evaluation_summary.invalidate_project(project_rowid)
         self.document_evaluation_summary.invalidate_project(project_rowid)
 
@@ -113,9 +119,11 @@ class CacheForDataLoaders:
     @invalidate.register
     def _(self, event: SpanEvaluationInsertionEvent) -> None:
         project_rowid, evaluation_name = event
+        self.annotation_summary.invalidate((project_rowid, evaluation_name, "span"))
         self.evaluation_summary.invalidate((project_rowid, evaluation_name, "span"))
 
     @invalidate.register
     def _(self, event: TraceEvaluationInsertionEvent) -> None:
         project_rowid, evaluation_name = event
+        self.annotation_summary.invalidate((project_rowid, evaluation_name, "trace"))
         self.evaluation_summary.invalidate((project_rowid, evaluation_name, "trace"))

phoenix/server/api/dataloaders/annotation_summaries.py

@@ -0,0 +1,146 @@
+from collections import defaultdict
+from datetime import datetime
+from typing import (
+    Any,
+    DefaultDict,
+    List,
+    Literal,
+    Optional,
+    Tuple,
+)
+
+import pandas as pd
+from aioitertools.itertools import groupby
+from cachetools import LFUCache, TTLCache
+from sqlalchemy import Select, func, or_, select
+from strawberry.dataloader import AbstractCache, DataLoader
+from typing_extensions import TypeAlias, assert_never
+
+from phoenix.db import models
+from phoenix.server.api.dataloaders.cache import TwoTierCache
+from phoenix.server.api.input_types.TimeRange import TimeRange
+from phoenix.server.api.types.AnnotationSummary import AnnotationSummary
+from phoenix.server.types import DbSessionFactory
+from phoenix.trace.dsl import SpanFilter
+
+Kind: TypeAlias = Literal["span", "trace"]
+ProjectRowId: TypeAlias = int
+TimeInterval: TypeAlias = Tuple[Optional[datetime], Optional[datetime]]
+FilterCondition: TypeAlias = Optional[str]
+EvalName: TypeAlias = str
+
+Segment: TypeAlias = Tuple[Kind, ProjectRowId, TimeInterval, FilterCondition]
+Param: TypeAlias = EvalName
+
+Key: TypeAlias = Tuple[Kind, ProjectRowId, Optional[TimeRange], FilterCondition, EvalName]
+Result: TypeAlias = Optional[AnnotationSummary]
+ResultPosition: TypeAlias = int
+DEFAULT_VALUE: Result = None
+
+
+def _cache_key_fn(key: Key) -> Tuple[Segment, Param]:
+    kind, project_rowid, time_range, filter_condition, eval_name = key
+    interval = (
+        (time_range.start, time_range.end) if isinstance(time_range, TimeRange) else (None, None)
+    )
+    return (kind, project_rowid, interval, filter_condition), eval_name
+
+
+_Section: TypeAlias = Tuple[ProjectRowId, EvalName, Kind]
+_SubKey: TypeAlias = Tuple[TimeInterval, FilterCondition]
+
+
+class AnnotationSummaryCache(
+    TwoTierCache[Key, Result, _Section, _SubKey],
+):
+    def __init__(self) -> None:
+        super().__init__(
+            # TTL=3600 (1-hour) because time intervals are always moving forward, but
+            # interval endpoints are rounded down to the hour by the UI, so anything
+            # older than an hour most likely won't be a cache-hit anyway.
+            main_cache=TTLCache(maxsize=64 * 32 * 2, ttl=3600),
+            sub_cache_factory=lambda: LFUCache(maxsize=2 * 2),
+        )
+
+    def invalidate_project(self, project_rowid: ProjectRowId) -> None:
+        for section in self._cache.keys():
+            if section[0] == project_rowid:
+                del self._cache[section]
+
+    def _cache_key(self, key: Key) -> Tuple[_Section, _SubKey]:
+        (kind, project_rowid, interval, filter_condition), eval_name = _cache_key_fn(key)
+        return (project_rowid, eval_name, kind), (interval, filter_condition)
+
+
+class AnnotationSummaryDataLoader(DataLoader[Key, Result]):
+    def __init__(
+        self,
+        db: DbSessionFactory,
+        cache_map: Optional[AbstractCache[Key, Result]] = None,
+    ) -> None:
+        super().__init__(
+            load_fn=self._load_fn,
+            cache_key_fn=_cache_key_fn,
+            cache_map=cache_map,
+        )
+        self._db = db
+
+    async def _load_fn(self, keys: List[Key]) -> List[Result]:
+        results: List[Result] = [DEFAULT_VALUE] * len(keys)
+        arguments: DefaultDict[
+            Segment,
+            DefaultDict[Param, List[ResultPosition]],
+        ] = defaultdict(lambda: defaultdict(list))
+        for position, key in enumerate(keys):
+            segment, param = _cache_key_fn(key)
+            arguments[segment][param].append(position)
+        for segment, params in arguments.items():
+            stmt = _get_stmt(segment, *params.keys())
+            async with self._db() as session:
+                data = await session.stream(stmt)
+                async for eval_name, group in groupby(data, lambda row: row.name):
+                    summary = AnnotationSummary(pd.DataFrame(group))
+                    for position in params[eval_name]:
+                        results[position] = summary
+        return results
+
+
+def _get_stmt(
+    segment: Segment,
+    *eval_names: Param,
+) -> Select[Any]:
+    kind, project_rowid, (start_time, end_time), filter_condition = segment
+    stmt = select()
+    if kind == "span":
+        msa = models.SpanAnnotation
+        name_column, label_column, score_column = msa.name, msa.label, msa.score
+        time_column = models.Span.start_time
+        stmt = stmt.join(models.Span).join_from(models.Span, models.Trace)
+        if filter_condition:
+            sf = SpanFilter(filter_condition)
+            stmt = sf(stmt)
+    elif kind == "trace":
+        mta = models.TraceAnnotation
+        name_column, label_column, score_column = mta.name, mta.label, mta.score
+        time_column = models.Trace.start_time
+        stmt = stmt.join(models.Trace)
+    else:
+        assert_never(kind)
+    stmt = stmt.add_columns(
+        name_column,
+        label_column,
+        func.count().label("record_count"),
+        func.count(label_column).label("label_count"),
+        func.count(score_column).label("score_count"),
+        func.sum(score_column).label("score_sum"),
+    )
+    stmt = stmt.group_by(name_column, label_column)
+    stmt = stmt.order_by(name_column, label_column)
+    stmt = stmt.where(models.Trace.project_rowid == project_rowid)
+    stmt = stmt.where(or_(score_column.is_not(None), label_column.is_not(None)))
+    stmt = stmt.where(name_column.in_(eval_names))
+    if start_time:
+        stmt = stmt.where(start_time <= time_column)
+    if end_time:
+        stmt = stmt.where(time_column < end_time)
+    return stmt

phoenix/server/api/mutations/span_annotations_mutations.py

@@ -73,6 +73,7 @@ class SpanAnnotationMutationMixin:
                             models.SpanAnnotation.annotator_kind,
                             annotation.annotator_kind.value
                             if annotation.annotator_kind is not None
+                            and annotation.annotator_kind is not UNSET
                             else None,
                             False,
                         ),

phoenix/server/api/types/AnnotationSummary.py

@@ -0,0 +1,55 @@
+from typing import List, Optional, Union, cast
+
+import pandas as pd
+import strawberry
+from strawberry import Private
+
+from phoenix.db import models
+from phoenix.server.api.types.LabelFraction import LabelFraction
+
+AnnotationType = Union[models.SpanAnnotation, models.TraceAnnotation]
+
+
+@strawberry.type
+class AnnotationSummary:
+    df: Private[pd.DataFrame]
+
+    def __init__(self, dataframe: pd.DataFrame) -> None:
+        self.df = dataframe
+
+    @strawberry.field
+    def count(self) -> int:
+        return cast(int, self.df.record_count.sum())
+
+    @strawberry.field
+    def labels(self) -> List[str]:
+        return self.df.label.dropna().tolist()
+
+    @strawberry.field
+    def label_fractions(self) -> List[LabelFraction]:
+        if not (n := self.df.label_count.sum()):
+            return []
+        return [
+            LabelFraction(
+                label=cast(str, row.label),
+                fraction=row.label_count / n,
+            )
+            for row in self.df.loc[
+                self.df.label.notna(),
+                ["label", "label_count"],
+            ].itertuples()
+        ]
+
+    @strawberry.field
+    def mean_score(self) -> Optional[float]:
+        if not (n := self.df.score_count.sum()):
+            return None
+        return cast(float, self.df.score_sum.sum() / n)
+
+    @strawberry.field
+    def score_count(self) -> int:
+        return cast(int, self.df.score_count.sum())
+
+    @strawberry.field
+    def label_count(self) -> int:
+        return cast(int, self.df.label_count.sum())

phoenix/server/api/types/EvaluationSummary.py

@@ -5,16 +5,11 @@ import strawberry
 from strawberry import Private
 
 from phoenix.db import models
+from phoenix.server.api.types.LabelFraction import LabelFraction
 
 AnnotationType = Union[models.SpanAnnotation, models.TraceAnnotation]
 
 
-@strawberry.type
-class LabelFraction:
-    label: str
-    fraction: float
-
-
 @strawberry.type
 class EvaluationSummary:
     df: Private[pd.DataFrame]

phoenix/server/api/types/LabelFraction.py

@@ -0,0 +1,7 @@
+import strawberry
+
+
+@strawberry.type
+class LabelFraction:
+    label: str
+    fraction: float

phoenix/server/api/types/Project.py

@@ -20,6 +20,7 @@ from phoenix.db import models
 from phoenix.server.api.context import Context
 from phoenix.server.api.input_types.SpanSort import SpanSort, SpanSortConfig
 from phoenix.server.api.input_types.TimeRange import TimeRange
+from phoenix.server.api.types.AnnotationSummary import AnnotationSummary
 from phoenix.server.api.types.DocumentEvaluationSummary import DocumentEvaluationSummary
 from phoenix.server.api.types.EvaluationSummary import EvaluationSummary
 from phoenix.server.api.types.pagination import (
@@ -356,6 +357,29 @@ class Project(Node):
             ("span", self.id_attr, time_range, filter_condition, evaluation_name),
         )
 
+    @strawberry.field
+    async def trace_annotation_summary(
+        self,
+        info: Info[Context, None],
+        evaluation_name: str,
+        time_range: Optional[TimeRange] = UNSET,
+    ) -> Optional[AnnotationSummary]:
+        return await info.context.data_loaders.annotation_summaries.load(
+            ("trace", self.id_attr, time_range, None, evaluation_name),
+        )
+
+    @strawberry.field
+    async def span_annotation_summary(
+        self,
+        info: Info[Context, None],
+        evaluation_name: str,
+        time_range: Optional[TimeRange] = UNSET,
+        filter_condition: Optional[str] = UNSET,
+    ) -> Optional[AnnotationSummary]:
+        return await info.context.data_loaders.annotation_summaries.load(
+            ("span", self.id_attr, time_range, filter_condition, evaluation_name),
+        )
+
     @strawberry.field
     async def document_evaluation_summary(
         self,

phoenix/server/app.py

@@ -58,6 +58,7 @@ from phoenix.exceptions import PhoenixMigrationError
 from phoenix.pointcloud.umap_parameters import UMAPParameters
 from phoenix.server.api.context import Context, DataLoaders
 from phoenix.server.api.dataloaders import (
+    AnnotationSummaryDataLoader,
     AverageExperimentRunLatencyDataLoader,
     CacheForDataLoaders,
     DatasetExampleRevisionsDataLoader,
@@ -286,6 +287,12 @@ def create_graphql_router(
                 ),
                 document_evaluations=DocumentEvaluationsDataLoader(db),
                 document_retrieval_metrics=DocumentRetrievalMetricsDataLoader(db),
+                annotation_summaries=AnnotationSummaryDataLoader(
+                    db,
+                    cache_map=cache_for_dataloaders.annotation_summary
+                    if cache_for_dataloaders
+                    else None,
+                ),
                 evaluation_summaries=EvaluationSummaryDataLoader(
                     db,
                     cache_map=cache_for_dataloaders.evaluation_summary

phoenix/server/static/.vite/manifest.json

@@ -1,32 +1,32 @@
 {
-  "_components-Ci5kMOk5.js": {
-    "file": "assets/components-Ci5kMOk5.js",
+  "_components-B2sHrnz0.js": {
+    "file": "assets/components-B2sHrnz0.js",
     "name": "components",
     "imports": [
-      "_vendor-CP0b0YG0.js",
-      "_vendor-arizeai-DTbiPGp6.js",
-      "_pages-BrevprVW.js",
+      "_vendor-BMWfu6zp.js",
+      "_vendor-arizeai-CkyzG9Wl.js",
+      "_pages-BTHjBGHq.js",
       "_vendor-three-DwGkEfCM.js",
-      "_vendor-codemirror-DtdPDzrv.js"
+      "_vendor-codemirror-DO3VqEcD.js"
     ]
   },
-  "_pages-BrevprVW.js": {
-    "file": "assets/pages-BrevprVW.js",
+  "_pages-BTHjBGHq.js": {
+    "file": "assets/pages-BTHjBGHq.js",
     "name": "pages",
     "imports": [
-      "_vendor-CP0b0YG0.js",
-      "_components-Ci5kMOk5.js",
-      "_vendor-arizeai-DTbiPGp6.js",
-      "_vendor-recharts-A0DA1O99.js",
-      "_vendor-codemirror-DtdPDzrv.js"
+      "_vendor-BMWfu6zp.js",
+      "_components-B2sHrnz0.js",
+      "_vendor-arizeai-CkyzG9Wl.js",
+      "_vendor-recharts-BGN0SxgJ.js",
+      "_vendor-codemirror-DO3VqEcD.js"
     ]
   },
   "_vendor-!~{003}~.js": {
     "file": "assets/vendor-DxkFTwjz.css",
     "src": "_vendor-!~{003}~.js"
   },
-  "_vendor-CP0b0YG0.js": {
-    "file": "assets/vendor-CP0b0YG0.js",
+  "_vendor-BMWfu6zp.js": {
+    "file": "assets/vendor-BMWfu6zp.js",
     "name": "vendor",
     "imports": [
       "_vendor-three-DwGkEfCM.js"
@@ -35,25 +35,25 @@
       "assets/vendor-DxkFTwjz.css"
     ]
   },
-  "_vendor-arizeai-DTbiPGp6.js": {
-    "file": "assets/vendor-arizeai-DTbiPGp6.js",
+  "_vendor-arizeai-CkyzG9Wl.js": {
+    "file": "assets/vendor-arizeai-CkyzG9Wl.js",
     "name": "vendor-arizeai",
     "imports": [
-      "_vendor-CP0b0YG0.js"
+      "_vendor-BMWfu6zp.js"
     ]
   },
-  "_vendor-codemirror-DtdPDzrv.js": {
-    "file": "assets/vendor-codemirror-DtdPDzrv.js",
+  "_vendor-codemirror-DO3VqEcD.js": {
+    "file": "assets/vendor-codemirror-DO3VqEcD.js",
     "name": "vendor-codemirror",
     "imports": [
-      "_vendor-CP0b0YG0.js"
+      "_vendor-BMWfu6zp.js"
     ]
   },
-  "_vendor-recharts-A0DA1O99.js": {
-    "file": "assets/vendor-recharts-A0DA1O99.js",
+  "_vendor-recharts-BGN0SxgJ.js": {
+    "file": "assets/vendor-recharts-BGN0SxgJ.js",
     "name": "vendor-recharts",
     "imports": [
-      "_vendor-CP0b0YG0.js"
+      "_vendor-BMWfu6zp.js"
     ]
   },
   "_vendor-three-DwGkEfCM.js": {
@@ -61,18 +61,18 @@
     "name": "vendor-three"
   },
   "index.tsx": {
-    "file": "assets/index-BQG5WVX7.js",
+    "file": "assets/index-BIFmgnip.js",
     "name": "index",
     "src": "index.tsx",
     "isEntry": true,
     "imports": [
-      "_vendor-CP0b0YG0.js",
-      "_vendor-arizeai-DTbiPGp6.js",
-      "_pages-BrevprVW.js",
-      "_components-Ci5kMOk5.js",
+      "_vendor-BMWfu6zp.js",
+      "_vendor-arizeai-CkyzG9Wl.js",
+      "_components-B2sHrnz0.js",
+      "_pages-BTHjBGHq.js",
       "_vendor-three-DwGkEfCM.js",
-      "_vendor-recharts-A0DA1O99.js",
-      "_vendor-codemirror-DtdPDzrv.js"
+      "_vendor-codemirror-DO3VqEcD.js",
+      "_vendor-recharts-BGN0SxgJ.js"
     ]
   }
 }