arize-phoenix 5.11.0__py3-none-any.whl → 6.0.0__py3-none-any.whl

{arize_phoenix-5.11.0.dist-info → arize_phoenix-6.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: arize-phoenix
-Version: 5.11.0
+Version: 6.0.0
 Summary: AI Observability and Evaluation
 Project-URL: Documentation, https://docs.arize.com/phoenix/
 Project-URL: Issues, https://github.com/Arize-ai/phoenix/issues

{arize_phoenix-5.11.0.dist-info → arize_phoenix-6.0.0.dist-info}/RECORD

@@ -6,7 +6,7 @@ phoenix/exceptions.py,sha256=n2L2KKuecrdflB9MsCdAYCiSEvGJptIsfRkXMoJle7A,169
 phoenix/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 phoenix/services.py,sha256=kpW1WL0kiB8XJsO6XycvZVJ-lBkNoenhQ7atCvBoSe8,5365
 phoenix/settings.py,sha256=ht-0oN-sMV6SPXrk7Tu1EZlngpAYkGNLYPhO8DyrdQI,661
-phoenix/version.py,sha256=6_DlfgAX4MBZnyPuaEthIkiqPKB1Zz3_f4Z2L40k-Ws,23
+phoenix/version.py,sha256=eSyQRXyZR8d0tJQPRLwDd_1tXyRgb7TUvtlI-hfN620,22
 phoenix/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 phoenix/core/embedding_dimension.py,sha256=zKGbcvwOXgLf-yrJBpQyKtd-LEOPRKHnUToyAU8Owis,87
 phoenix/core/model.py,sha256=qBFraOtmwCCnWJltKNP18DDG0mULXigytlFsa6YOz6k,4837
@@ -47,7 +47,7 @@ phoenix/experiments/utils.py,sha256=MZ1-OnTcavk_KUtbfGqt55Fk9TGtJpYG_K71WsN-zDk,
 phoenix/experiments/evaluators/__init__.py,sha256=j63fi3fa3U7-itVPHa82GowhjQRU-wO6yhO34u_lhsA,714
 phoenix/experiments/evaluators/base.py,sha256=zefFLqyYLMxJnZxDs1S0QCrnb43YJraoadCMqRMU72A,5576
 phoenix/experiments/evaluators/code_evaluators.py,sha256=JIu_8GezA3UKA7isUmZvBxjcJeXllIDESEgdF2zeDck,6755
-phoenix/experiments/evaluators/llm_evaluators.py,sha256=6pvCWP1a3tLMyufaU_U8TkItJO2Q03hkEf7kcI8gdGY,20549
+phoenix/experiments/evaluators/llm_evaluators.py,sha256=1HsZQsKNWA_19EHrjI6OXqm1aBwvIKS-qHYc3R0bOqc,20551
 phoenix/experiments/evaluators/utils.py,sha256=W-k2bHL67ST-B7ne8t5LvCFHGBaKCPHbOv3x1SlDENU,9330
 phoenix/inferences/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 phoenix/inferences/errors.py,sha256=AOsCs-xR1LV469Gygje16EOK3qGheiPGW0Ylg-MakhQ,8282
@@ -94,7 +94,7 @@ phoenix/server/api/exceptions.py,sha256=TA0JuY2YRnj35qGuMSQ8d0ToHum9gWm9W--3fSKH
 phoenix/server/api/interceptor.py,sha256=ykDnoC_apUd-llVli3m1CW18kNSIgjz2qZ6m5JmPDu8,1294
 phoenix/server/api/queries.py,sha256=4KJz8TUz3VUTup9MDjr_GoKX0SttWSvHBq2ncWZGxf8,27343
 phoenix/server/api/schema.py,sha256=tHyw2jTbue_-gu0fe9Sw7LUYtzJUCwp9SvccDgOkNPw,1696
-phoenix/server/api/subscriptions.py,sha256=HPb5agJPMPBM_v_sHjiwe7hwMAHsBsrKZZMLq5bxhcw,22062
+phoenix/server/api/subscriptions.py,sha256=nsiByh2rTPc1PS7OrPyuyyfFfYUUXeGmai19IfYP0dA,23201
 phoenix/server/api/utils.py,sha256=quCBRcusc6PUq9tJq7M8PgwFZp7nXgVAxtbw8feribY,833
 phoenix/server/api/dataloaders/__init__.py,sha256=jNYvfXjnZzgA2HWTG7AZdqWGla3ZysBUDUei8Zkz6N8,3290
 phoenix/server/api/dataloaders/annotation_summaries.py,sha256=2sHmIDX7n8tuPeBTs9bMKtlMKWn_Ph9awTZqmwn2Owc,5505
@@ -124,10 +124,10 @@ phoenix/server/api/dataloaders/users.py,sha256=Uh86Kny_xpqDdEvEklcTBUA_MELgu4Z0j
 phoenix/server/api/dataloaders/cache/__init__.py,sha256=SYoOM9n8FJaMdQarma5d1blu-jIg2GB8Shqg5ezSzZ8,106
 phoenix/server/api/dataloaders/cache/two_tier_cache.py,sha256=cmo8FUT3E91R139IEzh4yCga-6nTamc5KPXAfMrzNDM,2315
 phoenix/server/api/helpers/__init__.py,sha256=m2-xaSPqUiSs91k62JaRDjFNfl-1byxBfY-m_Vxw16U,272
-phoenix/server/api/helpers/dataset_helpers.py,sha256=14mldZp9to3rr9BdvvoFqEwZHHV_k2e7jPm8q9z2OdQ,6896
-phoenix/server/api/helpers/playground_clients.py,sha256=fTWY883qVNCK_IkiYm1yxFx3GaXMWhalMeV0zYzIyUM,37407
+phoenix/server/api/helpers/dataset_helpers.py,sha256=AMlKY9_e0wnTrTSSQemM5NHfnpwARSytx-m9YK6f6bY,8421
+phoenix/server/api/helpers/playground_clients.py,sha256=zgpYRn7c8Yi6-0ZW7miKTDKnKor9nq4U40-mjgtqIGY,36164
 phoenix/server/api/helpers/playground_registry.py,sha256=CPLMziFB2wmr-dfbx7VbzO2f8YIG_k5RftzvGXYGQ1w,2570
-phoenix/server/api/helpers/playground_spans.py,sha256=LJjndkVxr6DsfVFerrtMZWlRd5YL5AIbcM2CvFe8ajc,16489
+phoenix/server/api/helpers/playground_spans.py,sha256=qGk7V7IZK7EkRE1mvZyROpLN5kgOahOZifFzUWmqYFc,16546
 phoenix/server/api/input_types/AddExamplesToDatasetInput.py,sha256=mIQz0S_z8YdrktKIY6RCvtNJ2yZF9pYvTGgasUsI-54,430
 phoenix/server/api/input_types/AddSpansToDatasetInput.py,sha256=-StSstyMAVrba3tG1U30b-srkKCtu_svflQuSM19iJA,362
 phoenix/server/api/input_types/ChatCompletionInput.py,sha256=g_5ARuwylt-uCVAsGyZPEVtidEQiOhbKakvDQsZumzw,1451
@@ -164,8 +164,8 @@ phoenix/server/api/input_types/UserRoleInput.py,sha256=xxhFe0ITZOgRVEJbVem_W6F1I
 phoenix/server/api/input_types/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 phoenix/server/api/mutations/__init__.py,sha256=1wClieLNA3_Tin4Ah67rkrQvwSSZAdPU0EPsRiUxyAA,1103
 phoenix/server/api/mutations/api_key_mutations.py,sha256=OmPipsmlQIb6DKvAgO58mZUwkYJihlJB2N4lTyeUlAA,6164
-phoenix/server/api/mutations/chat_mutations.py,sha256=Rky_PwILAhDBjW6K2cxfaIVsT5iOP5pjuViYUpoVHWY,22540
-phoenix/server/api/mutations/dataset_mutations.py,sha256=siwsSmozKRIK8ZhPrfWl-GxKCL4lAmRPms862tG2KXY,27064
+phoenix/server/api/mutations/chat_mutations.py,sha256=YUZGgoUoA-652UVo5p7qWNnN4u7tdU6XfhDnTIxvbqU,22831
+phoenix/server/api/mutations/dataset_mutations.py,sha256=OkVek-GtDf_1eloI7Sdz6W6l5FFmwU_41LJM8-fw3q4,26088
 phoenix/server/api/mutations/experiment_mutations.py,sha256=p3CoLAa8nFPa3D759Y2A7De_PVJNGOL98mA3HoZBrRQ,3188
 phoenix/server/api/mutations/export_events_mutations.py,sha256=xoDnVWC7eA_8wNQP0-oyiHojyUZ0EhVVSrsAnztetC0,3993
 phoenix/server/api/mutations/project_mutations.py,sha256=KdcA3c6tgLZ8QhpIv37qJESTBPxs56SbfeYdDKzU7-8,2263
@@ -182,7 +182,7 @@ phoenix/server/api/routers/oauth2.py,sha256=bSrTZAAWW4WgZVwkr39xbo5jZEYL4w4wCbEe
 phoenix/server/api/routers/utils.py,sha256=M41BoH-fl37izhRuN2aX7lWm7jOC20A_3uClv9TVUUY,583
 phoenix/server/api/routers/v1/__init__.py,sha256=aLEHzzU8kQo4Oqsv2an35lH5VYUxAZQrcG7CXZA_Lx4,2214
 phoenix/server/api/routers/v1/datasets.py,sha256=tNh0CxAvSkWh-_5AwisGN1degQlUNGU3uufGa7MIbOw,36985
-phoenix/server/api/routers/v1/evaluations.py,sha256=g-Z5Dk33Epk7WrxlCyFsHHwneHvI2KI7x4Tb5nGw9m4,12648
+phoenix/server/api/routers/v1/evaluations.py,sha256=RpOkTylp5Da6BvPZGuN8ksnxz_BVXRIwyOvwX9Iko8U,12647
 phoenix/server/api/routers/v1/experiment_evaluations.py,sha256=1XuKqE5F6CC6R4kF4g4YdBtcYvch1rCk8Am5CGyqfJ4,4838
 phoenix/server/api/routers/v1/experiment_runs.py,sha256=xKXhol1-G4zAFECAQK9lAjKtSJsvB0Bp6bAXJYqJ7eI,6387
 phoenix/server/api/routers/v1/experiments.py,sha256=sBxOIE4k4eIt-sQ6lPo2HUv9Tf1aURXtTJTq8J-Rfto,11808
@@ -241,10 +241,10 @@ phoenix/server/api/types/Retrieval.py,sha256=OhMK2ncjoyp5h1yjKhjlKpoTbQrMHuxmgSF
 phoenix/server/api/types/ScalarDriftMetricEnum.py,sha256=IUAcRPpgL41WdoIgK6cNk2Te38SspXGyEs-S1fY23_A,232
 phoenix/server/api/types/Segments.py,sha256=vT2v0efoa5cuBKxLtxTnsUP5YJJCZfTloM71Spu0tMI,2915
 phoenix/server/api/types/SortDir.py,sha256=OUpXhlCzCxPoXSDkJJygEs9Rw9pMymfaZUG5zPTrw4Y,152
-phoenix/server/api/types/Span.py,sha256=6GS6MpJ3f8P2LrQUe2TWPrPf7ENxmde_wisQkJguphw,16919
+phoenix/server/api/types/Span.py,sha256=wJGvenSL6d1oDmpJt4hsLjIVTxmemJBRCBKHy4rYztc,15355
 phoenix/server/api/types/SpanAnnotation.py,sha256=6b5G-b_OoRvDL2ayWk7MkbqarLK-F-pQMx21CpUuNGY,1168
 phoenix/server/api/types/SystemApiKey.py,sha256=2ym8EgsTBIvxx1l9xZ-2YMovz58ZwYb_MaHBTJ9NH2E,166
-phoenix/server/api/types/TemplateLanguage.py,sha256=9yxW3zGXgHPnA35svT4tznDyRKGuaz_WlbcpiUtC7Ec,142
+phoenix/server/api/types/TemplateLanguage.py,sha256=6j_0uwO_GZIeCpR7sTOnxySXudT7qBSC6LFsjzbvW1o,160
 phoenix/server/api/types/TimeSeries.py,sha256=IIeGVRFdSMozYXxPg736DW_mKvj4-3WjYSYEnn4UEJc,5241
 phoenix/server/api/types/Trace.py,sha256=1RrdEedlPpNmWkQwosOZ81fabc9-B6PDHTYpr-hZj-Y,3240
 phoenix/server/api/types/TraceAnnotation.py,sha256=OW6A2zr1gomOuG0XQe55dk15XXX2DSM0DzatRbHWH5A,1256
@@ -273,15 +273,15 @@ phoenix/server/static/apple-touch-icon-76x76.png,sha256=CT_xT12I0u2i0WU8JzBZBuOQ
 phoenix/server/static/apple-touch-icon.png,sha256=fOfpjqGpWYbJ0eAurKsyoZP1EAs6ZVooBJ_SGk2ZkDs,3801
 phoenix/server/static/favicon.ico,sha256=bY0vvCKRftemZfPShwZtE93DiiQdaYaozkPGwNFr6H8,34494
 phoenix/server/static/modernizr.js,sha256=mvK-XtkNqjOral-QvzoqsyOMECXIMu5BQwSVN_wcU9c,2564
-phoenix/server/static/.vite/manifest.json,sha256=QVqqb_JM1hcyubZgGH4qh08LUHWKE_FR43j_YdVWk0M,1929
-phoenix/server/static/assets/components-C_HASv83.js,sha256=7xZ3sMiHZkpbHJjpzujdMII4znFs-h5DkFCvY8oX1Aw,306769
-phoenix/server/static/assets/index-D7UiCRtr.js,sha256=i6WbtADGyqEQix4x9_4yIbl1U7dc0NpIRoI5ckFGUZ8,7290
-phoenix/server/static/assets/pages-DYHcAdjT.js,sha256=aUloiI7AopSeUGfKhYmalMk1luUrQpMJYHnjZwLV9vs,632400
-phoenix/server/static/assets/vendor-BCxsh5i3.js,sha256=fPeA9hYHGj1n2Bg8JOlpsddXw7Vkulbvra3z4RCy0Ss,10899238
+phoenix/server/static/.vite/manifest.json,sha256=I8csmeLbMfXrOtIwIEqwR8H9yF364xgi1vt-F7T2DvE,1929
+phoenix/server/static/assets/components-Bo-xjXoV.js,sha256=-zGBOrc-bOd4e9dKbaw__9lCm8wNu1W-UROzW9UoXR4,312092
+phoenix/server/static/assets/index-i2KbnOZd.js,sha256=E8LWCCrBJl_AwuE2eVr3JU-tzQM9u9GWRBgjBXptjMc,8305
+phoenix/server/static/assets/pages-BfYVoiUE.js,sha256=UKyvB-BB5eyPwrYLIDc1ZN_HsdSs_7CnB7Q_5ddtixk,633454
 phoenix/server/static/assets/vendor-DxkFTwjz.css,sha256=nZrkr0u6NNElFGvpWHk9GTHeGoibCXCli1bE7mXZGZg,1816
-phoenix/server/static/assets/vendor-arizeai-C2CDZgMz.js,sha256=Isrlf-QaErzG0UZGd2VeCf1l-d6eghpvLr4v4K5A2Xk,308483
-phoenix/server/static/assets/vendor-codemirror-DYbtnCTn.js,sha256=-xGEPOe20e89kdfMNfTrthj084muiJ3gMKPdUtzdjmA,392709
-phoenix/server/static/assets/vendor-recharts-P6W8G0Mb.js,sha256=0B8yyQbqMwwsPvLkd-tZzntAKMNZpZGJ8OlMfXKvbu4,282859
+phoenix/server/static/assets/vendor-UNccFYqq.js,sha256=vzJ1ZI0VHzIhXJyu35Z8nG2iFAk2oq_u_1IE_h4ZovY,10899274
+phoenix/server/static/assets/vendor-arizeai-gSIqhzZY.js,sha256=p5nkI7UHuepmzGlED1HiNeSRCFKtZx4yD9i4C5NC6po,308489
+phoenix/server/static/assets/vendor-codemirror-fSYjW3f-.js,sha256=R0vjkRdXIPclHQMbu_0vy9EUfMwOYoH7iejIOgl3CT4,392709
+phoenix/server/static/assets/vendor-recharts-CVVyA2X0.js,sha256=ApyczsmduDBWHOZ0KyZqKT7rL3fDMtCyzor77olD9MA,282859
 phoenix/server/static/assets/vendor-three-DwGkEfCM.js,sha256=0D12ZgKzfKCTSdSTKJBFR2RZO_xxeMXrqDp0AszZqHY,620972
 phoenix/server/templates/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 phoenix/server/templates/index.html,sha256=ram6sfy2obf_F053ay35V30v-mnRWZ86rK-PstXLy1c,4457
@@ -321,10 +321,10 @@ phoenix/utilities/logging.py,sha256=NKvX43jWITgSTtHGCi6l5xp8jTjtQ8FQ_irEi9ybOxg,
 phoenix/utilities/project.py,sha256=auVpARXkDb-JgeX5f2aStyFIkeKvGwN9l7qrFeJMVxI,445
 phoenix/utilities/re.py,sha256=x8Xbk-Wa6qDMAtUd_7JtZvKtrYEuMY-bchB0n163_5c,2006
 phoenix/utilities/span_store.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-phoenix/utilities/template_formatters.py,sha256=JuOyvukMPLDHa1uVNw0kCFBUnIxy02dwAWNZimdIZU4,2423
-arize_phoenix-5.11.0.dist-info/METADATA,sha256=6ALW-ZaTA2y2Temcz899qgQfCOjqJVGx9A2jgWPkkOI,22671
-arize_phoenix-5.11.0.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
-arize_phoenix-5.11.0.dist-info/entry_points.txt,sha256=Pgpn8Upxx9P8z8joPXZWl2LlnAlGc3gcQoVchb06X1Q,94
-arize_phoenix-5.11.0.dist-info/licenses/IP_NOTICE,sha256=JBqyyCYYxGDfzQ0TtsQgjts41IJoa-hiwDrBjCb9gHM,469
-arize_phoenix-5.11.0.dist-info/licenses/LICENSE,sha256=HFkW9REuMOkvKRACuwLPT0hRydHb3zNg-fdFt94td18,3794
-arize_phoenix-5.11.0.dist-info/RECORD,,
+phoenix/utilities/template_formatters.py,sha256=gh9PJD6WEGw7TEYXfSst1UR4pWWwmjxMLrDVQ_CkpkQ,2779
+arize_phoenix-6.0.0.dist-info/METADATA,sha256=0q5IT7ux_4-FeqEvPjOWWki0udeqWvINeKA39rlC-BA,22670
+arize_phoenix-6.0.0.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
+arize_phoenix-6.0.0.dist-info/entry_points.txt,sha256=Pgpn8Upxx9P8z8joPXZWl2LlnAlGc3gcQoVchb06X1Q,94
+arize_phoenix-6.0.0.dist-info/licenses/IP_NOTICE,sha256=JBqyyCYYxGDfzQ0TtsQgjts41IJoa-hiwDrBjCb9gHM,469
+arize_phoenix-6.0.0.dist-info/licenses/LICENSE,sha256=HFkW9REuMOkvKRACuwLPT0hRydHb3zNg-fdFt94td18,3794
+arize_phoenix-6.0.0.dist-info/RECORD,,

phoenix/experiments/evaluators/llm_evaluators.py

@@ -232,7 +232,7 @@ class RelevanceEvaluator(LLMEvaluator):
         "query. First, explain step-by-step why you think the text is or is not relevant. "
         "Then provide a single word label; 'true' if the text is relevant or 'false' if the text "
         "is not relevant. "
-        "Here is an example template for your reponse:\n\n"
+        "Here is an example template for your response:\n\n"
         "CRITERIA: the response is 'relevant' to the query\n"
         "QUERY: *text that contains a query*\n"
         "RESPONSE: *a response that may or may not be relevant to the query*\n"
@@ -367,7 +367,7 @@ class LLMRelationalEvaluator(LLMEvaluator):
         "First, explain step-by-step why you think the response '{relation}' the reference. "
         "Then provide a single word label; 'true' if the response '{relation}' the reference or "
         "'false' if the text is not '{relation}' to the reference. "
-        "Here is an example template for your reponse:\n\n"
+        "Here is an example template for your response:\n\n"
         "CRITERIA: the response '{relation}' the reference\n"
         "REFERENCE: *text that contains a reference*\n"
         "RESPONSE: *a response that may or may not be '{relation}' to the reference*\n"

phoenix/server/api/helpers/dataset_helpers.py

@@ -1,54 +1,43 @@
 import json
 from collections.abc import Mapping
-from typing import Any, Literal, Optional, Protocol
+from typing import Any, Literal, Optional
 
 from openinference.semconv.trace import (
     MessageAttributes,
     OpenInferenceMimeTypeValues,
     OpenInferenceSpanKindValues,
+    SpanAttributes,
     ToolCallAttributes,
 )
 
+from phoenix.db.models import Span
 from phoenix.trace.attributes import get_attribute_value
 
 
-class HasSpanIO(Protocol):
-    """
-    An interface that contains the information needed to extract dataset example
-    input and output values from a span.
-    """
-
-    span_kind: Optional[str]
-    input_value: Any
-    input_mime_type: Optional[str]
-    output_value: Any
-    output_mime_type: Optional[str]
-    llm_prompt_template_variables: Any
-    llm_input_messages: Any
-    llm_output_messages: Any
-    retrieval_documents: Any
-
-
-def get_dataset_example_input(span: HasSpanIO) -> dict[str, Any]:
+def get_dataset_example_input(span: Span) -> dict[str, Any]:
     """
     Extracts the input value from a span and returns it as a dictionary. Input
     values from LLM spans are extracted from the input messages and prompt
     template variables (if present). For other span kinds, the input is
     extracted from the input value and input mime type attributes.
     """
-    input_value = span.input_value
-    input_mime_type = span.input_mime_type
-    if span.span_kind == OpenInferenceSpanKindValues.LLM.value:
+    span_kind = span.span_kind
+    attributes = span.attributes
+    input_value = get_attribute_value(attributes, INPUT_VALUE)
+    input_mime_type = get_attribute_value(attributes, INPUT_MIME_TYPE)
+    prompt_template_variables = get_attribute_value(attributes, LLM_PROMPT_TEMPLATE_VARIABLES)
+    input_messages = get_attribute_value(attributes, LLM_INPUT_MESSAGES)
+    if span_kind == LLM:
         return _get_llm_span_input(
-            input_messages=span.llm_input_messages,
+            input_messages=input_messages,
             input_value=input_value,
             input_mime_type=input_mime_type,
-            prompt_template_variables=span.llm_prompt_template_variables,
+            prompt_template_variables=prompt_template_variables,
         )
     return _get_generic_io_value(io_value=input_value, mime_type=input_mime_type, kind="input")
 
 
-def get_dataset_example_output(span: HasSpanIO) -> dict[str, Any]:
+def get_dataset_example_output(span: Span) -> dict[str, Any]:
     """
     Extracts the output value from a span and returns it as a dictionary. Output
     values from LLM spans are extracted from the output messages (if present).
@@ -56,18 +45,21 @@ def get_dataset_example_output(span: HasSpanIO) -> dict[str, Any]:
     present). For other span kinds, the output is extracted from the output
     value and output mime type attributes.
     """
-
-    output_value = span.output_value
-    output_mime_type = span.output_mime_type
-    if (span_kind := span.span_kind) == OpenInferenceSpanKindValues.LLM.value:
+    span_kind = span.span_kind
+    attributes = span.attributes
+    output_value = get_attribute_value(attributes, OUTPUT_VALUE)
+    output_mime_type = get_attribute_value(attributes, OUTPUT_MIME_TYPE)
+    output_messages = get_attribute_value(attributes, LLM_OUTPUT_MESSAGES)
+    retrieval_documents = get_attribute_value(attributes, RETRIEVAL_DOCUMENTS)
+    if span_kind == LLM:
         return _get_llm_span_output(
-            output_messages=span.llm_output_messages,
+            output_messages=output_messages,
             output_value=output_value,
             output_mime_type=output_mime_type,
         )
     if span_kind == OpenInferenceSpanKindValues.RETRIEVER.value:
         return _get_retriever_span_output(
-            retrieval_documents=span.retrieval_documents,
+            retrieval_documents=retrieval_documents,
             output_value=output_value,
             output_mime_type=output_mime_type,
         )
@@ -90,8 +82,8 @@ def _get_llm_span_input(
         input["messages"] = messages
     if not input:
         input = _get_generic_io_value(io_value=input_value, mime_type=input_mime_type, kind="input")
-    if prompt_template_variables:
-        input = {**input, "prompt_template_variables": prompt_template_variables}
+    if prompt_template_variables_data := _safely_json_decode(prompt_template_variables):
+        input["prompt_template_variables"] = prompt_template_variables_data
     return input
 
 
@@ -118,7 +110,7 @@ def _get_retriever_span_output(
     Extracts the output value from a retriever span and returns it as a dictionary.
     The output is extracted from the retrieval documents (if present).
     """
-    if retrieval_documents is not None:
+    if (retrieval_documents := _parse_retrieval_documents(retrieval_documents)) is not None:
         return {"documents": retrieval_documents}
     return _get_generic_io_value(io_value=output_value, mime_type=output_mime_type, kind="output")
 
@@ -130,12 +122,14 @@ def _get_generic_io_value(
     Makes a best-effort attempt to extract the input or output value from a span
     and returns it as a dictionary.
     """
-    if mime_type == OpenInferenceMimeTypeValues.JSON.value:
-        parsed_value = json.loads(io_value)
-        if isinstance(parsed_value, dict):
-            return parsed_value
+    if (
+        mime_type == OpenInferenceMimeTypeValues.JSON.value
+        and (io_value_data := _safely_json_decode(io_value)) is not None
+    ):
+        if isinstance(io_value_data, dict):
+            return io_value_data
         else:
-            return {kind: parsed_value}
+            return {kind: io_value_data}
     if isinstance(io_value, str):
         return {kind: io_value}
     return {}
@@ -169,6 +163,35 @@ def _get_message(message: Mapping[str, Any]) -> dict[str, Any]:
     }
 
 
+def _parse_retrieval_documents(retrieval_documents: Any) -> Optional[list[dict[str, Any]]]:
+    """
+    Safely un-nests a list of retrieval documents.
+
+    Example: [{"document": {"content": "..."}}] -> [{"content": "..."}]
+    """
+    if not isinstance(retrieval_documents, list):
+        return None
+    docs = []
+    for retrieval_doc in retrieval_documents:
+        if not isinstance(retrieval_doc, dict) or not (doc := retrieval_doc.get("document")):
+            return None
+        docs.append(doc)
+    return docs
+
+
+def _safely_json_decode(value: Any) -> Any:
+    """
+    Safely decodes a JSON-encoded value.
+    """
+    if not isinstance(value, str):
+        return None
+    try:
+        return json.loads(value)
+    except json.JSONDecodeError:
+        return None
+
+
+# MessageAttributes
 MESSAGE_CONTENT = MessageAttributes.MESSAGE_CONTENT
 MESSAGE_FUNCTION_CALL_ARGUMENTS_JSON = MessageAttributes.MESSAGE_FUNCTION_CALL_ARGUMENTS_JSON
 MESSAGE_FUNCTION_CALL_NAME = MessageAttributes.MESSAGE_FUNCTION_CALL_NAME
@@ -176,5 +199,19 @@ MESSAGE_NAME = MessageAttributes.MESSAGE_NAME
 MESSAGE_ROLE = MessageAttributes.MESSAGE_ROLE
 MESSAGE_TOOL_CALLS = MessageAttributes.MESSAGE_TOOL_CALLS
 
-TOOL_CALL_FUNCTION_NAME = ToolCallAttributes.TOOL_CALL_FUNCTION_NAME
+# OpenInferenceSpanKindValues
+LLM = OpenInferenceSpanKindValues.LLM.value
+
+# SpanAttributes
+INPUT_MIME_TYPE = SpanAttributes.INPUT_MIME_TYPE
+INPUT_VALUE = SpanAttributes.INPUT_VALUE
+LLM_INPUT_MESSAGES = SpanAttributes.LLM_INPUT_MESSAGES
+LLM_OUTPUT_MESSAGES = SpanAttributes.LLM_OUTPUT_MESSAGES
+LLM_PROMPT_TEMPLATE_VARIABLES = SpanAttributes.LLM_PROMPT_TEMPLATE_VARIABLES
+OUTPUT_MIME_TYPE = SpanAttributes.OUTPUT_MIME_TYPE
+OUTPUT_VALUE = SpanAttributes.OUTPUT_VALUE
+RETRIEVAL_DOCUMENTS = SpanAttributes.RETRIEVAL_DOCUMENTS
+
+# ToolCallAttributes
 TOOL_CALL_FUNCTION_ARGUMENTS_JSON = ToolCallAttributes.TOOL_CALL_FUNCTION_ARGUMENTS_JSON
+TOOL_CALL_FUNCTION_NAME = ToolCallAttributes.TOOL_CALL_FUNCTION_NAME

phoenix/server/api/helpers/playground_clients.py

@@ -131,8 +131,8 @@ class PlaygroundRateLimiter(RateLimiter, KeyedSingleton):
                 await self._throttler.async_wait_until_ready()
                 request_start_time = time.time()
                 maybe_coroutine = fn(*args, **kwargs)
-                if inspect.iscoroutine(maybe_coroutine):
-                    return await maybe_coroutine  # type: ignore
+                if inspect.isawaitable(maybe_coroutine):
+                    return await maybe_coroutine  # type: ignore[no-any-return]
                 else:
                     return maybe_coroutine
             except self._rate_limit_error:
@@ -144,10 +144,11 @@ class PlaygroundRateLimiter(RateLimiter, KeyedSingleton):
                             try:
                                 request_start_time = time.time()
                                 await self._throttler.async_wait_until_ready()
-                                if inspect.iscoroutinefunction(fn):
-                                    return await fn(*args, **kwargs)  # type: ignore
+                                maybe_coroutine = fn(*args, **kwargs)
+                                if inspect.isawaitable(maybe_coroutine):
+                                    return await maybe_coroutine  # type: ignore[no-any-return]
                                 else:
-                                    return fn(*args, **kwargs)
+                                    return maybe_coroutine
                             except self._rate_limit_error:
                                 self._throttler.on_rate_limit_error(
                                     request_start_time, verbose=self._verbose
@@ -283,7 +284,7 @@ class OpenAIStreamingClient(PlaygroundStreamingClient):
                 invocation_name="temperature",
                 canonical_name=CanonicalParameterName.TEMPERATURE,
                 label="Temperature",
-                default_value=0.0,
+                default_value=1.0,
                 min_value=0.0,
                 max_value=2.0,
             ),
@@ -295,12 +296,14 @@ class OpenAIStreamingClient(PlaygroundStreamingClient):
             BoundedFloatInvocationParameter(
                 invocation_name="frequency_penalty",
                 label="Frequency Penalty",
+                default_value=0.0,
                 min_value=-2.0,
                 max_value=2.0,
             ),
             BoundedFloatInvocationParameter(
                 invocation_name="presence_penalty",
                 label="Presence Penalty",
+                default_value=0.0,
                 min_value=-2.0,
                 max_value=2.0,
             ),
@@ -313,6 +316,7 @@ class OpenAIStreamingClient(PlaygroundStreamingClient):
                 invocation_name="top_p",
                 canonical_name=CanonicalParameterName.TOP_P,
                 label="Top P",
+                default_value=1.0,
                 min_value=0.0,
                 max_value=1.0,
             ),
@@ -345,7 +349,11 @@ class OpenAIStreamingClient(PlaygroundStreamingClient):
         from openai.types.chat import ChatCompletionStreamOptionsParam
 
         # Convert standard messages to OpenAI messages
-        openai_messages = [self.to_openai_chat_completion_param(*message) for message in messages]
+        openai_messages = []
+        for message in messages:
+            openai_message = self.to_openai_chat_completion_param(*message)
+            if openai_message is not None:
+                openai_messages.append(openai_message)
         tool_call_ids: dict[int, str] = {}
         token_usage: Optional["CompletionUsage"] = None
         throttled_create = self.rate_limiter._alimit(self.client.chat.completions.create)
@@ -395,7 +403,7 @@ class OpenAIStreamingClient(PlaygroundStreamingClient):
         content: JSONScalarType,
         tool_call_id: Optional[str] = None,
         tool_calls: Optional[list[JSONScalarType]] = None,
-    ) -> "ChatCompletionMessageParam":
+    ) -> Optional["ChatCompletionMessageParam"]:
         from openai.types.chat import (
             ChatCompletionAssistantMessageParam,
             ChatCompletionSystemMessageParam,
@@ -495,65 +503,7 @@ class OpenAIO1StreamingClient(OpenAIStreamingClient):
             ),
         ]
 
-    async def chat_completion_create(
-        self,
-        messages: list[
-            tuple[ChatCompletionMessageRole, str, Optional[str], Optional[list[JSONScalarType]]]
-        ],
-        tools: list[JSONScalarType],
-        **invocation_parameters: Any,
-    ) -> AsyncIterator[ChatCompletionChunk]:
-        from openai import NOT_GIVEN
-
-        # Convert standard messages to OpenAI messages
-        unfiltered_openai_messages = [
-            self.to_openai_o1_chat_completion_param(*message) for message in messages
-        ]
-
-        # filter out unsupported messages
-        openai_messages: list[ChatCompletionMessageParam] = [
-            message for message in unfiltered_openai_messages if message is not None
-        ]
-
-        tool_call_ids: dict[int, str] = {}
-
-        throttled_create = self.rate_limiter._alimit(self.client.chat.completions.create)
-        response = await throttled_create(
-            messages=openai_messages,
-            model=self.model_name,
-            tools=tools or NOT_GIVEN,
-            **invocation_parameters,
-        )
-
-        choice = response.choices[0]
-        message = choice.message
-        content = message.content
-
-        text_chunk = TextChunk(content=content)
-        yield text_chunk
-
-        if (tool_calls := message.tool_calls) is not None:
-            for tool_call_index, tool_call in enumerate(tool_calls):
-                tool_call_id = (
-                    tool_call.id
-                    if tool_call.id is not None
-                    else tool_call_ids.get(tool_call_index, f"tool_call_{tool_call_index}")
-                )
-                tool_call_ids[tool_call_index] = tool_call_id
-                if (function := tool_call.function) is not None:
-                    tool_call_chunk = ToolCallChunk(
-                        id=tool_call_id,
-                        function=FunctionCallChunk(
-                            name=function.name or "",
-                            arguments=function.arguments or "",
-                        ),
-                    )
-                    yield tool_call_chunk
-
-        if (usage := response.usage) is not None:
-            self._attributes.update(dict(self._llm_token_counts(usage)))
-
-    def to_openai_o1_chat_completion_param(
+    def to_openai_chat_completion_param(
         self,
         role: ChatCompletionMessageRole,
         content: JSONScalarType,
@@ -625,6 +575,8 @@ class AzureOpenAIStreamingClient(OpenAIStreamingClient):
         super().__init__(model=model, api_key=api_key)
         self._attributes[LLM_PROVIDER] = OpenInferenceLLMProviderValues.AZURE.value
         self._attributes[LLM_SYSTEM] = OpenInferenceLLMSystemValues.OPENAI.value
+        if not (api_key := api_key or os.environ.get("AZURE_OPENAI_API_KEY")):
+            raise BadRequest("An Azure API key is required for Azure OpenAI models")
         if not (endpoint := model.endpoint or os.environ.get("AZURE_OPENAI_ENDPOINT")):
             raise BadRequest("An Azure endpoint is required for Azure OpenAI models")
         if not (api_version := model.api_version or os.environ.get("OPENAI_API_VERSION")):
@@ -640,8 +592,12 @@ class AzureOpenAIStreamingClient(OpenAIStreamingClient):
     provider_key=GenerativeProviderKey.ANTHROPIC,
     model_names=[
         PROVIDER_DEFAULT,
+        "claude-3-5-sonnet-latest",
+        "claude-3-5-haiku-latest",
+        "claude-3-5-sonnet-20241022",
+        "claude-3-5-haiku-20241022",
         "claude-3-5-sonnet-20240620",
-        "claude-3-opus-20240229",
+        "claude-3-opus-latest",
         "claude-3-sonnet-20240229",
         "claude-3-haiku-20240307",
     ],
@@ -674,12 +630,14 @@ class AnthropicStreamingClient(PlaygroundStreamingClient):
                 invocation_name="max_tokens",
                 canonical_name=CanonicalParameterName.MAX_COMPLETION_TOKENS,
                 label="Max Tokens",
+                default_value=1024,
                 required=True,
             ),
             BoundedFloatInvocationParameter(
                 invocation_name="temperature",
                 canonical_name=CanonicalParameterName.TEMPERATURE,
                 label="Temperature",
+                default_value=1.0,
                 min_value=0.0,
                 max_value=1.0,
             ),
@@ -692,6 +650,7 @@ class AnthropicStreamingClient(PlaygroundStreamingClient):
                 invocation_name="top_p",
                 canonical_name=CanonicalParameterName.TOP_P,
                 label="Top P",
+                default_value=1.0,
                 min_value=0.0,
                 max_value=1.0,
             ),
@@ -850,7 +809,7 @@ class GeminiStreamingClient(PlaygroundStreamingClient):
                 invocation_name="temperature",
                 canonical_name=CanonicalParameterName.TEMPERATURE,
                 label="Temperature",
-                default_value=0.0,
+                default_value=1.0,
                 min_value=0.0,
                 max_value=2.0,
             ),
@@ -860,35 +819,31 @@ class GeminiStreamingClient(PlaygroundStreamingClient):
                 label="Max Output Tokens",
             ),
             StringListInvocationParameter(
-                invocation_name="stop",
+                invocation_name="stop_sequences",
                 canonical_name=CanonicalParameterName.STOP_SEQUENCES,
                 label="Stop Sequences",
             ),
             FloatInvocationParameter(
                 invocation_name="presence_penalty",
                 label="Presence Penalty",
+                default_value=0.0,
             ),
             FloatInvocationParameter(
                 invocation_name="frequency_penalty",
                 label="Frequency Penalty",
+                default_value=0.0,
             ),
             BoundedFloatInvocationParameter(
                 invocation_name="top_p",
                 canonical_name=CanonicalParameterName.TOP_P,
                 label="Top P",
+                default_value=1.0,
                 min_value=0.0,
                 max_value=1.0,
             ),
-            BoundedFloatInvocationParameter(
+            IntInvocationParameter(
                 invocation_name="top_k",
                 label="Top K",
-                min_value=0.0,
-                max_value=1.0,
-            ),
-            IntInvocationParameter(
-                invocation_name="seed",
-                canonical_name=CanonicalParameterName.RANDOM_SEED,
-                label="Seed",
             ),
         ]
 
@@ -923,6 +878,13 @@ class GeminiStreamingClient(PlaygroundStreamingClient):
         chat = client.start_chat(history=gemini_message_history)
         stream = await chat.send_message_async(**gemini_params)
         async for event in stream:
+            self._attributes.update(
+                {
+                    LLM_TOKEN_COUNT_PROMPT: event.usage_metadata.prompt_token_count,
+                    LLM_TOKEN_COUNT_COMPLETION: event.usage_metadata.candidates_token_count,
+                    LLM_TOKEN_COUNT_TOTAL: event.usage_metadata.total_token_count,
+                }
+            )
             yield TextChunk(content=event.text)
 
     def _build_gemini_messages(

phoenix/server/api/helpers/playground_spans.py

@@ -31,6 +31,7 @@ from typing_extensions import Self, TypeAlias, assert_never
 
 from phoenix.datetime_utils import local_now, normalize_datetime
 from phoenix.db import models
+from phoenix.server.api.helpers.dataset_helpers import get_dataset_example_output
 from phoenix.server.api.input_types.ChatCompletionInput import (
     ChatCompletionInput,
     ChatCompletionOverDatasetInput,
@@ -220,7 +221,7 @@ def get_db_experiment_run(
         dataset_example_id=example_id,
         trace_id=db_trace.trace_id,
         output=models.ExperimentRunOutput(
-            task_output=get_attribute_value(db_span.attributes, LLM_OUTPUT_MESSAGES),
+            task_output=get_dataset_example_output(db_span),
         ),
         repetition_number=1,
         start_time=db_span.start_time,