arize-phoenix 4.7.1__py3-none-any.whl → 4.8.0__py3-none-any.whl

{arize_phoenix-4.7.1.dist-info → arize_phoenix-4.8.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: arize-phoenix
-Version: 4.7.1
+Version: 4.8.0
 Summary: AI Observability and Evaluation
 Project-URL: Documentation, https://docs.arize.com/phoenix/
 Project-URL: Issues, https://github.com/Arize-ai/phoenix/issues

{arize_phoenix-4.7.1.dist-info → arize_phoenix-4.8.0.dist-info}/RECORD

@@ -5,7 +5,7 @@ phoenix/exceptions.py,sha256=n2L2KKuecrdflB9MsCdAYCiSEvGJptIsfRkXMoJle7A,169
 phoenix/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 phoenix/services.py,sha256=aTxhcOA1pZHB6U-B3TEcp6fqDF5oT0xCUvEUNMZVTUQ,5175
 phoenix/settings.py,sha256=cO-qgis_S27nHirTobYI9hHPfZH18R--WMmxNdsVUwc,273
-phoenix/version.py,sha256=7suIQcI50zQCokBqNExDKrJDN157VogyLAp_ZZ4xqCM,22
+phoenix/version.py,sha256=NsvNfym-LKhyklkaZu-iwzlvcIWUdbdxxglpRMhC1TU,22
 phoenix/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 phoenix/core/embedding_dimension.py,sha256=zKGbcvwOXgLf-yrJBpQyKtd-LEOPRKHnUToyAU8Owis,87
 phoenix/core/model.py,sha256=km_a--PBHOuA337ClRw9xqhOHhrUT6Rl9pz_zV0JYkQ,4843
@@ -16,7 +16,7 @@ phoenix/db/__init__.py,sha256=pDjEFXukHmJBM-1D8RjmXkvLsz85YWNxMQczt81ec3A,118
 phoenix/db/alembic.ini,sha256=p8DjVqGUs_tTx8oU56JP7qj-rMUebNFizItUSv_hPhs,3763
 phoenix/db/bulk_inserter.py,sha256=zbZGWZFDybKaGLGzpxgLwxAS5sC0_wXcvM0be4kUhh8,11286
 phoenix/db/engines.py,sha256=vLWaZlToMtDI7rJDxSidYkfOoojamxaZxaz8ND3zTus,4770
-phoenix/db/helpers.py,sha256=L2_jP1iIWpUREhKLYYb4_vf_6v_BiU1E73Z2PczGm6s,1589
+phoenix/db/helpers.py,sha256=mTBhPzdy_aU9gD7hNzUZJkAnV77ko5CdaXyoWH3snPA,2982
 phoenix/db/migrate.py,sha256=MuhtNWnR24riROvarvKfbRb4_D5xuQi6P760vBUKl1E,2270
 phoenix/db/models.py,sha256=7DBWbxY3cx3ve2P1I0kkDKXzlt04zEFJuRPJWsVpH-I,20422
 phoenix/db/insertion/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -31,9 +31,9 @@ phoenix/db/migrations/types.py,sha256=Frq1AKSyBKQQ0FLzON-EmgTqE4kNkOpHMsbWnI-WgC
 phoenix/db/migrations/versions/10460e46d750_datasets.py,sha256=l69yZfScFrjfZZpY0gnqwhsDUEctLeo02qMgA_aOGDg,8155
 phoenix/db/migrations/versions/cf03bd6bae1d_init.py,sha256=CbWT3ZTR0CZqeT3zWLoTWhboFmnOy3Ju1z6Ztpq8WIM,8122
 phoenix/experiments/__init__.py,sha256=6JGwgUd7xCbGpuHqYZlsmErmYvVgv7N_j43bn3dUqsk,123
-phoenix/experiments/functions.py,sha256=m4PlpkTLoZxmt7ZV2KqwcWY5q18Yu3ZTrcEzAqLN-o4,31656
+phoenix/experiments/functions.py,sha256=2ZWCcIs0dh_UNB21t0J_PzWcYhu_32Ai3J6sEMjbBGE,32310
 phoenix/experiments/tracing.py,sha256=wVpt8Ie9WNPoi1djJdcrkwCokHdTO0bicXViLg3O-1Y,2831
-phoenix/experiments/types.py,sha256=HQ9k7dUTlOLZl0iGtZOnToUtZBYGos6afwvO44subAM,24035
+phoenix/experiments/types.py,sha256=VuvDCcvUGeHIQuXS_xpz7Jq5xHdt3qu-O_C7IQ3DvF8,23397
 phoenix/experiments/utils.py,sha256=wLu5Kvt1b4a8rGPRWq5G8RQ9XSiV8fCIVm51zWBI3-g,758
 phoenix/experiments/evaluators/__init__.py,sha256=j63fi3fa3U7-itVPHa82GowhjQRU-wO6yhO34u_lhsA,714
 phoenix/experiments/evaluators/base.py,sha256=ani0F2TN7DMN0KLhV89LIr9-W4g-ccEl2YQJgfp44Js,5325
@@ -71,6 +71,7 @@ phoenix/server/api/context.py,sha256=4jcy203Gtx38399FP21iU3HmFsq-50EKFJlX4IW2Los
 phoenix/server/api/interceptor.py,sha256=ykDnoC_apUd-llVli3m1CW18kNSIgjz2qZ6m5JmPDu8,1294
 phoenix/server/api/queries.py,sha256=wp5BlapuxDIoaQJm7mzG0dURfVxR32vXSJVC0JqG4_Y,19845
 phoenix/server/api/schema.py,sha256=BcxdqO5CSGqpKd-AAJHMjFlzaK9oJA8GJuxmMfcdjn4,434
+phoenix/server/api/utils.py,sha256=Y1lGu8J8r8BSBX9OzffgewI8QMziovbG-ePDvZrrwGI,949
 phoenix/server/api/dataloaders/__init__.py,sha256=qehXL37vGdw7v5PFs3kbZVIuhuzrVNVeZACDQjYpwyo,4847
 phoenix/server/api/dataloaders/average_experiment_run_latency.py,sha256=RiO0AKC6Y5byafsV0zTJEIOt8Nudjte73f1T78cBe1k,1817
 phoenix/server/api/dataloaders/dataset_example_revisions.py,sha256=Vpr5IEKSR4QnAVxE5NM7u92fPNgeHQV2ieYc6JakCj0,3788
@@ -85,7 +86,7 @@ phoenix/server/api/dataloaders/experiment_run_counts.py,sha256=wxHv08aZELJ91KTjH
 phoenix/server/api/dataloaders/experiment_sequence_number.py,sha256=Va1KuoHOd-wzvrlKykoV4kLRFW4JsJvGp_DUI4HYZX4,1631
 phoenix/server/api/dataloaders/latency_ms_quantile.py,sha256=pEc7QjB2iiNOQm_Fmo99F5O_DKOJWgGmcnT0OADJzYE,7423
 phoenix/server/api/dataloaders/min_start_or_max_end_times.py,sha256=IoFX5PtSpvQdMk_7-oB8TpIse3Q4PMxep4qKggkHpzo,2902
-phoenix/server/api/dataloaders/project_by_name.py,sha256=ziYp8fZGy2duD-f_oTqNLwbrGmBQFesqlTEHBDMtZlw,1170
+phoenix/server/api/dataloaders/project_by_name.py,sha256=W4q-ddeVnja2DNwgg5l8mV2GNADNJf1CTXNcJaBWdfE,1165
 phoenix/server/api/dataloaders/record_counts.py,sha256=mp3KlhwFw-Iy7o6bFxtJKC6B5kGinPIh5PxxNkrxf8o,4283
 phoenix/server/api/dataloaders/span_descendants.py,sha256=djqXXwupWaXkFdscoy-iILYijuxlqr8hJcv6JawsV6s,2162
 phoenix/server/api/dataloaders/span_evaluations.py,sha256=quXGyj_OYvHrcWfzXlYIv7C1OCZiL1K7cWU-YEAlsNA,1316
@@ -121,8 +122,8 @@ phoenix/server/api/input_types/TimeRange.py,sha256=yzx-gxj8mDeGLft1FzU_x1MVEgIG5
 phoenix/server/api/input_types/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 phoenix/server/api/mutations/__init__.py,sha256=Jcz-pM6SklhEGKcjA7AIB2WJXGhnxGsghrIq131kyDo,502
 phoenix/server/api/mutations/auth.py,sha256=vPRFoj7J6PV6QeODewG4K0PhoOebS5AfMRpbi_wuhyQ,311
-phoenix/server/api/mutations/dataset_mutations.py,sha256=Zp2sFWyGyubILUQboR6bafRWafsfeRO2ffUWnkLlfgI,22532
-phoenix/server/api/mutations/experiment_mutations.py,sha256=Fw_yEdITGJ6A33M5JZ-2YnBTDoBqZUUFON6vy8JoVjE,2569
+phoenix/server/api/mutations/dataset_mutations.py,sha256=CuKhxsYfvwVcdN_9EXhKxB6444BQfObzKzzyfAeg-n8,23199
+phoenix/server/api/mutations/experiment_mutations.py,sha256=vV2lbJ7ccXZqe-LY7nXx6QxWqhKQE4UNZAFcML-KQ8I,3011
 phoenix/server/api/mutations/export_events_mutations.py,sha256=t_wYBxaqvBJYRoHslh3Bmoxmwlzoy0u8SsBKWIKN5hE,4028
 phoenix/server/api/mutations/project_mutations.py,sha256=3SVDCZqxB0Iv60cOwBL8c-rY3QUUPs8PXbp-C_K1mWY,2267
 phoenix/server/api/openapi/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -130,12 +131,12 @@ phoenix/server/api/openapi/main.py,sha256=WY0pj3B7siQyyYqKyhqnzWC7P8MtEtiukOBUjG
 phoenix/server/api/openapi/schema.py,sha256=uuSYe1Ecu72aXRgTNjyMu-9ZPE13DAHJPKtedS-MsSs,451
 phoenix/server/api/routers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 phoenix/server/api/routers/utils.py,sha256=M41BoH-fl37izhRuN2aX7lWm7jOC20A_3uClv9TVUUY,583
-phoenix/server/api/routers/v1/__init__.py,sha256=vvdpUa2LJPWEg8HbvDm_ANkBAwubPIFPbbHi7elOUws,2808
+phoenix/server/api/routers/v1/__init__.py,sha256=Ir5fsO6gQXW58HGm7s2sMUq0vya7mfcWneLcLJy6_q8,2895
 phoenix/server/api/routers/v1/dataset_examples.py,sha256=XfqOvDKF1oxb0pkeYfBycwwGt3LnSyyGdMLKC5VKoGQ,6690
-phoenix/server/api/routers/v1/datasets.py,sha256=f2gLG-geu-_wtEw4mKSzNWK2cFb5TYOyRL3tQ7Fl7Es,31544
+phoenix/server/api/routers/v1/datasets.py,sha256=r0WcNxF8SKVa3-4rrTIg4Andwr4NmRmW1ybpKuxR9qw,33639
 phoenix/server/api/routers/v1/evaluations.py,sha256=8g6P_e2BweV3RDU0esFmpkb0L5fCwonQPXiJ0y6HLwg,9126
 phoenix/server/api/routers/v1/experiment_evaluations.py,sha256=H_psVyuGUQImo0oxdEAKAMQ-oyVwkVIq5yaMHzHIiPc,5455
-phoenix/server/api/routers/v1/experiment_runs.py,sha256=u4Kgz1i5AffmCF2LHtC9Oo1hlGscZ3Dm8JlTRhM55yU,8307
+phoenix/server/api/routers/v1/experiment_runs.py,sha256=jy4SynmzdtQMoUzlowmG6wsVU14SsLAzfcW4JOhXjeQ,8154
 phoenix/server/api/routers/v1/experiments.py,sha256=cG-LyIGRdB1jVTL42Xi2__nsXibVe9Up7m3hFiTIYYY,11886
 phoenix/server/api/routers/v1/spans.py,sha256=PFeS3ayKj4cUle0CH-f-CpM1fRi-JicEG7BEtkANzAo,4074
 phoenix/server/api/routers/v1/traces.py,sha256=dYEf5pThenAQCgfQljHdrnwd4tC_tAXm6Kvk6GphPYs,2774
@@ -202,7 +203,7 @@ phoenix/server/static/apple-touch-icon-76x76.png,sha256=CT_xT12I0u2i0WU8JzBZBuOQ
 phoenix/server/static/apple-touch-icon.png,sha256=fOfpjqGpWYbJ0eAurKsyoZP1EAs6ZVooBJ_SGk2ZkDs,3801
 phoenix/server/static/favicon.ico,sha256=bY0vvCKRftemZfPShwZtE93DiiQdaYaozkPGwNFr6H8,34494
 phoenix/server/static/index.css,sha256=KKGpx4iwF91VGRm0YN-4cn8oC-oIqC6HecoPf0x3ZM8,1885
-phoenix/server/static/index.js,sha256=tmyBBBaYS-HaFhd0dqqXiIsy5Lnvf-klOxQ0gRHAoDM,3521766
+phoenix/server/static/index.js,sha256=sUkm_imwx5uXpCQxQEPmX-axDr094HE1wGdBznF8J_E,3524949
 phoenix/server/static/modernizr.js,sha256=mvK-XtkNqjOral-QvzoqsyOMECXIMu5BQwSVN_wcU9c,2564
 phoenix/server/templates/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 phoenix/server/templates/index.html,sha256=S4z7qSoNSwnKFAH9r96AR-YJEyoKMd-VMWVlJ_IdzME,2039
@@ -247,8 +248,8 @@ phoenix/utilities/logging.py,sha256=lDXd6EGaamBNcQxL4vP1au9-i_SXe0OraUDiJOcszSw,
 phoenix/utilities/project.py,sha256=qWsvKnG1oKhOFUowXf9qiOL2ia7jaFe_ijFFHEt8GJo,431
 phoenix/utilities/re.py,sha256=PDve_OLjRTM8yQQJHC8-n3HdIONi7aNils3ZKRZ5uBM,2045
 phoenix/utilities/span_store.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-arize_phoenix-4.7.1.dist-info/METADATA,sha256=4l6GZ3dH0ALP9ifyZ7mSk32mjC_i7rWj_ahg2Mrvf-c,11451
-arize_phoenix-4.7.1.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-arize_phoenix-4.7.1.dist-info/licenses/IP_NOTICE,sha256=JBqyyCYYxGDfzQ0TtsQgjts41IJoa-hiwDrBjCb9gHM,469
-arize_phoenix-4.7.1.dist-info/licenses/LICENSE,sha256=HFkW9REuMOkvKRACuwLPT0hRydHb3zNg-fdFt94td18,3794
-arize_phoenix-4.7.1.dist-info/RECORD,,
+arize_phoenix-4.8.0.dist-info/METADATA,sha256=UNwc2BbfIMe5RbFT-IuzfKQnX21dFA4zY5EUXbo5bC8,11451
+arize_phoenix-4.8.0.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+arize_phoenix-4.8.0.dist-info/licenses/IP_NOTICE,sha256=JBqyyCYYxGDfzQ0TtsQgjts41IJoa-hiwDrBjCb9gHM,469
+arize_phoenix-4.8.0.dist-info/licenses/LICENSE,sha256=HFkW9REuMOkvKRACuwLPT0hRydHb3zNg-fdFt94td18,3794
+arize_phoenix-4.8.0.dist-info/RECORD,,

phoenix/db/helpers.py

@@ -1,12 +1,12 @@
 from enum import Enum
-from typing import Any
+from typing import Any, Optional, Tuple
 
 from openinference.semconv.trace import (
     OpenInferenceSpanKindValues,
     RerankerAttributes,
     SpanAttributes,
 )
-from sqlalchemy import Integer, SQLColumnExpression, case, func
+from sqlalchemy import Integer, Select, SQLColumnExpression, case, distinct, func, select
 from typing_extensions import assert_never
 
 from phoenix.db import models
@@ -45,3 +45,38 @@ def num_docs_col(dialect: SupportedSQLDialect) -> SQLColumnExpression[Integer]:
 
 _RETRIEVAL_DOCUMENTS = SpanAttributes.RETRIEVAL_DOCUMENTS.split(".")
 _RERANKER_OUTPUT_DOCUMENTS = RerankerAttributes.RERANKER_OUTPUT_DOCUMENTS.split(".")
+
+
+def get_eval_trace_ids_for_datasets(*dataset_ids: int) -> Select[Tuple[Optional[str]]]:
+    return (
+        select(distinct(models.ExperimentRunAnnotation.trace_id))
+        .join(models.ExperimentRun)
+        .join_from(models.ExperimentRun, models.Experiment)
+        .where(models.Experiment.dataset_id.in_(set(dataset_ids)))
+        .where(models.ExperimentRunAnnotation.trace_id.isnot(None))
+    )
+
+
+def get_project_names_for_datasets(*dataset_ids: int) -> Select[Tuple[Optional[str]]]:
+    return (
+        select(distinct(models.Experiment.project_name))
+        .where(models.Experiment.dataset_id.in_(set(dataset_ids)))
+        .where(models.Experiment.project_name.isnot(None))
+    )
+
+
+def get_eval_trace_ids_for_experiments(*experiment_ids: int) -> Select[Tuple[Optional[str]]]:
+    return (
+        select(distinct(models.ExperimentRunAnnotation.trace_id))
+        .join(models.ExperimentRun)
+        .where(models.ExperimentRun.experiment_id.in_(set(experiment_ids)))
+        .where(models.ExperimentRunAnnotation.trace_id.isnot(None))
+    )
+
+
+def get_project_names_for_experiments(*experiment_ids: int) -> Select[Tuple[Optional[str]]]:
+    return (
+        select(distinct(models.Experiment.project_name))
+        .where(models.Experiment.id.in_(set(experiment_ids)))
+        .where(models.Experiment.project_name.isnot(None))
+    )

phoenix/experiments/functions.py

@@ -1,3 +1,4 @@
+import asyncio
 import functools
 import inspect
 import json
@@ -62,7 +63,6 @@ from phoenix.experiments.types import (
     ExperimentEvaluationRun,
     ExperimentParameters,
     ExperimentRun,
-    ExperimentRunOutput,
     ExperimentTask,
     RanExperiment,
     TaskSummary,
@@ -284,7 +284,7 @@ def run_experiment(
             experiment_id=experiment.id,
             dataset_example_id=example.id,
             repetition_number=repetition_number,
-            experiment_run_output=ExperimentRunOutput(task_output=output),
+            output=output,
             error=repr(error) if error else None,
             trace_id=_str_trace_id(span.get_span_context().trace_id),  # type: ignore[no-untyped-call]
         )
@@ -345,13 +345,21 @@ def run_experiment(
             experiment_id=experiment.id,
             dataset_example_id=example.id,
             repetition_number=repetition_number,
-            experiment_run_output=ExperimentRunOutput(task_output=output),
+            output=output,
             error=repr(error) if error else None,
             trace_id=_str_trace_id(span.get_span_context().trace_id),  # type: ignore[no-untyped-call]
         )
         if not dry_run:
-            resp = await async_client.post(
-                f"/v1/experiments/{experiment.id}/runs", json=jsonify(exp_run)
+            # Below is a workaround to avoid timeout errors sometimes
+            # encountered when the task is a synchronous function that
+            # blocks for too long.
+            resp = await asyncio.get_running_loop().run_in_executor(
+                None,
+                functools.partial(
+                    sync_client.post,
+                    url=f"/v1/experiments/{experiment.id}/runs",
+                    json=jsonify(exp_run),
+                ),
             )
             resp.raise_for_status()
             exp_run = replace(exp_run, id=resp.json()["data"]["id"])
@@ -393,7 +401,7 @@ def run_experiment(
     ran_experiment.__init__(  # type: ignore[misc]
         params=params,
         dataset=dataset,
-        runs={r.id: r for r in task_runs},
+        runs={r.id: r for r in task_runs if r is not None},
         task_summary=task_summary,
         **_asdict(experiment),
     )
@@ -498,7 +506,7 @@ def evaluate_experiment(
             stack.enter_context(capture_spans(resource))
             try:
                 result = evaluator.evaluate(
-                    output=experiment_run.output,
+                    output=deepcopy(experiment_run.output),
                     expected=example.output,
                     reference=example.output,
                     input=example.input,
@@ -550,7 +558,7 @@ def evaluate_experiment(
             stack.enter_context(capture_spans(resource))
             try:
                 result = await evaluator.async_evaluate(
-                    output=experiment_run.output,
+                    output=deepcopy(experiment_run.output),
                     expected=example.output,
                     reference=example.output,
                     input=example.input,
@@ -582,7 +590,17 @@ def evaluate_experiment(
             trace_id=_str_trace_id(span.get_span_context().trace_id),  # type: ignore[no-untyped-call]
         )
         if not dry_run:
-            resp = await async_client.post("/v1/experiment_evaluations", json=jsonify(eval_run))
+            # Below is a workaround to avoid timeout errors sometimes
+            # encountered when the evaluator is a synchronous function
+            # that blocks for too long.
+            resp = await asyncio.get_running_loop().run_in_executor(
+                None,
+                functools.partial(
+                    sync_client.post,
+                    url="/v1/experiment_evaluations",
+                    json=jsonify(eval_run),
+                ),
+            )
             resp.raise_for_status()
             eval_run = replace(eval_run, id=resp.json()["data"]["id"])
         return eval_run

phoenix/experiments/types.py

@@ -198,20 +198,6 @@ class Experiment:
         )
 
 
-@dataclass(frozen=True)
-class ExperimentRunOutput:
-    task_output: TaskOutput
-
-    def __post_init__(self) -> None:
-        object.__setattr__(self, "task_output", _make_read_only(self.task_output))
-
-    @classmethod
-    def from_dict(cls, obj: Optional[Mapping[str, Any]]) -> ExperimentRunOutput:
-        if not obj:
-            return cls(task_output=None)
-        return cls(task_output=obj["task_output"])
-
-
 @dataclass(frozen=True)
 class ExperimentRun:
     start_time: datetime
@@ -219,15 +205,11 @@ class ExperimentRun:
     experiment_id: ExperimentId
     dataset_example_id: ExampleId
     repetition_number: RepetitionNumber
-    experiment_run_output: ExperimentRunOutput
+    output: JSONSerializable
     error: Optional[str] = None
     id: ExperimentRunId = field(default_factory=_dry_run_id)
     trace_id: Optional[TraceId] = None
 
-    @property
-    def output(self) -> Optional[TaskOutput]:
-        return deepcopy(self.experiment_run_output.task_output)
-
     @classmethod
     def from_dict(cls, obj: Mapping[str, Any]) -> ExperimentRun:
         return cls(
@@ -236,14 +218,14 @@ class ExperimentRun:
             experiment_id=obj["experiment_id"],
             dataset_example_id=obj["dataset_example_id"],
             repetition_number=obj.get("repetition_number") or 1,
-            experiment_run_output=ExperimentRunOutput.from_dict(obj["experiment_run_output"]),
+            output=_make_read_only(obj.get("output")),
             error=obj.get("error"),
             id=obj["id"],
             trace_id=obj.get("trace_id"),
         )
 
     def __post_init__(self) -> None:
-        if bool(self.experiment_run_output) == bool(self.error):
+        if bool(self.output) == bool(self.error):
             ValueError("Must specify exactly one of experiment_run_output or error")
 
 
@@ -381,7 +363,7 @@ class EvaluationSummary(_HasStats):
         if df.empty:
             df = pd.DataFrame.from_records(
                 [
-                    {"evaluator": name, "error": True, "score": None, "label": None}
+                    {"evaluator": name, "error": None, "score": None, "label": None}
                     for name in params.eval_names
                 ]
             )
@@ -571,7 +553,7 @@ class RanExperiment(Experiment):
                 {
                     "run_id": run.id,
                     "error": run.error,
-                    "output": deepcopy(run.experiment_run_output.task_output),
+                    "output": deepcopy(run.output),
                     "input": deepcopy((ex := self.dataset.examples[run.dataset_example_id]).input),
                     "expected": deepcopy(ex.output),
                     "metadata": deepcopy(ex.metadata),

phoenix/server/api/dataloaders/project_by_name.py

@@ -28,4 +28,4 @@ class ProjectByNameDataLoader(DataLoader[Key, Result]):
             async for project in data:
                 projects_by_name[project.name] = project
 
-        return [projects_by_name[project_name] for project_name in project_names]
+        return [projects_by_name.get(project_name) for project_name in keys]

phoenix/server/api/mutations/dataset_mutations.py

@@ -1,3 +1,4 @@
+import asyncio
 from datetime import datetime
 from typing import Any, Dict
 
@@ -10,6 +11,7 @@ from strawberry import UNSET
 from strawberry.types import Info
 
 from phoenix.db import models
+from phoenix.db.helpers import get_eval_trace_ids_for_datasets, get_project_names_for_datasets
 from phoenix.server.api.context import Context
 from phoenix.server.api.helpers.dataset_helpers import (
     get_dataset_example_input,
@@ -30,6 +32,7 @@ from phoenix.server.api.types.Dataset import Dataset, to_gql_dataset
 from phoenix.server.api.types.DatasetExample import DatasetExample
 from phoenix.server.api.types.node import from_global_id_with_expected_type
 from phoenix.server.api.types.Span import Span
+from phoenix.server.api.utils import delete_projects, delete_traces
 
 
 @strawberry.type
@@ -274,21 +277,28 @@ class DatasetMutationMixin:
         info: Info[Context, None],
         input: DeleteDatasetInput,
     ) -> DatasetMutationPayload:
-        dataset_id = input.dataset_id
-        dataset_rowid = from_global_id_with_expected_type(
-            global_id=dataset_id, expected_type_name=Dataset.__name__
+        try:
+            dataset_id = from_global_id_with_expected_type(
+                global_id=input.dataset_id,
+                expected_type_name=Dataset.__name__,
+            )
+        except ValueError:
+            raise ValueError(f"Unknown dataset: {input.dataset_id}")
+        project_names_stmt = get_project_names_for_datasets(dataset_id)
+        eval_trace_ids_stmt = get_eval_trace_ids_for_datasets(dataset_id)
+        stmt = (
+            delete(models.Dataset).where(models.Dataset.id == dataset_id).returning(models.Dataset)
         )
-
         async with info.context.db() as session:
-            delete_result = await session.execute(
-                delete(models.Dataset)
-                .where(models.Dataset.id == dataset_rowid)
-                .returning(models.Dataset)
-            )
-            if not (datasets := delete_result.first()):
-                raise ValueError(f"Unknown dataset: {dataset_id}")
-
-        dataset = datasets[0]
+            project_names = await session.scalars(project_names_stmt)
+            eval_trace_ids = await session.scalars(eval_trace_ids_stmt)
+            if not (dataset := await session.scalar(stmt)):
+                raise ValueError(f"Unknown dataset: {input.dataset_id}")
+        await asyncio.gather(
+            delete_projects(info.context.db, *project_names),
+            delete_traces(info.context.db, *eval_trace_ids),
+            return_exceptions=True,
+        )
         return DatasetMutationPayload(dataset=to_gql_dataset(dataset))
 
     @strawberry.mutation(permission_classes=[IsAuthenticated])  # type: ignore

phoenix/server/api/mutations/experiment_mutations.py

@@ -1,3 +1,4 @@
+import asyncio
 from typing import List
 
 import strawberry
@@ -6,11 +7,13 @@ from strawberry.relay import GlobalID
 from strawberry.types import Info
 
 from phoenix.db import models
+from phoenix.db.helpers import get_eval_trace_ids_for_experiments, get_project_names_for_experiments
 from phoenix.server.api.context import Context
 from phoenix.server.api.input_types.DeleteExperimentsInput import DeleteExperimentsInput
 from phoenix.server.api.mutations.auth import IsAuthenticated
 from phoenix.server.api.types.Experiment import Experiment, to_gql_experiment
 from phoenix.server.api.types.node import from_global_id_with_expected_type
+from phoenix.server.api.utils import delete_projects, delete_traces
 
 
 @strawberry.type
@@ -30,7 +33,11 @@ class ExperimentMutationMixin:
             from_global_id_with_expected_type(experiment_id, Experiment.__name__)
             for experiment_id in input.experiment_ids
         ]
+        project_names_stmt = get_project_names_for_experiments(*experiment_ids)
+        eval_trace_ids_stmt = get_eval_trace_ids_for_experiments(*experiment_ids)
         async with info.context.db() as session:
+            project_names = await session.scalars(project_names_stmt)
+            eval_trace_ids = await session.scalars(eval_trace_ids_stmt)
             savepoint = await session.begin_nested()
             experiments = {
                 experiment.id: experiment
@@ -54,10 +61,11 @@ class ExperimentMutationMixin:
                         ]
                     )
                 )
-            if project_names := set(filter(bool, (e.project_name for e in experiments.values()))):
-                await session.execute(
-                    delete(models.Project).where(models.Project.name.in_(project_names))
-                )
+        await asyncio.gather(
+            delete_projects(info.context.db, *project_names),
+            delete_traces(info.context.db, *eval_trace_ids),
+            return_exceptions=True,
+        )
         return ExperimentMutationPayload(
             experiments=[
                 to_gql_experiment(experiments[experiment_id]) for experiment_id in experiment_ids

phoenix/server/api/routers/v1/__init__.py

@@ -44,6 +44,7 @@ V1_ROUTES = [
     Route("/v1/spans", spans.get_spans_handler, methods=["GET"]),
     Route("/v1/datasets/upload", datasets.post_datasets_upload, methods=["POST"]),
     Route("/v1/datasets", datasets.list_datasets, methods=["GET"]),
+    Route("/v1/datasets/{id:str}", datasets.delete_dataset_by_id, methods=["DELETE"]),
     Route("/v1/datasets/{id:str}", datasets.get_dataset_by_id, methods=["GET"]),
     Route("/v1/datasets/{id:str}/csv", datasets.get_dataset_csv, methods=["GET"]),
     Route(

phoenix/server/api/routers/v1/datasets.py

@@ -26,13 +26,15 @@ from typing import (
 
 import pandas as pd
 import pyarrow as pa
-from sqlalchemy import and_, func, select
+from sqlalchemy import and_, delete, func, select
 from sqlalchemy.ext.asyncio import AsyncSession
+from starlette.background import BackgroundTasks
 from starlette.concurrency import run_in_threadpool
 from starlette.datastructures import FormData, UploadFile
 from starlette.requests import Request
 from starlette.responses import JSONResponse, Response
 from starlette.status import (
+    HTTP_204_NO_CONTENT,
     HTTP_404_NOT_FOUND,
     HTTP_409_CONFLICT,
     HTTP_422_UNPROCESSABLE_ENTITY,
@@ -42,6 +44,7 @@ from strawberry.relay import GlobalID
 from typing_extensions import TypeAlias, assert_never
 
 from phoenix.db import models
+from phoenix.db.helpers import get_eval_trace_ids_for_datasets, get_project_names_for_datasets
 from phoenix.db.insertion.dataset import (
     DatasetAction,
     DatasetExampleAdditionEvent,
@@ -52,6 +55,7 @@ from phoenix.server.api.types.Dataset import Dataset
 from phoenix.server.api.types.DatasetExample import DatasetExample
 from phoenix.server.api.types.DatasetVersion import DatasetVersion
 from phoenix.server.api.types.node import from_global_id_with_expected_type
+from phoenix.server.api.utils import delete_projects, delete_traces
 
 logger = logging.getLogger(__name__)
 
@@ -163,6 +167,60 @@ async def list_datasets(request: Request) -> Response:
         return JSONResponse(content={"next_cursor": next_cursor, "data": data})
 
 
+async def delete_dataset_by_id(request: Request) -> Response:
+    """
+    summary: Delete dataset by ID
+    operationId: deleteDatasetById
+    tags:
+      - datasets
+    parameters:
+      - in: path
+        name: id
+        required: true
+        schema:
+          type: string
+    responses:
+      204:
+        description: Success
+      403:
+        description: Forbidden
+      404:
+        description: Dataset not found
+      422:
+        description: Dataset ID is invalid
+    """
+    if id_ := request.path_params.get("id"):
+        try:
+            dataset_id = from_global_id_with_expected_type(
+                GlobalID.from_id(id_),
+                Dataset.__name__,
+            )
+        except ValueError:
+            return Response(
+                content=f"Invalid Dataset ID: {id_}",
+                status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+            )
+    else:
+        return Response(
+            content="Missing Dataset ID",
+            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+        )
+    project_names_stmt = get_project_names_for_datasets(dataset_id)
+    eval_trace_ids_stmt = get_eval_trace_ids_for_datasets(dataset_id)
+    stmt = (
+        delete(models.Dataset).where(models.Dataset.id == dataset_id).returning(models.Dataset.id)
+    )
+    async with request.app.state.db() as session:
+        project_names = await session.scalars(project_names_stmt)
+        eval_trace_ids = await session.scalars(eval_trace_ids_stmt)
+        if (await session.scalar(stmt)) is None:
+            return Response(content="Dataset does not exist", status_code=HTTP_404_NOT_FOUND)
+    tasks = BackgroundTasks()
+    tasks.add_task(delete_projects, request.app.state.db, *project_names)
+    tasks.add_task(delete_traces, request.app.state.db, *eval_trace_ids)
+    return Response(status_code=HTTP_204_NO_CONTENT, background=tasks)
+
+
 async def get_dataset_by_id(request: Request) -> Response:
     """
     summary: Get dataset by ID

phoenix/server/api/routers/v1/experiment_runs.py

@@ -7,9 +7,8 @@ from starlette.status import HTTP_404_NOT_FOUND
 from strawberry.relay import GlobalID
 
 from phoenix.db import models
-from phoenix.experiments.types import ExperimentRun, ExperimentRunOutput
+from phoenix.db.models import ExperimentRunOutput
 from phoenix.server.api.types.node import from_global_id_with_expected_type
-from phoenix.utilities.json import jsonify
 
 
 async def create_experiment_run(request: Request) -> Response:
@@ -39,9 +38,8 @@ async def create_experiment_run(request: Request) -> Response:
               trace_id:
                 type: string
                 description: Optional trace ID for tracking
-              experiment_run_output:
-                type: object
-                description: The output of the experiment run
+              output:
+                description: The output of the experiment task
               repetition_number:
                 type: integer
                 description: The repetition number of the experiment run
@@ -101,7 +99,7 @@ async def create_experiment_run(request: Request) -> Response:
         )
 
     trace_id = payload.get("trace_id", None)
-    output = payload["experiment_run_output"]
+    task_output = payload["output"]
     repetition_number = payload["repetition_number"]
     start_time = payload["start_time"]
     end_time = payload["end_time"]
@@ -112,7 +110,7 @@ async def create_experiment_run(request: Request) -> Response:
             experiment_id=experiment_id,
             dataset_example_id=dataset_example_id,
             trace_id=trace_id,
-            output=output,
+            output=ExperimentRunOutput(task_output=task_output),
             repetition_number=repetition_number,
             start_time=datetime.fromisoformat(start_time),
             end_time=datetime.fromisoformat(end_time),
@@ -170,9 +168,8 @@ async def list_experiment_runs(request: Request) -> Response:
                         type: string
                         format: date-time
                         description: The end time of the experiment run in ISO format
-                      experiment_run_output:
-                        type: object
-                        description: The output of the experiment run
+                      output:
+                        description: The output of the experiment task
                       error:
                         type: string
                         description: Error message if the experiment run encountered an error
@@ -205,16 +202,16 @@ async def list_experiment_runs(request: Request) -> Response:
             experiment_gid = GlobalID("Experiment", str(exp_run.experiment_id))
             example_gid = GlobalID("DatasetExample", str(exp_run.dataset_example_id))
             runs.append(
-                ExperimentRun(
-                    start_time=exp_run.start_time,
-                    end_time=exp_run.end_time,
+                dict(
+                    start_time=exp_run.start_time.isoformat(),
+                    end_time=exp_run.end_time.isoformat(),
                     experiment_id=str(experiment_gid),
                     dataset_example_id=str(example_gid),
                     repetition_number=exp_run.repetition_number,
-                    experiment_run_output=ExperimentRunOutput.from_dict(exp_run.output),
+                    output=exp_run.output.get("task_output"),
                     error=exp_run.error,
                     id=str(run_gid),
                     trace_id=exp_run.trace_id,
                 )
             )
-    return JSONResponse(content={"data": jsonify(runs)}, status_code=200)
+    return JSONResponse(content={"data": runs}, status_code=200)

phoenix/server/api/utils.py

@@ -0,0 +1,36 @@
+from typing import AsyncContextManager, Callable, List
+
+from sqlalchemy import delete
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from phoenix.db import models
+
+
+async def delete_projects(
+    db: Callable[[], AsyncContextManager[AsyncSession]],
+    *project_names: str,
+) -> List[int]:
+    if not project_names:
+        return []
+    stmt = (
+        delete(models.Project)
+        .where(models.Project.name.in_(set(project_names)))
+        .returning(models.Project.id)
+    )
+    async with db() as session:
+        return list(await session.scalars(stmt))
+
+
+async def delete_traces(
+    db: Callable[[], AsyncContextManager[AsyncSession]],
+    *trace_ids: str,
+) -> List[int]:
+    if not trace_ids:
+        return []
+    stmt = (
+        delete(models.Trace)
+        .where(models.Trace.trace_id.in_(set(trace_ids)))
+        .returning(models.Trace.id)
+    )
+    async with db() as session:
+        return list(await session.scalars(stmt))