arize-phoenix 11.32.1__py3-none-any.whl → 11.34.0__py3-none-any.whl

phoenix/db/bulk_inserter.py

@@ -1,17 +1,17 @@
 import asyncio
 import logging
 from asyncio import Queue, as_completed
-from collections.abc import AsyncIterator, Awaitable, Callable, Iterable
+from collections import deque
 from dataclasses import dataclass, field
 from functools import singledispatchmethod
-from itertools import islice
-from time import perf_counter
-from typing import Any, Optional, cast
+from time import perf_counter, time
+from typing import Any, AsyncIterator, Awaitable, Callable, Iterable, Optional, cast
 
 from openinference.semconv.trace import SpanAttributes
 from typing_extensions import TypeAlias
 
 import phoenix.trace.v1 as pb
+from phoenix.db import models
 from phoenix.db.insertion.constants import DEFAULT_RETRY_ALLOWANCE, DEFAULT_RETRY_DELAY_SEC
 from phoenix.db.insertion.document_annotation import DocumentAnnotationQueueInserter
 from phoenix.db.insertion.evaluation import (
@@ -29,15 +29,23 @@ from phoenix.db.insertion.trace_annotation import TraceAnnotationQueueInserter
 from phoenix.db.insertion.types import Insertables, Precursors
 from phoenix.server.daemons.span_cost_calculator import (
     SpanCostCalculator,
-    SpanCostCalculatorQueueItem,
 )
 from phoenix.server.dml_event import DmlEvent, SpanInsertEvent
+from phoenix.server.prometheus import (
+    BULK_LOADER_EVALUATION_INSERTIONS,
+    BULK_LOADER_EXCEPTIONS,
+    BULK_LOADER_LAST_ACTIVITY,
+    BULK_LOADER_SPAN_EXCEPTIONS,
+    BULK_LOADER_SPAN_INSERTION_TIME,
+    SPAN_QUEUE_SIZE,
+)
 from phoenix.server.types import CanPutItem, DbSessionFactory
 from phoenix.trace.schemas import Span
 
 logger = logging.getLogger(__name__)
 
 ProjectRowId: TypeAlias = int
+ProjectName: TypeAlias = str
 
 
 @dataclass(frozen=True)
@@ -52,12 +60,12 @@ class BulkInserter:
         *,
         event_queue: CanPutItem[DmlEvent],
         span_cost_calculator: SpanCostCalculator,
-        initial_batch_of_spans: Optional[Iterable[tuple[Span, str]]] = None,
-        initial_batch_of_evaluations: Optional[Iterable[pb.Evaluation]] = None,
+        initial_batch_of_spans: Iterable[tuple[Span, ProjectName]] = (),
+        initial_batch_of_evaluations: Iterable[pb.Evaluation] = (),
         sleep: float = 0.1,
         max_ops_per_transaction: int = 1000,
         max_queue_size: int = 1000,
-        enable_prometheus: bool = False,
+        max_spans_queue_size: Optional[int] = None,
         retry_delay_sec: float = DEFAULT_RETRY_DELAY_SEC,
         retry_allowance: int = DEFAULT_RETRY_ALLOWANCE,
     ) -> None:
@@ -68,7 +76,6 @@ class BulkInserter:
         :param max_ops_per_transaction: The maximum number of operations to dequeue from
         the operations queue for each transaction.
         :param max_queue_size: The maximum length of the operations queue.
-        :param enable_prometheus: Whether Prometheus is enabled.
         """
         self._db = db
         self._running = False
@@ -76,20 +83,20 @@ class BulkInserter:
         self._max_ops_per_transaction = max_ops_per_transaction
         self._operations: Optional[Queue[DataManipulation]] = None
         self._max_queue_size = max_queue_size
-        self._spans: list[tuple[Span, str]] = (
-            [] if initial_batch_of_spans is None else list(initial_batch_of_spans)
-        )
-        self._evaluations: list[pb.Evaluation] = (
-            [] if initial_batch_of_evaluations is None else list(initial_batch_of_evaluations)
-        )
+        self._max_spans_queue_size = max_spans_queue_size
+        self._spans: deque[tuple[Span, ProjectName]] = deque(initial_batch_of_spans)
+        self._evaluations: deque[pb.Evaluation] = deque(initial_batch_of_evaluations)
         self._task: Optional[asyncio.Task[None]] = None
         self._event_queue = event_queue
-        self._enable_prometheus = enable_prometheus
         self._retry_delay_sec = retry_delay_sec
         self._retry_allowance = retry_allowance
         self._queue_inserters = _QueueInserters(db, self._retry_delay_sec, self._retry_allowance)
         self._span_cost_calculator = span_cost_calculator
 
+    @property
+    def is_full(self) -> bool:
+        return bool(self._max_spans_queue_size and self._max_spans_queue_size <= len(self._spans))
+
     async def __aenter__(
         self,
     ) -> tuple[
@@ -102,9 +109,9 @@ class BulkInserter:
         self._operations = Queue(maxsize=self._max_queue_size)
         self._task = asyncio.create_task(self._bulk_insert())
         return (
-            self._enqueue,
-            self._queue_span,
-            self._queue_evaluation,
+            self._enqueue_annotations,
+            self._enqueue_span,
+            self._enqueue_evaluation,
             self._enqueue_operation,
         )
 
@@ -114,23 +121,22 @@ class BulkInserter:
             self._task.cancel()
             self._task = None
 
-    async def _enqueue(self, *items: Any) -> None:
+    async def _enqueue_annotations(self, *items: Any) -> None:
         await self._queue_inserters.enqueue(*items)
 
     def _enqueue_operation(self, operation: DataManipulation) -> None:
         cast("Queue[DataManipulation]", self._operations).put_nowait(operation)
 
-    async def _queue_span(self, span: Span, project_name: str) -> None:
+    async def _enqueue_span(self, span: Span, project_name: str) -> None:
         self._spans.append((span, project_name))
 
-    async def _queue_evaluation(self, evaluation: pb.Evaluation) -> None:
+    async def _enqueue_evaluation(self, evaluation: pb.Evaluation) -> None:
         self._evaluations.append(evaluation)
 
     async def _process_events(self, events: Iterable[Optional[DataManipulationEvent]]) -> None: ...
 
     async def _bulk_insert(self) -> None:
         assert isinstance(self._operations, Queue)
-        spans_buffer, evaluations_buffer = None, None
         # start first insert immediately if the inserter has not run recently
         while (
             self._running
@@ -139,6 +145,8 @@ class BulkInserter:
             or self._spans
             or self._evaluations
         ):
+            BULK_LOADER_LAST_ACTIVITY.set(time())
+            SPAN_QUEUE_SIZE.set(len(self._spans))
             if (
                 self._queue_inserters.empty
                 and self._operations.empty()
@@ -156,113 +164,100 @@ class BulkInserter:
                         async with session.begin_nested():
                             await op(session)
                     except Exception as e:
-                        if self._enable_prometheus:
-                            from phoenix.server.prometheus import BULK_LOADER_EXCEPTIONS
-
-                            BULK_LOADER_EXCEPTIONS.inc()
+                        BULK_LOADER_EXCEPTIONS.inc()
                         logger.exception(str(e))
             # It's important to grab the buffers at the same time so there's
             # no race condition, since an eval insertion will fail if the span
             # it references doesn't exist. Grabbing the eval buffer later may
             # include an eval whose span is in the queue but missed being
             # included in the span buffer that was grabbed previously.
-            if self._spans:
-                spans_buffer = self._spans
-                self._spans = []
-            if self._evaluations:
-                evaluations_buffer = self._evaluations
-                self._evaluations = []
+            num_spans_to_insert = min(self._max_ops_per_transaction, len(self._spans))
+            num_evals_to_insert = min(self._max_ops_per_transaction, len(self._evaluations))
             # Spans should be inserted before the evaluations, since an evaluation
             # insertion will fail if the span it references doesn't exist.
-            if spans_buffer:
-                await self._insert_spans(spans_buffer)
-                spans_buffer = None
-            if evaluations_buffer:
-                await self._insert_evaluations(evaluations_buffer)
-                evaluations_buffer = None
+            await self._insert_spans(num_spans_to_insert)
+            await self._insert_evaluations(num_evals_to_insert)
             async for event in self._queue_inserters.insert():
                 self._event_queue.put(event)
             await asyncio.sleep(self._sleep)
 
-    async def _insert_spans(self, spans: list[tuple[Span, str]]) -> None:
+    async def _insert_spans(self, num_spans_to_insert: int) -> None:
+        if not num_spans_to_insert or not self._spans:
+            return
         project_ids = set()
-        span_cost_calculator_queue: list[SpanCostCalculatorQueueItem] = []
-        for i in range(0, len(spans), self._max_ops_per_transaction):
-            try:
-                start = perf_counter()
-                async with self._db() as session:
-                    for span, project_name in islice(spans, i, i + self._max_ops_per_transaction):
-                        if self._enable_prometheus:
-                            from phoenix.server.prometheus import BULK_LOADER_SPAN_INSERTIONS
-
-                            BULK_LOADER_SPAN_INSERTIONS.inc()
-                        result: Optional[SpanInsertionEvent] = None
-                        try:
-                            async with session.begin_nested():
-                                result = await insert_span(session, span, project_name)
-                        except Exception:
-                            if self._enable_prometheus:
-                                from phoenix.server.prometheus import BULK_LOADER_EXCEPTIONS
-
-                                BULK_LOADER_EXCEPTIONS.inc()
-                            logger.exception(
-                                f"Failed to insert span with span_id={span.context.span_id}"
-                            )
-                        if result is not None:
-                            project_ids.add(result.project_rowid)
-                            if should_calculate_span_cost(span.attributes):
-                                span_cost_calculator_queue.append(
-                                    SpanCostCalculatorQueueItem(
-                                        span_rowid=result.span_rowid,
-                                        trace_rowid=result.trace_rowid,
-                                        attributes=span.attributes,
-                                        span_start_time=span.start_time,
-                                    )
-                                )
-
-                if self._enable_prometheus:
-                    from phoenix.server.prometheus import BULK_LOADER_INSERTION_TIME
-
-                    BULK_LOADER_INSERTION_TIME.observe(perf_counter() - start)
-            except Exception:
-                if self._enable_prometheus:
-                    from phoenix.server.prometheus import BULK_LOADER_EXCEPTIONS
-
-                    BULK_LOADER_EXCEPTIONS.inc()
-                logger.exception("Failed to insert spans")
-        self._event_queue.put(SpanInsertEvent(tuple(project_ids)))
-        for item in span_cost_calculator_queue:
-            self._span_cost_calculator.put_nowait(item)
-
-    async def _insert_evaluations(self, evaluations: list[pb.Evaluation]) -> None:
-        for i in range(0, len(evaluations), self._max_ops_per_transaction):
-            try:
-                start = perf_counter()
-                async with self._db() as session:
-                    for evaluation in islice(evaluations, i, i + self._max_ops_per_transaction):
-                        if self._enable_prometheus:
-                            from phoenix.server.prometheus import BULK_LOADER_EVALUATION_INSERTIONS
-
-                            BULK_LOADER_EVALUATION_INSERTIONS.inc()
-                        try:
-                            async with session.begin_nested():
-                                await insert_evaluation(session, evaluation)
-                        except InsertEvaluationError as error:
-                            if self._enable_prometheus:
-                                from phoenix.server.prometheus import BULK_LOADER_EXCEPTIONS
-
-                                BULK_LOADER_EXCEPTIONS.inc()
-                            logger.exception(f"Failed to insert evaluation: {str(error)}")
-                if self._enable_prometheus:
-                    from phoenix.server.prometheus import BULK_LOADER_INSERTION_TIME
-
-                    BULK_LOADER_INSERTION_TIME.observe(perf_counter() - start)
-            except Exception:
-                if self._enable_prometheus:
-                    from phoenix.server.prometheus import BULK_LOADER_EXCEPTIONS
+        span_costs: list[models.SpanCost] = []
+        try:
+            start = perf_counter()
+            async with self._db() as session:
+                while num_spans_to_insert > 0:
+                    num_spans_to_insert -= 1
+                    if not self._spans:
+                        break
+                    span, project_name = self._spans.popleft()
+                    result: Optional[SpanInsertionEvent] = None
+                    try:
+                        async with session.begin_nested():
+                            result = await insert_span(session, span, project_name)
+                    except Exception:
+                        BULK_LOADER_SPAN_EXCEPTIONS.inc()
+                        logger.exception(
+                            f"Failed to insert span with span_id={span.context.span_id}"
+                        )
+                    if result is None:
+                        continue
+                    project_ids.add(result.project_rowid)
+                    try:
+                        if not should_calculate_span_cost(span.attributes):
+                            continue
+                        span_cost = self._span_cost_calculator.calculate_cost(
+                            span.start_time,
+                            span.attributes,
+                        )
+                    except Exception:
+                        logger.exception(
+                            f"Failed to calculate span cost for span with "
+                            f"span_id={span.context.span_id}"
+                        )
+                    else:
+                        if span_cost is None:
+                            continue
+                        span_cost.span_rowid = result.span_rowid
+                        span_cost.trace_rowid = result.trace_rowid
+                        span_costs.append(span_cost)
+            BULK_LOADER_SPAN_INSERTION_TIME.observe(perf_counter() - start)
+        except Exception:
+            BULK_LOADER_SPAN_EXCEPTIONS.inc()
+            logger.exception("Failed to insert spans")
+        if project_ids:
+            self._event_queue.put(SpanInsertEvent(tuple(project_ids)))
+        if not span_costs:
+            return
+        try:
+            async with self._db() as session:
+                session.add_all(span_costs)
+        except Exception:
+            logger.exception("Failed to insert span costs")
 
-                    BULK_LOADER_EXCEPTIONS.inc()
-                logger.exception("Failed to insert evaluations")
+    async def _insert_evaluations(self, num_evals_to_insert: int) -> None:
+        if not num_evals_to_insert or not self._evaluations:
+            return
+        try:
+            async with self._db() as session:
+                while num_evals_to_insert > 0:
+                    num_evals_to_insert -= 1
+                    if not self._evaluations:
+                        break
+                    evaluation = self._evaluations.popleft()
+                    BULK_LOADER_EVALUATION_INSERTIONS.inc()
+                    try:
+                        async with session.begin_nested():
+                            await insert_evaluation(session, evaluation)
+                    except InsertEvaluationError as error:
+                        BULK_LOADER_EXCEPTIONS.inc()
+                        logger.exception(f"Failed to insert evaluation: {str(error)}")
+        except Exception:
+            BULK_LOADER_EXCEPTIONS.inc()
+            logger.exception("Failed to insert evaluations")
 
 
 class _QueueInserters:

phoenix/inferences/inferences.py

@@ -13,11 +13,10 @@ from pandas import DataFrame, Series, Timestamp, read_parquet
 from pandas.api.types import (
     is_numeric_dtype,
 )
-from typing_extensions import TypeAlias
+from typing_extensions import TypeAlias, deprecated
 
 from phoenix.config import GENERATED_INFERENCES_NAME_PREFIX, INFERENCES_DIR
 from phoenix.datetime_utils import normalize_timestamps
-from phoenix.utilities.deprecation import deprecated
 
 from . import errors as err
 from .schema import (

phoenix/server/api/context.py

@@ -17,15 +17,20 @@ from phoenix.db import models
 from phoenix.server.api.dataloaders import (
     AnnotationConfigsByProjectDataLoader,
     AnnotationSummaryDataLoader,
+    AverageExperimentRepeatedRunGroupLatencyDataLoader,
     AverageExperimentRunLatencyDataLoader,
     CacheForDataLoaders,
     DatasetExampleRevisionsDataLoader,
+    DatasetExamplesAndVersionsByExperimentRunDataLoader,
     DatasetExampleSpansDataLoader,
     DocumentEvaluationsDataLoader,
     DocumentEvaluationSummaryDataLoader,
     DocumentRetrievalMetricsDataLoader,
     ExperimentAnnotationSummaryDataLoader,
     ExperimentErrorRatesDataLoader,
+    ExperimentRepeatedRunGroupAnnotationSummariesDataLoader,
+    ExperimentRepeatedRunGroupsDataLoader,
+    ExperimentRepetitionCountsDataLoader,
     ExperimentRunAnnotations,
     ExperimentRunCountsDataLoader,
     ExperimentSequenceNumberDataLoader,
@@ -52,6 +57,7 @@ from phoenix.server.api.dataloaders import (
     SpanCostDetailSummaryEntriesBySpanDataLoader,
     SpanCostDetailSummaryEntriesByTraceDataLoader,
     SpanCostSummaryByExperimentDataLoader,
+    SpanCostSummaryByExperimentRepeatedRunGroupDataLoader,
     SpanCostSummaryByExperimentRunDataLoader,
     SpanCostSummaryByGenerativeModelDataLoader,
     SpanCostSummaryByProjectDataLoader,
@@ -85,14 +91,25 @@ from phoenix.server.types import (
 class DataLoaders:
     annotation_configs_by_project: AnnotationConfigsByProjectDataLoader
     annotation_summaries: AnnotationSummaryDataLoader
+    average_experiment_repeated_run_group_latency: (
+        AverageExperimentRepeatedRunGroupLatencyDataLoader
+    )
     average_experiment_run_latency: AverageExperimentRunLatencyDataLoader
     dataset_example_revisions: DatasetExampleRevisionsDataLoader
     dataset_example_spans: DatasetExampleSpansDataLoader
+    dataset_examples_and_versions_by_experiment_run: (
+        DatasetExamplesAndVersionsByExperimentRunDataLoader
+    )
     document_evaluation_summaries: DocumentEvaluationSummaryDataLoader
     document_evaluations: DocumentEvaluationsDataLoader
     document_retrieval_metrics: DocumentRetrievalMetricsDataLoader
     experiment_annotation_summaries: ExperimentAnnotationSummaryDataLoader
     experiment_error_rates: ExperimentErrorRatesDataLoader
+    experiment_repeated_run_group_annotation_summaries: (
+        ExperimentRepeatedRunGroupAnnotationSummariesDataLoader
+    )
+    experiment_repeated_run_groups: ExperimentRepeatedRunGroupsDataLoader
+    experiment_repetition_counts: ExperimentRepetitionCountsDataLoader
     experiment_run_annotations: ExperimentRunAnnotations
     experiment_run_counts: ExperimentRunCountsDataLoader
     experiment_sequence_number: ExperimentSequenceNumberDataLoader
@@ -128,6 +145,9 @@ class DataLoaders:
     span_cost_details_by_span_cost: SpanCostDetailsBySpanCostDataLoader
     span_cost_fields: TableFieldsDataLoader
     span_cost_summary_by_experiment: SpanCostSummaryByExperimentDataLoader
+    span_cost_summary_by_experiment_repeated_run_group: (
+        SpanCostSummaryByExperimentRepeatedRunGroupDataLoader
+    )
     span_cost_summary_by_experiment_run: SpanCostSummaryByExperimentRunDataLoader
     span_cost_summary_by_generative_model: SpanCostSummaryByGenerativeModelDataLoader
     span_cost_summary_by_project: SpanCostSummaryByProjectDataLoader

phoenix/server/api/dataloaders/__init__.py

@@ -6,9 +6,15 @@ from phoenix.server.api.dataloaders.span_cost_detail_summary_entries_by_project_
 
 from .annotation_configs_by_project import AnnotationConfigsByProjectDataLoader
 from .annotation_summaries import AnnotationSummaryCache, AnnotationSummaryDataLoader
+from .average_experiment_repeated_run_group_latency import (
+    AverageExperimentRepeatedRunGroupLatencyDataLoader,
+)
 from .average_experiment_run_latency import AverageExperimentRunLatencyDataLoader
 from .dataset_example_revisions import DatasetExampleRevisionsDataLoader
 from .dataset_example_spans import DatasetExampleSpansDataLoader
+from .dataset_examples_and_versions_by_experiment_run import (
+    DatasetExamplesAndVersionsByExperimentRunDataLoader,
+)
 from .document_evaluation_summaries import (
     DocumentEvaluationSummaryCache,
     DocumentEvaluationSummaryDataLoader,
@@ -17,6 +23,11 @@ from .document_evaluations import DocumentEvaluationsDataLoader
 from .document_retrieval_metrics import DocumentRetrievalMetricsDataLoader
 from .experiment_annotation_summaries import ExperimentAnnotationSummaryDataLoader
 from .experiment_error_rates import ExperimentErrorRatesDataLoader
+from .experiment_repeated_run_group_annotation_summaries import (
+    ExperimentRepeatedRunGroupAnnotationSummariesDataLoader,
+)
+from .experiment_repeated_run_groups import ExperimentRepeatedRunGroupsDataLoader
+from .experiment_repetition_counts import ExperimentRepetitionCountsDataLoader
 from .experiment_run_annotations import ExperimentRunAnnotations
 from .experiment_run_counts import ExperimentRunCountsDataLoader
 from .experiment_sequence_number import ExperimentSequenceNumberDataLoader
@@ -44,6 +55,9 @@ from .span_cost_detail_summary_entries_by_span import SpanCostDetailSummaryEntri
 from .span_cost_detail_summary_entries_by_trace import SpanCostDetailSummaryEntriesByTraceDataLoader
 from .span_cost_details_by_span_cost import SpanCostDetailsBySpanCostDataLoader
 from .span_cost_summary_by_experiment import SpanCostSummaryByExperimentDataLoader
+from .span_cost_summary_by_experiment_repeated_run_group import (
+    SpanCostSummaryByExperimentRepeatedRunGroupDataLoader,
+)
 from .span_cost_summary_by_experiment_run import SpanCostSummaryByExperimentRunDataLoader
 from .span_cost_summary_by_generative_model import SpanCostSummaryByGenerativeModelDataLoader
 from .span_cost_summary_by_project import SpanCostSummaryByProjectDataLoader, SpanCostSummaryCache
@@ -64,15 +78,20 @@ from .users import UsersDataLoader
 __all__ = [
     "AnnotationConfigsByProjectDataLoader",
     "AnnotationSummaryDataLoader",
+    "AverageExperimentRepeatedRunGroupLatencyDataLoader",
     "AverageExperimentRunLatencyDataLoader",
     "CacheForDataLoaders",
     "DatasetExampleRevisionsDataLoader",
     "DatasetExampleSpansDataLoader",
+    "DatasetExamplesAndVersionsByExperimentRunDataLoader",
     "DocumentEvaluationSummaryDataLoader",
     "DocumentEvaluationsDataLoader",
     "DocumentRetrievalMetricsDataLoader",
     "ExperimentAnnotationSummaryDataLoader",
     "ExperimentErrorRatesDataLoader",
+    "ExperimentRepeatedRunGroupsDataLoader",
+    "ExperimentRepeatedRunGroupAnnotationSummariesDataLoader",
+    "ExperimentRepetitionCountsDataLoader",
     "ExperimentRunAnnotations",
     "ExperimentRunCountsDataLoader",
     "ExperimentSequenceNumberDataLoader",
@@ -99,6 +118,7 @@ __all__ = [
     "SpanCostDetailSummaryEntriesByTraceDataLoader",
     "SpanCostDetailsBySpanCostDataLoader",
     "SpanCostSummaryByExperimentDataLoader",
+    "SpanCostSummaryByExperimentRepeatedRunGroupDataLoader",
     "SpanCostSummaryByExperimentRunDataLoader",
     "SpanCostSummaryByGenerativeModelDataLoader",
     "SpanCostSummaryByProjectDataLoader",

phoenix/server/api/dataloaders/average_experiment_repeated_run_group_latency.py

@@ -0,0 +1,50 @@
+from typing import Optional
+
+from sqlalchemy import func, select, tuple_
+from strawberry.dataloader import DataLoader
+from typing_extensions import TypeAlias
+
+from phoenix.db import models
+from phoenix.server.types import DbSessionFactory
+
+ExperimentID: TypeAlias = int
+DatasetExampleID: TypeAlias = int
+RunLatency: TypeAlias = float
+Key: TypeAlias = tuple[ExperimentID, DatasetExampleID]
+Result: TypeAlias = Optional[RunLatency]
+
+
+class AverageExperimentRepeatedRunGroupLatencyDataLoader(DataLoader[Key, Result]):
+    def __init__(
+        self,
+        db: DbSessionFactory,
+    ) -> None:
+        super().__init__(load_fn=self._load_fn)
+        self._db = db
+
+    async def _load_fn(self, keys: list[Key]) -> list[Result]:
+        average_latency_query = (
+            select(
+                models.ExperimentRun.experiment_id.label("experiment_id"),
+                models.ExperimentRun.dataset_example_id.label("example_id"),
+                func.avg(models.ExperimentRun.latency_ms).label("average_repetition_latency_ms"),
+            )
+            .select_from(models.ExperimentRun)
+            .where(
+                tuple_(
+                    models.ExperimentRun.experiment_id, models.ExperimentRun.dataset_example_id
+                ).in_(set(keys))
+            )
+            .group_by(models.ExperimentRun.experiment_id, models.ExperimentRun.dataset_example_id)
+        )
+        async with self._db() as session:
+            average_run_latencies_ms = {
+                (experiment_id, example_id): average_run_latency_ms
+                async for experiment_id, example_id, average_run_latency_ms in await session.stream(
+                    average_latency_query
+                )
+            }
+        return [
+            average_run_latencies_ms.get((experiment_id, example_id))
+            for experiment_id, example_id in keys
+        ]

phoenix/server/api/dataloaders/dataset_example_revisions.py

@@ -91,7 +91,6 @@ class DatasetExampleRevisionsDataLoader(DataLoader[Key, Result]):
                 onclause=revision_ids.c.version_id == models.DatasetVersion.id,
                 isouter=True,  # keep rows where the version id is null
             )
-            .where(models.DatasetExampleRevision.revision_kind != "DELETE")
         )
         async with self._db() as session:
             results = {

phoenix/server/api/dataloaders/dataset_examples_and_versions_by_experiment_run.py

@@ -0,0 +1,47 @@
+from sqlalchemy import select
+from strawberry.dataloader import DataLoader
+from typing_extensions import TypeAlias
+
+from phoenix.db import models
+from phoenix.server.types import DbSessionFactory
+
+ExperimentRunID: TypeAlias = int
+DatasetExampleID: TypeAlias = int
+DatasetVersionID: TypeAlias = int
+Key: TypeAlias = ExperimentRunID
+Result: TypeAlias = tuple[models.DatasetExample, DatasetVersionID]
+
+
+class DatasetExamplesAndVersionsByExperimentRunDataLoader(DataLoader[Key, Result]):
+    def __init__(self, db: DbSessionFactory) -> None:
+        super().__init__(load_fn=self._load_fn)
+        self._db = db
+
+    async def _load_fn(self, keys: list[Key]) -> list[Result]:
+        experiment_run_ids = set(keys)
+        examples_and_versions_query = (
+            select(
+                models.ExperimentRun.id.label("experiment_run_id"),
+                models.DatasetExample,
+                models.Experiment.dataset_version_id.label("dataset_version_id"),
+            )
+            .select_from(models.ExperimentRun)
+            .join(
+                models.DatasetExample,
+                models.DatasetExample.id == models.ExperimentRun.dataset_example_id,
+            )
+            .join(
+                models.Experiment,
+                models.Experiment.id == models.ExperimentRun.experiment_id,
+            )
+            .where(models.ExperimentRun.id.in_(experiment_run_ids))
+        )
+        async with self._db() as session:
+            examples_and_versions = {
+                experiment_run_id: (example, version_id)
+                for experiment_run_id, example, version_id in (
+                    await session.execute(examples_and_versions_query)
+                ).all()
+            }
+
+        return [examples_and_versions[key] for key in keys]