arize-phoenix 11.32.1__tar.gz → 11.33.0__tar.gz

{arize_phoenix-11.32.1 → arize_phoenix-11.33.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: arize-phoenix
-Version: 11.32.1
+Version: 11.33.0
 Summary: AI Observability and Evaluation
 Project-URL: Documentation, https://arize.com/docs/phoenix/
 Project-URL: Issues, https://github.com/Arize-ai/phoenix/issues

{arize_phoenix-11.32.1 → arize_phoenix-11.33.0}/src/phoenix/inferences/inferences.py

@@ -13,11 +13,10 @@ from pandas import DataFrame, Series, Timestamp, read_parquet
 from pandas.api.types import (
     is_numeric_dtype,
 )
-from typing_extensions import TypeAlias
+from typing_extensions import TypeAlias, deprecated
 
 from phoenix.config import GENERATED_INFERENCES_NAME_PREFIX, INFERENCES_DIR
 from phoenix.datetime_utils import normalize_timestamps
-from phoenix.utilities.deprecation import deprecated
 
 from . import errors as err
 from .schema import (

{arize_phoenix-11.32.1 → arize_phoenix-11.33.0}/src/phoenix/server/api/context.py

@@ -17,15 +17,20 @@ from phoenix.db import models
 from phoenix.server.api.dataloaders import (
     AnnotationConfigsByProjectDataLoader,
     AnnotationSummaryDataLoader,
+    AverageExperimentRepeatedRunGroupLatencyDataLoader,
     AverageExperimentRunLatencyDataLoader,
     CacheForDataLoaders,
     DatasetExampleRevisionsDataLoader,
+    DatasetExamplesAndVersionsByExperimentRunDataLoader,
     DatasetExampleSpansDataLoader,
     DocumentEvaluationsDataLoader,
     DocumentEvaluationSummaryDataLoader,
     DocumentRetrievalMetricsDataLoader,
     ExperimentAnnotationSummaryDataLoader,
     ExperimentErrorRatesDataLoader,
+    ExperimentRepeatedRunGroupAnnotationSummariesDataLoader,
+    ExperimentRepeatedRunGroupsDataLoader,
+    ExperimentRepetitionCountsDataLoader,
     ExperimentRunAnnotations,
     ExperimentRunCountsDataLoader,
     ExperimentSequenceNumberDataLoader,
@@ -52,6 +57,7 @@ from phoenix.server.api.dataloaders import (
     SpanCostDetailSummaryEntriesBySpanDataLoader,
     SpanCostDetailSummaryEntriesByTraceDataLoader,
     SpanCostSummaryByExperimentDataLoader,
+    SpanCostSummaryByExperimentRepeatedRunGroupDataLoader,
     SpanCostSummaryByExperimentRunDataLoader,
     SpanCostSummaryByGenerativeModelDataLoader,
     SpanCostSummaryByProjectDataLoader,
@@ -85,14 +91,25 @@ from phoenix.server.types import (
 class DataLoaders:
     annotation_configs_by_project: AnnotationConfigsByProjectDataLoader
     annotation_summaries: AnnotationSummaryDataLoader
+    average_experiment_repeated_run_group_latency: (
+        AverageExperimentRepeatedRunGroupLatencyDataLoader
+    )
     average_experiment_run_latency: AverageExperimentRunLatencyDataLoader
     dataset_example_revisions: DatasetExampleRevisionsDataLoader
     dataset_example_spans: DatasetExampleSpansDataLoader
+    dataset_examples_and_versions_by_experiment_run: (
+        DatasetExamplesAndVersionsByExperimentRunDataLoader
+    )
     document_evaluation_summaries: DocumentEvaluationSummaryDataLoader
     document_evaluations: DocumentEvaluationsDataLoader
     document_retrieval_metrics: DocumentRetrievalMetricsDataLoader
     experiment_annotation_summaries: ExperimentAnnotationSummaryDataLoader
     experiment_error_rates: ExperimentErrorRatesDataLoader
+    experiment_repeated_run_group_annotation_summaries: (
+        ExperimentRepeatedRunGroupAnnotationSummariesDataLoader
+    )
+    experiment_repeated_run_groups: ExperimentRepeatedRunGroupsDataLoader
+    experiment_repetition_counts: ExperimentRepetitionCountsDataLoader
     experiment_run_annotations: ExperimentRunAnnotations
     experiment_run_counts: ExperimentRunCountsDataLoader
     experiment_sequence_number: ExperimentSequenceNumberDataLoader
@@ -128,6 +145,9 @@ class DataLoaders:
     span_cost_details_by_span_cost: SpanCostDetailsBySpanCostDataLoader
     span_cost_fields: TableFieldsDataLoader
     span_cost_summary_by_experiment: SpanCostSummaryByExperimentDataLoader
+    span_cost_summary_by_experiment_repeated_run_group: (
+        SpanCostSummaryByExperimentRepeatedRunGroupDataLoader
+    )
     span_cost_summary_by_experiment_run: SpanCostSummaryByExperimentRunDataLoader
     span_cost_summary_by_generative_model: SpanCostSummaryByGenerativeModelDataLoader
     span_cost_summary_by_project: SpanCostSummaryByProjectDataLoader

{arize_phoenix-11.32.1 → arize_phoenix-11.33.0}/src/phoenix/server/api/dataloaders/__init__.py

@@ -6,9 +6,15 @@ from phoenix.server.api.dataloaders.span_cost_detail_summary_entries_by_project_
 
 from .annotation_configs_by_project import AnnotationConfigsByProjectDataLoader
 from .annotation_summaries import AnnotationSummaryCache, AnnotationSummaryDataLoader
+from .average_experiment_repeated_run_group_latency import (
+    AverageExperimentRepeatedRunGroupLatencyDataLoader,
+)
 from .average_experiment_run_latency import AverageExperimentRunLatencyDataLoader
 from .dataset_example_revisions import DatasetExampleRevisionsDataLoader
 from .dataset_example_spans import DatasetExampleSpansDataLoader
+from .dataset_examples_and_versions_by_experiment_run import (
+    DatasetExamplesAndVersionsByExperimentRunDataLoader,
+)
 from .document_evaluation_summaries import (
     DocumentEvaluationSummaryCache,
     DocumentEvaluationSummaryDataLoader,
@@ -17,6 +23,11 @@ from .document_evaluations import DocumentEvaluationsDataLoader
 from .document_retrieval_metrics import DocumentRetrievalMetricsDataLoader
 from .experiment_annotation_summaries import ExperimentAnnotationSummaryDataLoader
 from .experiment_error_rates import ExperimentErrorRatesDataLoader
+from .experiment_repeated_run_group_annotation_summaries import (
+    ExperimentRepeatedRunGroupAnnotationSummariesDataLoader,
+)
+from .experiment_repeated_run_groups import ExperimentRepeatedRunGroupsDataLoader
+from .experiment_repetition_counts import ExperimentRepetitionCountsDataLoader
 from .experiment_run_annotations import ExperimentRunAnnotations
 from .experiment_run_counts import ExperimentRunCountsDataLoader
 from .experiment_sequence_number import ExperimentSequenceNumberDataLoader
@@ -44,6 +55,9 @@ from .span_cost_detail_summary_entries_by_span import SpanCostDetailSummaryEntri
 from .span_cost_detail_summary_entries_by_trace import SpanCostDetailSummaryEntriesByTraceDataLoader
 from .span_cost_details_by_span_cost import SpanCostDetailsBySpanCostDataLoader
 from .span_cost_summary_by_experiment import SpanCostSummaryByExperimentDataLoader
+from .span_cost_summary_by_experiment_repeated_run_group import (
+    SpanCostSummaryByExperimentRepeatedRunGroupDataLoader,
+)
 from .span_cost_summary_by_experiment_run import SpanCostSummaryByExperimentRunDataLoader
 from .span_cost_summary_by_generative_model import SpanCostSummaryByGenerativeModelDataLoader
 from .span_cost_summary_by_project import SpanCostSummaryByProjectDataLoader, SpanCostSummaryCache
@@ -64,15 +78,20 @@ from .users import UsersDataLoader
 __all__ = [
     "AnnotationConfigsByProjectDataLoader",
     "AnnotationSummaryDataLoader",
+    "AverageExperimentRepeatedRunGroupLatencyDataLoader",
     "AverageExperimentRunLatencyDataLoader",
     "CacheForDataLoaders",
     "DatasetExampleRevisionsDataLoader",
     "DatasetExampleSpansDataLoader",
+    "DatasetExamplesAndVersionsByExperimentRunDataLoader",
     "DocumentEvaluationSummaryDataLoader",
     "DocumentEvaluationsDataLoader",
     "DocumentRetrievalMetricsDataLoader",
     "ExperimentAnnotationSummaryDataLoader",
     "ExperimentErrorRatesDataLoader",
+    "ExperimentRepeatedRunGroupsDataLoader",
+    "ExperimentRepeatedRunGroupAnnotationSummariesDataLoader",
+    "ExperimentRepetitionCountsDataLoader",
     "ExperimentRunAnnotations",
     "ExperimentRunCountsDataLoader",
     "ExperimentSequenceNumberDataLoader",
@@ -99,6 +118,7 @@ __all__ = [
     "SpanCostDetailSummaryEntriesByTraceDataLoader",
     "SpanCostDetailsBySpanCostDataLoader",
     "SpanCostSummaryByExperimentDataLoader",
+    "SpanCostSummaryByExperimentRepeatedRunGroupDataLoader",
     "SpanCostSummaryByExperimentRunDataLoader",
     "SpanCostSummaryByGenerativeModelDataLoader",
     "SpanCostSummaryByProjectDataLoader",

arize_phoenix-11.33.0/src/phoenix/server/api/dataloaders/average_experiment_repeated_run_group_latency.py

@@ -0,0 +1,50 @@
+from typing import Optional
+
+from sqlalchemy import func, select, tuple_
+from strawberry.dataloader import DataLoader
+from typing_extensions import TypeAlias
+
+from phoenix.db import models
+from phoenix.server.types import DbSessionFactory
+
+ExperimentID: TypeAlias = int
+DatasetExampleID: TypeAlias = int
+RunLatency: TypeAlias = float
+Key: TypeAlias = tuple[ExperimentID, DatasetExampleID]
+Result: TypeAlias = Optional[RunLatency]
+
+
+class AverageExperimentRepeatedRunGroupLatencyDataLoader(DataLoader[Key, Result]):
+    def __init__(
+        self,
+        db: DbSessionFactory,
+    ) -> None:
+        super().__init__(load_fn=self._load_fn)
+        self._db = db
+
+    async def _load_fn(self, keys: list[Key]) -> list[Result]:
+        average_latency_query = (
+            select(
+                models.ExperimentRun.experiment_id.label("experiment_id"),
+                models.ExperimentRun.dataset_example_id.label("example_id"),
+                func.avg(models.ExperimentRun.latency_ms).label("average_repetition_latency_ms"),
+            )
+            .select_from(models.ExperimentRun)
+            .where(
+                tuple_(
+                    models.ExperimentRun.experiment_id, models.ExperimentRun.dataset_example_id
+                ).in_(set(keys))
+            )
+            .group_by(models.ExperimentRun.experiment_id, models.ExperimentRun.dataset_example_id)
+        )
+        async with self._db() as session:
+            average_run_latencies_ms = {
+                (experiment_id, example_id): average_run_latency_ms
+                async for experiment_id, example_id, average_run_latency_ms in await session.stream(
+                    average_latency_query
+                )
+            }
+        return [
+            average_run_latencies_ms.get((experiment_id, example_id))
+            for experiment_id, example_id in keys
+        ]

{arize_phoenix-11.32.1 → arize_phoenix-11.33.0}/src/phoenix/server/api/dataloaders/dataset_example_revisions.py

@@ -91,7 +91,6 @@ class DatasetExampleRevisionsDataLoader(DataLoader[Key, Result]):
                 onclause=revision_ids.c.version_id == models.DatasetVersion.id,
                 isouter=True,  # keep rows where the version id is null
             )
-            .where(models.DatasetExampleRevision.revision_kind != "DELETE")
         )
         async with self._db() as session:
             results = {

arize_phoenix-11.33.0/src/phoenix/server/api/dataloaders/dataset_examples_and_versions_by_experiment_run.py

@@ -0,0 +1,47 @@
+from sqlalchemy import select
+from strawberry.dataloader import DataLoader
+from typing_extensions import TypeAlias
+
+from phoenix.db import models
+from phoenix.server.types import DbSessionFactory
+
+ExperimentRunID: TypeAlias = int
+DatasetExampleID: TypeAlias = int
+DatasetVersionID: TypeAlias = int
+Key: TypeAlias = ExperimentRunID
+Result: TypeAlias = tuple[models.DatasetExample, DatasetVersionID]
+
+
+class DatasetExamplesAndVersionsByExperimentRunDataLoader(DataLoader[Key, Result]):
+    def __init__(self, db: DbSessionFactory) -> None:
+        super().__init__(load_fn=self._load_fn)
+        self._db = db
+
+    async def _load_fn(self, keys: list[Key]) -> list[Result]:
+        experiment_run_ids = set(keys)
+        examples_and_versions_query = (
+            select(
+                models.ExperimentRun.id.label("experiment_run_id"),
+                models.DatasetExample,
+                models.Experiment.dataset_version_id.label("dataset_version_id"),
+            )
+            .select_from(models.ExperimentRun)
+            .join(
+                models.DatasetExample,
+                models.DatasetExample.id == models.ExperimentRun.dataset_example_id,
+            )
+            .join(
+                models.Experiment,
+                models.Experiment.id == models.ExperimentRun.experiment_id,
+            )
+            .where(models.ExperimentRun.id.in_(experiment_run_ids))
+        )
+        async with self._db() as session:
+            examples_and_versions = {
+                experiment_run_id: (example, version_id)
+                for experiment_run_id, example, version_id in (
+                    await session.execute(examples_and_versions_query)
+                ).all()
+            }
+
+        return [examples_and_versions[key] for key in keys]

arize_phoenix-11.33.0/src/phoenix/server/api/dataloaders/experiment_repeated_run_group_annotation_summaries.py

@@ -0,0 +1,77 @@
+from dataclasses import dataclass
+from typing import Optional
+
+from sqlalchemy import func, select, tuple_
+from strawberry.dataloader import DataLoader
+from typing_extensions import TypeAlias
+
+from phoenix.db import models
+from phoenix.server.types import DbSessionFactory
+
+ExperimentID: TypeAlias = int
+DatasetExampleID: TypeAlias = int
+AnnotationName: TypeAlias = str
+MeanAnnotationScore: TypeAlias = float
+
+
+@dataclass
+class AnnotationSummary:
+    annotation_name: AnnotationName
+    mean_score: Optional[MeanAnnotationScore]
+
+
+Key: TypeAlias = tuple[ExperimentID, DatasetExampleID]
+Result: TypeAlias = list[AnnotationSummary]
+
+
+class ExperimentRepeatedRunGroupAnnotationSummariesDataLoader(DataLoader[Key, Result]):
+    def __init__(
+        self,
+        db: DbSessionFactory,
+    ) -> None:
+        super().__init__(load_fn=self._load_fn)
+        self._db = db
+
+    async def _load_fn(self, keys: list[Key]) -> list[Result]:
+        annotation_summaries_query = (
+            select(
+                models.ExperimentRun.experiment_id.label("experiment_id"),
+                models.ExperimentRun.dataset_example_id.label("dataset_example_id"),
+                models.ExperimentRunAnnotation.name.label("annotation_name"),
+                func.avg(models.ExperimentRunAnnotation.score).label("mean_score"),
+            )
+            .select_from(models.ExperimentRunAnnotation)
+            .join(
+                models.ExperimentRun,
+                models.ExperimentRunAnnotation.experiment_run_id == models.ExperimentRun.id,
+            )
+            .where(
+                tuple_(
+                    models.ExperimentRun.experiment_id, models.ExperimentRun.dataset_example_id
+                ).in_(set(keys))
+            )
+            .group_by(
+                models.ExperimentRun.experiment_id,
+                models.ExperimentRun.dataset_example_id,
+                models.ExperimentRunAnnotation.name,
+            )
+        )
+        async with self._db() as session:
+            annotation_summaries = (await session.execute(annotation_summaries_query)).all()
+        annotation_summaries_by_key: dict[Key, list[AnnotationSummary]] = {}
+        for summary in annotation_summaries:
+            key = (summary.experiment_id, summary.dataset_example_id)
+            gql_summary = AnnotationSummary(
+                annotation_name=summary.annotation_name,
+                mean_score=summary.mean_score,
+            )
+            if key not in annotation_summaries_by_key:
+                annotation_summaries_by_key[key] = []
+            annotation_summaries_by_key[key].append(gql_summary)
+        return [
+            sorted(
+                annotation_summaries_by_key.get(key, []),
+                key=lambda summary: summary.annotation_name,
+            )
+            for key in keys
+        ]

arize_phoenix-11.33.0/src/phoenix/server/api/dataloaders/experiment_repeated_run_groups.py

@@ -0,0 +1,59 @@
+from dataclasses import dataclass
+
+from sqlalchemy import select, tuple_
+from sqlalchemy.orm import joinedload
+from strawberry.dataloader import DataLoader
+from typing_extensions import TypeAlias
+
+from phoenix.db import models
+from phoenix.server.types import DbSessionFactory
+
+ExperimentID: TypeAlias = int
+DatasetExampleID: TypeAlias = int
+Key: TypeAlias = tuple[ExperimentID, DatasetExampleID]
+
+
+@dataclass
+class ExperimentRepeatedRunGroup:
+    experiment_rowid: int
+    dataset_example_rowid: int
+    runs: list[models.ExperimentRun]
+
+
+Result: TypeAlias = ExperimentRepeatedRunGroup
+
+
+class ExperimentRepeatedRunGroupsDataLoader(DataLoader[Key, Result]):
+    def __init__(self, db: DbSessionFactory) -> None:
+        super().__init__(load_fn=self._load_fn)
+        self._db = db
+
+    async def _load_fn(self, keys: list[Key]) -> list[Result]:
+        repeated_run_groups_query = (
+            select(models.ExperimentRun)
+            .where(
+                tuple_(
+                    models.ExperimentRun.experiment_id,
+                    models.ExperimentRun.dataset_example_id,
+                ).in_(set(keys))
+            )
+            .order_by(models.ExperimentRun.repetition_number)
+            .options(joinedload(models.ExperimentRun.trace).load_only(models.Trace.trace_id))
+        )
+
+        async with self._db() as session:
+            runs_by_key: dict[Key, list[models.ExperimentRun]] = {}
+            for run in (await session.scalars(repeated_run_groups_query)).all():
+                key = (run.experiment_id, run.dataset_example_id)
+                if key not in runs_by_key:
+                    runs_by_key[key] = []
+                runs_by_key[key].append(run)
+
+        return [
+            ExperimentRepeatedRunGroup(
+                experiment_rowid=experiment_id,
+                dataset_example_rowid=dataset_example_id,
+                runs=runs_by_key.get((experiment_id, dataset_example_id), []),
+            )
+            for (experiment_id, dataset_example_id) in keys
+        ]

arize_phoenix-11.33.0/src/phoenix/server/api/dataloaders/experiment_repetition_counts.py

@@ -0,0 +1,39 @@
+from sqlalchemy import func, select
+from strawberry.dataloader import DataLoader
+from typing_extensions import TypeAlias
+
+from phoenix.db import models
+from phoenix.server.types import DbSessionFactory
+
+ExperimentID: TypeAlias = int
+RepetitionCount: TypeAlias = int
+Key: TypeAlias = ExperimentID
+Result: TypeAlias = RepetitionCount
+
+
+class ExperimentRepetitionCountsDataLoader(DataLoader[Key, Result]):
+    def __init__(
+        self,
+        db: DbSessionFactory,
+    ) -> None:
+        super().__init__(load_fn=self._load_fn)
+        self._db = db
+
+    async def _load_fn(self, keys: list[Key]) -> list[Result]:
+        experiment_ids = keys
+        repetition_counts_query = (
+            select(
+                models.ExperimentRun.experiment_id,
+                func.max(models.ExperimentRun.repetition_number).label("repetition_count"),
+            )
+            .group_by(models.ExperimentRun.experiment_id)
+            .where(models.ExperimentRun.experiment_id.in_(experiment_ids))
+        )
+        async with self._db() as session:
+            repetition_counts = {
+                experiment_id: repetition_count
+                for experiment_id, repetition_count in await session.execute(
+                    repetition_counts_query
+                )
+            }
+        return [repetition_counts.get(experiment_id, 0) for experiment_id in keys]

arize_phoenix-11.33.0/src/phoenix/server/api/dataloaders/span_cost_summary_by_experiment_repeated_run_group.py

@@ -0,0 +1,64 @@
+from collections import defaultdict
+
+from sqlalchemy import func, select, tuple_
+from strawberry.dataloader import DataLoader
+from typing_extensions import TypeAlias
+
+from phoenix.db import models
+from phoenix.server.api.dataloaders.types import CostBreakdown, SpanCostSummary
+from phoenix.server.types import DbSessionFactory
+
+ExperimentId: TypeAlias = int
+DatasetExampleId: TypeAlias = int
+Key: TypeAlias = tuple[ExperimentId, DatasetExampleId]
+Result: TypeAlias = SpanCostSummary
+
+
+class SpanCostSummaryByExperimentRepeatedRunGroupDataLoader(DataLoader[Key, Result]):
+    def __init__(self, db: DbSessionFactory) -> None:
+        super().__init__(load_fn=self._load_fn)
+        self._db = db
+
+    async def _load_fn(self, keys: list[Key]) -> list[Result]:
+        stmt = (
+            select(
+                models.ExperimentRun.experiment_id,
+                models.ExperimentRun.dataset_example_id,
+                func.sum(models.SpanCost.prompt_cost).label("prompt_cost"),
+                func.sum(models.SpanCost.completion_cost).label("completion_cost"),
+                func.sum(models.SpanCost.total_cost).label("total_cost"),
+                func.sum(models.SpanCost.prompt_tokens).label("prompt_tokens"),
+                func.sum(models.SpanCost.completion_tokens).label("completion_tokens"),
+                func.sum(models.SpanCost.total_tokens).label("total_tokens"),
+            )
+            .select_from(models.ExperimentRun)
+            .join(models.Trace, models.ExperimentRun.trace_id == models.Trace.trace_id)
+            .join(models.SpanCost, models.SpanCost.trace_rowid == models.Trace.id)
+            .where(
+                tuple_(
+                    models.ExperimentRun.experiment_id, models.ExperimentRun.dataset_example_id
+                ).in_(set(keys))
+            )
+            .group_by(models.ExperimentRun.experiment_id, models.ExperimentRun.dataset_example_id)
+        )
+
+        results: defaultdict[Key, Result] = defaultdict(SpanCostSummary)
+        async with self._db() as session:
+            data = await session.stream(stmt)
+            async for (
+                experiment_id,
+                dataset_example_id,
+                prompt_cost,
+                completion_cost,
+                total_cost,
+                prompt_tokens,
+                completion_tokens,
+                total_tokens,
+            ) in data:
+                summary = SpanCostSummary(
+                    prompt=CostBreakdown(tokens=prompt_tokens, cost=prompt_cost),
+                    completion=CostBreakdown(tokens=completion_tokens, cost=completion_cost),
+                    total=CostBreakdown(tokens=total_tokens, cost=total_cost),
+                )
+                results[(experiment_id, dataset_example_id)] = summary
+        return [results.get(key, SpanCostSummary()) for key in keys]

{arize_phoenix-11.32.1 → arize_phoenix-11.33.0}/src/phoenix/server/api/helpers/playground_clients.py

@@ -1669,7 +1669,11 @@ class AnthropicReasoningStreamingClient(AnthropicStreamingClient):
     provider_key=GenerativeProviderKey.GOOGLE,
     model_names=[
         PROVIDER_DEFAULT,
+        "gemini-2.5-flash",
+        "gemini-2.5-flash-lite",
+        "gemini-2.5-pro",
         "gemini-2.5-pro-preview-03-25",
+        "gemini-2.0-flash",
         "gemini-2.0-flash-lite",
         "gemini-2.0-flash-001",
         "gemini-2.0-flash-thinking-exp-01-21",