arize-phoenix 11.32.1__py3-none-any.whl → 11.33.0__py3-none-any.whl

phoenix/server/api/types/ExperimentRepeatedRunGroup.py

@@ -0,0 +1,146 @@
+import re
+from base64 import b64decode
+from typing import Optional
+
+import strawberry
+from sqlalchemy import func, select
+from strawberry.relay import GlobalID, Node
+from strawberry.types import Info
+from typing_extensions import Self, TypeAlias
+
+from phoenix.db import models
+from phoenix.server.api.context import Context
+from phoenix.server.api.types.CostBreakdown import CostBreakdown
+from phoenix.server.api.types.ExperimentRepeatedRunGroupAnnotationSummary import (
+    ExperimentRepeatedRunGroupAnnotationSummary,
+)
+from phoenix.server.api.types.ExperimentRun import ExperimentRun
+from phoenix.server.api.types.SpanCostDetailSummaryEntry import SpanCostDetailSummaryEntry
+from phoenix.server.api.types.SpanCostSummary import SpanCostSummary
+
+ExperimentRowId: TypeAlias = int
+DatasetExampleRowId: TypeAlias = int
+
+
+@strawberry.type
+class ExperimentRepeatedRunGroup(Node):
+    experiment_rowid: strawberry.Private[ExperimentRowId]
+    dataset_example_rowid: strawberry.Private[DatasetExampleRowId]
+    runs: list[ExperimentRun]
+
+    @classmethod
+    def resolve_id(
+        cls,
+        root: Self,
+        *,
+        info: Info,
+    ) -> str:
+        return (
+            f"experiment_id={root.experiment_rowid}:dataset_example_id={root.dataset_example_rowid}"
+        )
+
+    @strawberry.field
+    def experiment_id(self) -> strawberry.ID:
+        from phoenix.server.api.types.Experiment import Experiment
+
+        return strawberry.ID(str(GlobalID(Experiment.__name__, str(self.experiment_rowid))))
+
+    @strawberry.field
+    async def average_latency_ms(self, info: Info[Context, None]) -> Optional[float]:
+        return await info.context.data_loaders.average_experiment_repeated_run_group_latency.load(
+            (self.experiment_rowid, self.dataset_example_rowid)
+        )
+
+    @strawberry.field
+    async def cost_summary(self, info: Info[Context, None]) -> SpanCostSummary:
+        experiment_id = self.experiment_rowid
+        example_id = self.dataset_example_rowid
+        summary = (
+            await info.context.data_loaders.span_cost_summary_by_experiment_repeated_run_group.load(
+                (experiment_id, example_id)
+            )
+        )
+        return SpanCostSummary(
+            prompt=CostBreakdown(
+                tokens=summary.prompt.tokens,
+                cost=summary.prompt.cost,
+            ),
+            completion=CostBreakdown(
+                tokens=summary.completion.tokens,
+                cost=summary.completion.cost,
+            ),
+            total=CostBreakdown(
+                tokens=summary.total.tokens,
+                cost=summary.total.cost,
+            ),
+        )
+
+    @strawberry.field
+    async def cost_detail_summary_entries(
+        self, info: Info[Context, None]
+    ) -> list[SpanCostDetailSummaryEntry]:
+        experiment_id = self.experiment_rowid
+        example_id = self.dataset_example_rowid
+        stmt = (
+            select(
+                models.SpanCostDetail.token_type,
+                models.SpanCostDetail.is_prompt,
+                func.sum(models.SpanCostDetail.cost).label("cost"),
+                func.sum(models.SpanCostDetail.tokens).label("tokens"),
+            )
+            .select_from(models.SpanCostDetail)
+            .join(models.SpanCost, models.SpanCostDetail.span_cost_id == models.SpanCost.id)
+            .join(models.Trace, models.SpanCost.trace_rowid == models.Trace.id)
+            .join(models.ExperimentRun, models.ExperimentRun.trace_id == models.Trace.trace_id)
+            .where(models.ExperimentRun.experiment_id == experiment_id)
+            .where(models.ExperimentRun.dataset_example_id == example_id)
+            .group_by(models.SpanCostDetail.token_type, models.SpanCostDetail.is_prompt)
+        )
+
+        async with info.context.db() as session:
+            data = await session.stream(stmt)
+            return [
+                SpanCostDetailSummaryEntry(
+                    token_type=token_type,
+                    is_prompt=is_prompt,
+                    value=CostBreakdown(tokens=tokens, cost=cost),
+                )
+                async for token_type, is_prompt, cost, tokens in data
+            ]
+
+    @strawberry.field
+    async def annotation_summaries(
+        self,
+        info: Info[Context, None],
+    ) -> list[ExperimentRepeatedRunGroupAnnotationSummary]:
+        loader = info.context.data_loaders.experiment_repeated_run_group_annotation_summaries
+        summaries = await loader.load((self.experiment_rowid, self.dataset_example_rowid))
+        return [
+            ExperimentRepeatedRunGroupAnnotationSummary(
+                annotation_name=summary.annotation_name,
+                mean_score=summary.mean_score,
+            )
+            for summary in summaries
+        ]
+
+
+_EXPERIMENT_REPEATED_RUN_GROUP_NODE_ID_PATTERN = re.compile(
+    r"ExperimentRepeatedRunGroup:experiment_id=(\d+):dataset_example_id=(\d+)"
+)
+
+
+def parse_experiment_repeated_run_group_node_id(
+    node_id: str,
+) -> tuple[ExperimentRowId, DatasetExampleRowId]:
+    decoded_node_id = _base64_decode(node_id)
+    match = re.match(_EXPERIMENT_REPEATED_RUN_GROUP_NODE_ID_PATTERN, decoded_node_id)
+    if not match:
+        raise ValueError(f"Invalid node ID format: {node_id}")
+
+    experiment_id = int(match.group(1))
+    dataset_example_id = int(match.group(2))
+    return experiment_id, dataset_example_id
+
+
+def _base64_decode(string: str) -> str:
+    return b64decode(string.encode()).decode()

phoenix/server/api/types/ExperimentRepeatedRunGroupAnnotationSummary.py

@@ -0,0 +1,9 @@
+from typing import Optional
+
+import strawberry
+
+
+@strawberry.type
+class ExperimentRepeatedRunGroupAnnotationSummary:
+    annotation_name: str
+    mean_score: Optional[float]

phoenix/server/api/types/ExperimentRun.py

@@ -3,7 +3,6 @@ from typing import TYPE_CHECKING, Annotated, Optional
 
 import strawberry
 from sqlalchemy import func, select
-from sqlalchemy.orm import load_only
 from sqlalchemy.sql.functions import coalesce
 from strawberry import UNSET
 from strawberry.relay import Connection, GlobalID, Node, NodeID
@@ -34,12 +33,17 @@ if TYPE_CHECKING:
 class ExperimentRun(Node):
     id_attr: NodeID[int]
     experiment_id: GlobalID
+    repetition_number: int
     trace_id: Optional[str]
     output: Optional[JSON]
     start_time: datetime
     end_time: datetime
     error: Optional[str]
 
+    @strawberry.field
+    def latency_ms(self) -> float:
+        return (self.end_time - self.start_time).total_seconds() * 1000
+
     @strawberry.field
     async def annotations(
         self,
@@ -78,24 +82,12 @@ class ExperimentRun(Node):
     ]:  # use lazy types to avoid circular import: https://strawberry.rocks/docs/types/lazy
         from phoenix.server.api.types.DatasetExample import DatasetExample
 
-        async with info.context.db() as session:
-            assert (
-                result := await session.execute(
-                    select(models.DatasetExample, models.Experiment.dataset_version_id)
-                    .select_from(models.ExperimentRun)
-                    .join(
-                        models.DatasetExample,
-                        models.DatasetExample.id == models.ExperimentRun.dataset_example_id,
-                    )
-                    .join(
-                        models.Experiment,
-                        models.Experiment.id == models.ExperimentRun.experiment_id,
-                    )
-                    .where(models.ExperimentRun.id == self.id_attr)
-                    .options(load_only(models.DatasetExample.id, models.DatasetExample.created_at))
-                )
-            ) is not None
-            example, version_id = result.first()
+        (
+            example,
+            version_id,
+        ) = await info.context.data_loaders.dataset_examples_and_versions_by_experiment_run.load(
+            self.id_attr
+        )
         return DatasetExample(
             id_attr=example.id,
             created_at=example.created_at,
@@ -165,6 +157,7 @@ def to_gql_experiment_run(run: models.ExperimentRun) -> ExperimentRun:
     return ExperimentRun(
         id_attr=run.id,
         experiment_id=GlobalID(Experiment.__name__, str(run.experiment_id)),
+        repetition_number=run.repetition_number,
         trace_id=run.trace.trace_id if run.trace else None,
         output=run.output.get("task_output"),
         start_time=run.start_time,

phoenix/server/api/types/Prompt.py

@@ -19,6 +19,7 @@ from phoenix.server.api.types.pagination import (
     connection_from_list,
 )
 
+from .PromptLabel import PromptLabel, to_gql_prompt_label
 from .PromptVersion import (
     PromptVersion,
     to_gql_prompt_version,
@@ -116,6 +117,16 @@ class Prompt(Node):
                 raise NotFound(f"Source prompt not found: {self.source_prompt_id}")
             return to_gql_prompt_from_orm(source_prompt)
 
+    @strawberry.field
+    async def labels(self, info: Info[Context, None]) -> list["PromptLabel"]:
+        async with info.context.db() as session:
+            labels = await session.scalars(
+                select(models.PromptLabel)
+                .join(models.PromptPromptLabel)
+                .where(models.PromptPromptLabel.prompt_id == self.id_attr)
+            )
+            return [to_gql_prompt_label(label) for label in labels]
+
 
 def to_gql_prompt_from_orm(orm_model: "models.Prompt") -> Prompt:
     if not orm_model.source_prompt_id:

phoenix/server/api/types/PromptLabel.py

@@ -1,14 +1,10 @@
 from typing import Optional
 
 import strawberry
-from sqlalchemy import select
 from strawberry.relay import Node, NodeID
-from strawberry.types import Info
 
 from phoenix.db import models
-from phoenix.server.api.context import Context
 from phoenix.server.api.types.Identifier import Identifier
-from phoenix.server.api.types.Prompt import Prompt, to_gql_prompt_from_orm
 
 
 @strawberry.type
@@ -16,21 +12,7 @@ class PromptLabel(Node):
     id_attr: NodeID[int]
     name: Identifier
     description: Optional[str] = None
-
-    @strawberry.field
-    async def prompts(self, info: Info[Context, None]) -> list[Prompt]:
-        async with info.context.db() as session:
-            statement = (
-                select(models.Prompt)
-                .join(
-                    models.PromptPromptLabel, models.Prompt.id == models.PromptPromptLabel.prompt_id
-                )
-                .where(models.PromptPromptLabel.prompt_label_id == self.id_attr)
-            )
-            return [
-                to_gql_prompt_from_orm(prompt_orm)
-                async for prompt_orm in await session.stream_scalars(statement)
-            ]
+    color: str
 
 
 def to_gql_prompt_label(label_orm: models.PromptLabel) -> PromptLabel:
@@ -38,4 +20,5 @@ def to_gql_prompt_label(label_orm: models.PromptLabel) -> PromptLabel:
         id_attr=label_orm.id,
         name=Identifier(label_orm.name),
         description=label_orm.description,
+        color=label_orm.color,
     )

phoenix/server/api/types/node.py

@@ -1,5 +1,15 @@
+import re
+from base64 import b64decode
+
 from strawberry.relay import GlobalID
 
+_GLOBAL_ID_PATTERN = re.compile(r"[a-zA-Z]+:[0-9]+")
+
+
+def is_global_id(node_id: str) -> bool:
+    decoded_node_id = b64decode(node_id).decode()
+    return _GLOBAL_ID_PATTERN.match(decoded_node_id) is not None
+
 
 def from_global_id(global_id: GlobalID) -> tuple[str, int]:
     """

phoenix/server/app.py

@@ -81,15 +81,20 @@ from phoenix.server.api.context import Context, DataLoaders
 from phoenix.server.api.dataloaders import (
     AnnotationConfigsByProjectDataLoader,
     AnnotationSummaryDataLoader,
+    AverageExperimentRepeatedRunGroupLatencyDataLoader,
     AverageExperimentRunLatencyDataLoader,
     CacheForDataLoaders,
     DatasetExampleRevisionsDataLoader,
+    DatasetExamplesAndVersionsByExperimentRunDataLoader,
     DatasetExampleSpansDataLoader,
     DocumentEvaluationsDataLoader,
     DocumentEvaluationSummaryDataLoader,
     DocumentRetrievalMetricsDataLoader,
     ExperimentAnnotationSummaryDataLoader,
     ExperimentErrorRatesDataLoader,
+    ExperimentRepeatedRunGroupAnnotationSummariesDataLoader,
+    ExperimentRepeatedRunGroupsDataLoader,
+    ExperimentRepetitionCountsDataLoader,
     ExperimentRunAnnotations,
     ExperimentRunCountsDataLoader,
     ExperimentSequenceNumberDataLoader,
@@ -116,6 +121,7 @@ from phoenix.server.api.dataloaders import (
     SpanCostDetailSummaryEntriesBySpanDataLoader,
     SpanCostDetailSummaryEntriesByTraceDataLoader,
     SpanCostSummaryByExperimentDataLoader,
+    SpanCostSummaryByExperimentRepeatedRunGroupDataLoader,
     SpanCostSummaryByExperimentRunDataLoader,
     SpanCostSummaryByGenerativeModelDataLoader,
     SpanCostSummaryByProjectDataLoader,
@@ -663,9 +669,15 @@ def create_graphql_router(
             event_queue=event_queue,
             data_loaders=DataLoaders(
                 annotation_configs_by_project=AnnotationConfigsByProjectDataLoader(db),
+                average_experiment_repeated_run_group_latency=AverageExperimentRepeatedRunGroupLatencyDataLoader(
+                    db
+                ),
                 average_experiment_run_latency=AverageExperimentRunLatencyDataLoader(db),
                 dataset_example_revisions=DatasetExampleRevisionsDataLoader(db),
                 dataset_example_spans=DatasetExampleSpansDataLoader(db),
+                dataset_examples_and_versions_by_experiment_run=DatasetExamplesAndVersionsByExperimentRunDataLoader(
+                    db
+                ),
                 document_evaluation_summaries=DocumentEvaluationSummaryDataLoader(
                     db,
                     cache_map=(
@@ -684,6 +696,11 @@ def create_graphql_router(
                 ),
                 experiment_annotation_summaries=ExperimentAnnotationSummaryDataLoader(db),
                 experiment_error_rates=ExperimentErrorRatesDataLoader(db),
+                experiment_repeated_run_group_annotation_summaries=ExperimentRepeatedRunGroupAnnotationSummariesDataLoader(
+                    db
+                ),
+                experiment_repeated_run_groups=ExperimentRepeatedRunGroupsDataLoader(db),
+                experiment_repetition_counts=ExperimentRepetitionCountsDataLoader(db),
                 experiment_run_annotations=ExperimentRunAnnotations(db),
                 experiment_run_counts=ExperimentRunCountsDataLoader(db),
                 experiment_sequence_number=ExperimentSequenceNumberDataLoader(db),
@@ -740,6 +757,11 @@ def create_graphql_router(
                 span_cost_details_by_span_cost=SpanCostDetailsBySpanCostDataLoader(db),
                 span_cost_detail_fields=TableFieldsDataLoader(db, models.SpanCostDetail),
                 span_cost_fields=TableFieldsDataLoader(db, models.SpanCost),
+                span_cost_summary_by_experiment=SpanCostSummaryByExperimentDataLoader(db),
+                span_cost_summary_by_experiment_repeated_run_group=SpanCostSummaryByExperimentRepeatedRunGroupDataLoader(
+                    db
+                ),
+                span_cost_summary_by_experiment_run=SpanCostSummaryByExperimentRunDataLoader(db),
                 span_cost_summary_by_generative_model=SpanCostSummaryByGenerativeModelDataLoader(
                     db
                 ),
@@ -768,8 +790,6 @@ def create_graphql_router(
                 project_by_name=ProjectByNameDataLoader(db),
                 users=UsersDataLoader(db),
                 user_roles=UserRolesDataLoader(db),
-                span_cost_summary_by_experiment=SpanCostSummaryByExperimentDataLoader(db),
-                span_cost_summary_by_experiment_run=SpanCostSummaryByExperimentRunDataLoader(db),
             ),
             cache_for_dataloaders=cache_for_dataloaders,
             read_only=read_only,

phoenix/server/cost_tracking/model_cost_manifest.json

@@ -698,7 +698,7 @@
           "token_type": "input"
         },
         {
-          "base_rate": 2.5e-6,
+          "base_rate": 0.00003,
           "is_prompt": false,
           "token_type": "output"
         },

phoenix/server/static/.vite/manifest.json

@@ -1,73 +1,73 @@
 {
-  "_components-Cs9c4Nxp.js": {
-    "file": "assets/components-Cs9c4Nxp.js",
+  "_components-YTHUASXI.js": {
+    "file": "assets/components-YTHUASXI.js",
     "name": "components",
     "imports": [
-      "_vendor-RdRDaQiR.js",
-      "_pages-Cbj9SjBx.js",
-      "_vendor-arizeai-DsYDNOqt.js",
-      "_vendor-codemirror-BzJDUbEx.js",
+      "_vendor-CRRxHwSp.js",
+      "_pages-4Qu8GNlt.js",
+      "_vendor-arizeai-CUN6lRd9.js",
+      "_vendor-codemirror-sJXwoqrE.js",
       "_vendor-three-BLWp5bic.js"
     ]
   },
-  "_pages-Cbj9SjBx.js": {
-    "file": "assets/pages-Cbj9SjBx.js",
+  "_pages-4Qu8GNlt.js": {
+    "file": "assets/pages-4Qu8GNlt.js",
     "name": "pages",
     "imports": [
-      "_vendor-RdRDaQiR.js",
-      "_vendor-arizeai-DsYDNOqt.js",
-      "_components-Cs9c4Nxp.js",
-      "_vendor-codemirror-BzJDUbEx.js",
-      "_vendor-recharts-BTHn5Y2R.js"
+      "_vendor-CRRxHwSp.js",
+      "_vendor-arizeai-CUN6lRd9.js",
+      "_components-YTHUASXI.js",
+      "_vendor-codemirror-sJXwoqrE.js",
+      "_vendor-recharts-BT_PeGhc.js"
     ]
   },
-  "_vendor-CqDb5u4o.css": {
-    "file": "assets/vendor-CqDb5u4o.css",
-    "src": "_vendor-CqDb5u4o.css"
+  "_vendor-BGzfc4EU.css": {
+    "file": "assets/vendor-BGzfc4EU.css",
+    "src": "_vendor-BGzfc4EU.css"
   },
-  "_vendor-RdRDaQiR.js": {
-    "file": "assets/vendor-RdRDaQiR.js",
+  "_vendor-CRRxHwSp.js": {
+    "file": "assets/vendor-CRRxHwSp.js",
     "name": "vendor",
     "imports": [
       "_vendor-three-BLWp5bic.js"
     ],
     "css": [
-      "assets/vendor-CqDb5u4o.css"
+      "assets/vendor-BGzfc4EU.css"
     ]
   },
-  "_vendor-arizeai-DsYDNOqt.js": {
-    "file": "assets/vendor-arizeai-DsYDNOqt.js",
+  "_vendor-arizeai-CUN6lRd9.js": {
+    "file": "assets/vendor-arizeai-CUN6lRd9.js",
     "name": "vendor-arizeai",
     "imports": [
-      "_vendor-RdRDaQiR.js"
+      "_vendor-CRRxHwSp.js"
     ]
   },
-  "_vendor-codemirror-BzJDUbEx.js": {
-    "file": "assets/vendor-codemirror-BzJDUbEx.js",
+  "_vendor-codemirror-sJXwoqrE.js": {
+    "file": "assets/vendor-codemirror-sJXwoqrE.js",
     "name": "vendor-codemirror",
     "imports": [
-      "_vendor-RdRDaQiR.js",
-      "_vendor-shiki-BAcocHFl.js"
+      "_vendor-CRRxHwSp.js",
+      "_vendor-shiki-1F3op0QC.js"
     ],
     "dynamicImports": [
-      "_vendor-shiki-BAcocHFl.js",
-      "_vendor-shiki-BAcocHFl.js",
-      "_vendor-shiki-BAcocHFl.js"
+      "_vendor-shiki-1F3op0QC.js",
+      "_vendor-shiki-1F3op0QC.js",
+      "_vendor-shiki-1F3op0QC.js"
     ]
   },
-  "_vendor-recharts-BTHn5Y2R.js": {
-    "file": "assets/vendor-recharts-BTHn5Y2R.js",
+  "_vendor-recharts-BT_PeGhc.js": {
+    "file": "assets/vendor-recharts-BT_PeGhc.js",
     "name": "vendor-recharts",
     "imports": [
-      "_vendor-RdRDaQiR.js"
+      "_vendor-CRRxHwSp.js"
     ]
   },
-  "_vendor-shiki-BAcocHFl.js": {
-    "file": "assets/vendor-shiki-BAcocHFl.js",
+  "_vendor-shiki-1F3op0QC.js": {
+    "file": "assets/vendor-shiki-1F3op0QC.js",
     "name": "vendor-shiki",
     "isDynamicEntry": true,
     "imports": [
-      "_vendor-RdRDaQiR.js"
+      "_vendor-CRRxHwSp.js"
     ]
   },
   "_vendor-three-BLWp5bic.js": {
@@ -75,19 +75,19 @@
     "name": "vendor-three"
   },
   "index.tsx": {
-    "file": "assets/index-D1FDMBMV.js",
+    "file": "assets/index-CugQp26L.js",
     "name": "index",
     "src": "index.tsx",
     "isEntry": true,
     "imports": [
-      "_vendor-RdRDaQiR.js",
-      "_vendor-arizeai-DsYDNOqt.js",
-      "_pages-Cbj9SjBx.js",
-      "_components-Cs9c4Nxp.js",
+      "_vendor-CRRxHwSp.js",
+      "_vendor-arizeai-CUN6lRd9.js",
+      "_pages-4Qu8GNlt.js",
+      "_components-YTHUASXI.js",
       "_vendor-three-BLWp5bic.js",
-      "_vendor-codemirror-BzJDUbEx.js",
-      "_vendor-shiki-BAcocHFl.js",
-      "_vendor-recharts-BTHn5Y2R.js"
+      "_vendor-codemirror-sJXwoqrE.js",
+      "_vendor-shiki-1F3op0QC.js",
+      "_vendor-recharts-BT_PeGhc.js"
     ]
   }
 }