arize-phoenix 12.5.0__tar.gz → 12.6.1__tar.gz

{arize_phoenix-12.5.0 → arize_phoenix-12.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: arize-phoenix
-Version: 12.5.0
+Version: 12.6.1
 Summary: AI Observability and Evaluation
 Project-URL: Documentation, https://arize.com/docs/phoenix/
 Project-URL: Issues, https://github.com/Arize-ai/phoenix/issues

arize_phoenix-12.6.1/src/phoenix/server/api/input_types/ExperimentRunSort.py

@@ -0,0 +1,237 @@
+import operator
+from enum import Enum, auto
+from typing import Any, Optional
+
+import strawberry
+from sqlalchemy import ColumnElement, Select, and_, func, literal, or_, select, tuple_
+from sqlalchemy.sql.selectable import NamedFromClause
+from strawberry import Maybe
+from typing_extensions import assert_never
+
+from phoenix.db import models
+from phoenix.server.api.types.pagination import (
+    Cursor,
+    CursorSortColumn,
+    CursorSortColumnDataType,
+    CursorSortColumnValue,
+)
+from phoenix.server.api.types.SortDir import SortDir
+
+
+@strawberry.enum
+class ExperimentRunMetric(Enum):
+    latencyMs = auto()
+
+
+@strawberry.input(one_of=True)
+class ExperimentRunColumn:
+    metric: Maybe[ExperimentRunMetric]
+    annotation_name: Maybe[str]
+
+
+@strawberry.input(description="The sort key and direction for experiment run connections")
+class ExperimentRunSort:
+    col: ExperimentRunColumn
+    dir: SortDir
+
+
+def get_experiment_run_cursor(
+    run: models.ExperimentRun, annotation_score: Optional[float], sort: Optional[ExperimentRunSort]
+) -> Cursor:
+    sort_column: Optional[CursorSortColumn] = None
+    if sort:
+        if sort.col.metric:
+            metric = sort.col.metric.value
+            assert metric is not None
+            if metric is ExperimentRunMetric.latencyMs:
+                sort_column = CursorSortColumn(
+                    type=CursorSortColumnDataType.FLOAT,
+                    value=run.latency_ms,
+                )
+            else:
+                assert_never(metric)
+        elif sort.col.annotation_name:
+            data_type = (
+                CursorSortColumnDataType.FLOAT
+                if annotation_score is not None
+                else CursorSortColumnDataType.NULL
+            )
+            sort_column = CursorSortColumn(
+                type=data_type,
+                value=annotation_score,
+            )
+    return Cursor(rowid=run.id, sort_column=sort_column)
+
+
+def add_order_by_and_page_start_to_query(
+    query: Select[Any],
+    sort: Optional[ExperimentRunSort],
+    experiment_rowid: int,
+    after_experiment_run_rowid: Optional[int],
+    after_sort_column_value: Optional[CursorSortColumnValue] = None,
+) -> Select[Any]:
+    mean_annotation_scores: Optional[NamedFromClause] = None
+    if sort and sort.col.annotation_name:
+        annotation_name = sort.col.annotation_name.value
+        assert annotation_name is not None
+        mean_annotation_scores = _get_mean_annotation_scores_subquery(annotation_name)
+    order_by_columns = _get_order_by_columns(
+        sort=sort, experiment_rowid=experiment_rowid, mean_annotation_scores=mean_annotation_scores
+    )
+    query = query.order_by(*order_by_columns)
+    if after_experiment_run_rowid is not None:
+        query = _add_after_expression(
+            query=query,
+            sort=sort,
+            experiment_run_rowid=after_experiment_run_rowid,
+            after_sort_column_value=after_sort_column_value,
+            mean_annotation_scores=mean_annotation_scores,
+        )
+    query = _add_joins_and_selects_to_query(
+        query=query,
+        sort=sort,
+        mean_annotation_scores=mean_annotation_scores,
+    )
+    return query
+
+
+def _get_order_by_columns(
+    sort: Optional[ExperimentRunSort],
+    experiment_rowid: int,
+    mean_annotation_scores: Optional[NamedFromClause],
+) -> tuple[ColumnElement[Any], ...]:
+    if not sort:
+        # Ideally, this would sort the runs by (example_id, repetition_number),
+        # but this would require making the cursor more complex or adding an additional query
+        # to handle the after cursor.
+        return (models.ExperimentRun.id.asc(),)
+    sort_direction = sort.dir
+    if sort.col.metric:
+        metric = sort.col.metric.value
+        assert metric is not None
+        if metric is ExperimentRunMetric.latencyMs:
+            if sort_direction is SortDir.asc:
+                return (models.ExperimentRun.latency_ms.asc(), models.ExperimentRun.id.asc())
+            else:
+                return (models.ExperimentRun.latency_ms.desc(), models.ExperimentRun.id.desc())
+        else:
+            assert_never(metric)
+    elif sort.col.annotation_name:
+        annotation_name = sort.col.annotation_name.value
+        assert annotation_name is not None
+        assert mean_annotation_scores is not None
+        if sort_direction is SortDir.asc:
+            return (
+                mean_annotation_scores.c.score.asc().nulls_last(),
+                models.ExperimentRun.id.asc(),
+            )
+        else:
+            return (
+                mean_annotation_scores.c.score.desc().nulls_last(),
+                models.ExperimentRun.id.desc(),
+            )
+    raise NotImplementedError
+
+
+def _add_after_expression(
+    query: Select[Any],
+    sort: Optional[ExperimentRunSort],
+    experiment_run_rowid: int,
+    after_sort_column_value: Optional[CursorSortColumnValue],
+    mean_annotation_scores: Optional[NamedFromClause],
+) -> Select[Any]:
+    if not sort:
+        # Ideally, this would return the runs sorted by (example_id, repetition_number),
+        # but this would require making the cursor more complex or adding an additional query.
+        return query.where(models.ExperimentRun.id > literal(experiment_run_rowid))
+    sort_direction = sort.dir
+    compare_fn = operator.gt if sort_direction is SortDir.asc else operator.lt
+    if sort.col.metric:
+        metric = sort.col.metric.value
+        assert metric is not None
+        if metric is ExperimentRunMetric.latencyMs:
+            assert after_sort_column_value is not None
+            return query.where(
+                compare_fn(
+                    tuple_(models.ExperimentRun.latency_ms, models.ExperimentRun.id),
+                    tuple_(
+                        literal(after_sort_column_value),
+                        literal(experiment_run_rowid),
+                    ),
+                )
+            )
+        else:
+            assert_never(metric)
+    elif sort.col.annotation_name:
+        annotation_name = sort.col.annotation_name.value
+        assert annotation_name is not None
+        assert mean_annotation_scores is not None
+        if after_sort_column_value is None:
+            return query.where(
+                and_(
+                    compare_fn(models.ExperimentRun.id, literal(experiment_run_rowid)),
+                    mean_annotation_scores.c.score.is_(None),
+                )
+            )
+        else:
+            return query.where(
+                or_(
+                    compare_fn(
+                        tuple_(mean_annotation_scores.c.score, models.ExperimentRun.id),
+                        tuple_(
+                            literal(after_sort_column_value),
+                            literal(experiment_run_rowid),
+                        ),
+                    ),
+                    mean_annotation_scores.c.score.is_(None),
+                )
+            )
+    raise NotImplementedError
+
+
+def _get_mean_annotation_scores_subquery(annotation_name: str) -> NamedFromClause:
+    return (
+        select(
+            func.avg(models.ExperimentRunAnnotation.score).label("score"),
+            models.ExperimentRunAnnotation.experiment_run_id.label("experiment_run_id"),
+        )
+        .select_from(models.ExperimentRunAnnotation)
+        .join(
+            models.ExperimentRun,
+            models.ExperimentRunAnnotation.experiment_run_id == models.ExperimentRun.id,
+        )
+        .where(models.ExperimentRunAnnotation.name == annotation_name)
+        .group_by(models.ExperimentRunAnnotation.experiment_run_id)
+        .subquery()
+        .alias("mean_annotation_scores")
+    )
+
+
+def _add_joins_and_selects_to_query(
+    query: Select[tuple[models.ExperimentRun]],
+    sort: Optional[ExperimentRunSort],
+    mean_annotation_scores: Optional[NamedFromClause],
+) -> Select[tuple[models.ExperimentRun]]:
+    if not sort:
+        return query
+    if sort.col.metric:
+        metric = sort.col.metric.value
+        assert metric is not None
+        if metric is ExperimentRunMetric.latencyMs:
+            return query
+        else:
+            assert_never(metric)
+    elif sort.col.annotation_name:
+        annotation_name = sort.col.annotation_name.value
+        assert annotation_name is not None
+        assert mean_annotation_scores is not None
+        query = query.join(
+            mean_annotation_scores,
+            mean_annotation_scores.c.experiment_run_id == models.ExperimentRun.id,
+            isouter=True,
+        )
+        query = query.add_columns(
+            mean_annotation_scores.c.score.label("score")
+        )  # the score must be in the select so that the value can be included in the cursor
+        return query
+    raise NotImplementedError

{arize_phoenix-12.5.0 → arize_phoenix-12.6.1}/src/phoenix/server/api/types/Experiment.py

@@ -11,19 +11,27 @@ from strawberry.types import Info
 
 from phoenix.db import models
 from phoenix.server.api.context import Context
+from phoenix.server.api.exceptions import BadRequest
+from phoenix.server.api.input_types.ExperimentRunSort import (
+    ExperimentRunSort,
+    add_order_by_and_page_start_to_query,
+    get_experiment_run_cursor,
+)
 from phoenix.server.api.types.CostBreakdown import CostBreakdown
 from phoenix.server.api.types.DatasetVersion import DatasetVersion
 from phoenix.server.api.types.ExperimentAnnotationSummary import ExperimentAnnotationSummary
 from phoenix.server.api.types.ExperimentRun import ExperimentRun, to_gql_experiment_run
 from phoenix.server.api.types.pagination import (
-    ConnectionArgs,
+    Cursor,
     CursorString,
-    connection_from_list,
+    connection_from_cursors_and_nodes,
 )
 from phoenix.server.api.types.Project import Project
 from phoenix.server.api.types.SpanCostDetailSummaryEntry import SpanCostDetailSummaryEntry
 from phoenix.server.api.types.SpanCostSummary import SpanCostSummary
 
+_DEFAULT_EXPERIMENT_RUNS_PAGE_SIZE = 50
+
 
 @strawberry.type
 class Experiment(Node):
@@ -57,33 +65,60 @@ class Experiment(Node):
     async def runs(
         self,
         info: Info[Context, None],
-        first: Optional[int] = 50,
-        last: Optional[int] = UNSET,
+        first: Optional[int] = _DEFAULT_EXPERIMENT_RUNS_PAGE_SIZE,
         after: Optional[CursorString] = UNSET,
-        before: Optional[CursorString] = UNSET,
+        sort: Optional[ExperimentRunSort] = UNSET,
     ) -> Connection[ExperimentRun]:
-        args = ConnectionArgs(
-            first=first,
-            after=after if isinstance(after, CursorString) else None,
-            last=last,
-            before=before if isinstance(before, CursorString) else None,
+        if first is not None and first <= 0:
+            raise BadRequest("first must be a positive integer if set")
+        experiment_rowid = self.id_attr
+        page_size = first or _DEFAULT_EXPERIMENT_RUNS_PAGE_SIZE
+        experiment_runs_query = (
+            select(models.ExperimentRun)
+            .where(models.ExperimentRun.experiment_id == experiment_rowid)
+            .options(joinedload(models.ExperimentRun.trace).load_only(models.Trace.trace_id))
+            .limit(page_size + 1)
         )
-        experiment_id = self.id_attr
+
+        after_experiment_run_rowid = None
+        after_sort_column_value = None
+        if after:
+            cursor = Cursor.from_string(after)
+            after_experiment_run_rowid = cursor.rowid
+            if cursor.sort_column is not None:
+                after_sort_column_value = cursor.sort_column.value
+
+        experiment_runs_query = add_order_by_and_page_start_to_query(
+            query=experiment_runs_query,
+            sort=sort,
+            experiment_rowid=experiment_rowid,
+            after_experiment_run_rowid=after_experiment_run_rowid,
+            after_sort_column_value=after_sort_column_value,
+        )
+
         async with info.context.db() as session:
-            runs = (
-                await session.scalars(
-                    select(models.ExperimentRun)
-                    .where(models.ExperimentRun.experiment_id == experiment_id)
-                    .order_by(
-                        models.ExperimentRun.dataset_example_id.asc(),
-                        models.ExperimentRun.repetition_number.asc(),
-                    )
-                    .options(
-                        joinedload(models.ExperimentRun.trace).load_only(models.Trace.trace_id)
-                    )
-                )
-            ).all()
-        return connection_from_list([to_gql_experiment_run(run) for run in runs], args)
+            results = (await session.execute(experiment_runs_query)).all()
+
+        has_next_page = False
+        if len(results) > page_size:
+            results = results[:page_size]
+            has_next_page = True
+
+        cursors_and_nodes = []
+        for result in results:
+            run = result[0]
+            annotation_score = result[1] if len(result) > 1 else None
+            gql_run = to_gql_experiment_run(run)
+            cursor = get_experiment_run_cursor(
+                run=run, annotation_score=annotation_score, sort=sort
+            )
+            cursors_and_nodes.append((cursor, gql_run))
+
+        return connection_from_cursors_and_nodes(
+            cursors_and_nodes=cursors_and_nodes,
+            has_previous_page=False,  # set to false since we are only doing forward pagination (https://relay.dev/graphql/connections.htm#sec-undefined.PageInfo.Fields) # noqa: E501
+            has_next_page=has_next_page,
+        )
 
     @strawberry.field
     async def run_count(self, info: Info[Context, None]) -> int:

arize_phoenix-12.6.1/src/phoenix/server/static/.vite/manifest.json

@@ -0,0 +1,93 @@
+{
+  "_components-XKc983B9.js": {
+    "file": "assets/components-XKc983B9.js",
+    "name": "components",
+    "imports": [
+      "_vendor-CQ4tN9P7.js",
+      "_pages-CSZW-lt0.js",
+      "_vendor-arizeai-Cb1ncvYH.js",
+      "_vendor-codemirror-CckmKopH.js",
+      "_vendor-three-BtCyLs1w.js"
+    ]
+  },
+  "_pages-CSZW-lt0.js": {
+    "file": "assets/pages-CSZW-lt0.js",
+    "name": "pages",
+    "imports": [
+      "_vendor-CQ4tN9P7.js",
+      "_components-XKc983B9.js",
+      "_vendor-arizeai-Cb1ncvYH.js",
+      "_vendor-codemirror-CckmKopH.js",
+      "_vendor-recharts-BC1ysIKu.js"
+    ]
+  },
+  "_vendor-BGzfc4EU.css": {
+    "file": "assets/vendor-BGzfc4EU.css",
+    "src": "_vendor-BGzfc4EU.css"
+  },
+  "_vendor-CQ4tN9P7.js": {
+    "file": "assets/vendor-CQ4tN9P7.js",
+    "name": "vendor",
+    "imports": [
+      "_vendor-three-BtCyLs1w.js"
+    ],
+    "css": [
+      "assets/vendor-BGzfc4EU.css"
+    ]
+  },
+  "_vendor-arizeai-Cb1ncvYH.js": {
+    "file": "assets/vendor-arizeai-Cb1ncvYH.js",
+    "name": "vendor-arizeai",
+    "imports": [
+      "_vendor-CQ4tN9P7.js"
+    ]
+  },
+  "_vendor-codemirror-CckmKopH.js": {
+    "file": "assets/vendor-codemirror-CckmKopH.js",
+    "name": "vendor-codemirror",
+    "imports": [
+      "_vendor-CQ4tN9P7.js",
+      "_vendor-shiki-B45T-YxN.js"
+    ],
+    "dynamicImports": [
+      "_vendor-shiki-B45T-YxN.js",
+      "_vendor-shiki-B45T-YxN.js",
+      "_vendor-shiki-B45T-YxN.js"
+    ]
+  },
+  "_vendor-recharts-BC1ysIKu.js": {
+    "file": "assets/vendor-recharts-BC1ysIKu.js",
+    "name": "vendor-recharts",
+    "imports": [
+      "_vendor-CQ4tN9P7.js"
+    ]
+  },
+  "_vendor-shiki-B45T-YxN.js": {
+    "file": "assets/vendor-shiki-B45T-YxN.js",
+    "name": "vendor-shiki",
+    "isDynamicEntry": true,
+    "imports": [
+      "_vendor-CQ4tN9P7.js"
+    ]
+  },
+  "_vendor-three-BtCyLs1w.js": {
+    "file": "assets/vendor-three-BtCyLs1w.js",
+    "name": "vendor-three"
+  },
+  "index.tsx": {
+    "file": "assets/index-DG8e74sg.js",
+    "name": "index",
+    "src": "index.tsx",
+    "isEntry": true,
+    "imports": [
+      "_vendor-CQ4tN9P7.js",
+      "_vendor-arizeai-Cb1ncvYH.js",
+      "_pages-CSZW-lt0.js",
+      "_components-XKc983B9.js",
+      "_vendor-three-BtCyLs1w.js",
+      "_vendor-codemirror-CckmKopH.js",
+      "_vendor-shiki-B45T-YxN.js",
+      "_vendor-recharts-BC1ysIKu.js"
+    ]
+  }
+}