arize-phoenix 4.16.1__tar.gz → 4.18.0__tar.gz

{arize_phoenix-4.16.1 → arize_phoenix-4.18.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: arize-phoenix
-Version: 4.16.1
+Version: 4.18.0
 Summary: AI Observability and Evaluation
 Project-URL: Documentation, https://docs.arize.com/phoenix/
 Project-URL: Issues, https://github.com/Arize-ai/phoenix/issues
@@ -28,10 +28,10 @@ Requires-Dist: hdbscan>=0.8.33
 Requires-Dist: httpx
 Requires-Dist: jinja2
 Requires-Dist: numpy<2
-Requires-Dist: openinference-instrumentation
-Requires-Dist: openinference-instrumentation-langchain>=0.1.12
-Requires-Dist: openinference-instrumentation-llama-index>=1.2.0
-Requires-Dist: openinference-instrumentation-openai>=0.1.4
+Requires-Dist: openinference-instrumentation-langchain>=0.1.26
+Requires-Dist: openinference-instrumentation-llama-index>=2.2.1
+Requires-Dist: openinference-instrumentation-openai>=0.1.11
+Requires-Dist: openinference-instrumentation>=0.1.12
 Requires-Dist: openinference-semantic-conventions>=0.1.9
 Requires-Dist: opentelemetry-exporter-otlp
 Requires-Dist: opentelemetry-proto>=1.12.0

{arize_phoenix-4.16.1 → arize_phoenix-4.18.0}/pyproject.toml

@@ -47,10 +47,10 @@ dependencies = [
   "opentelemetry-exporter-otlp",
   "opentelemetry-semantic-conventions",
   "openinference-semantic-conventions>=0.1.9",
-  "openinference-instrumentation",
-  "openinference-instrumentation-langchain>=0.1.12",
-  "openinference-instrumentation-llama-index>=1.2.0",
-  "openinference-instrumentation-openai>=0.1.4",
+  "openinference-instrumentation>=0.1.12",
+  "openinference-instrumentation-langchain>=0.1.26",
+  "openinference-instrumentation-llama-index>=2.2.1",
+  "openinference-instrumentation-openai>=0.1.11",
   "sqlalchemy[asyncio]>=2.0.4, <3",
   "alembic>=1.3.0, <2",
   "aiosqlite",

{arize_phoenix-4.16.1 → arize_phoenix-4.18.0}/src/phoenix/server/api/context.py

@@ -8,6 +8,7 @@ from typing_extensions import TypeAlias
 
 from phoenix.core.model_schema import Model
 from phoenix.server.api.dataloaders import (
+    AnnotationSummaryDataLoader,
     AverageExperimentRunLatencyDataLoader,
     CacheForDataLoaders,
     DatasetExampleRevisionsDataLoader,
@@ -44,6 +45,7 @@ class DataLoaders:
     document_evaluation_summaries: DocumentEvaluationSummaryDataLoader
     document_evaluations: DocumentEvaluationsDataLoader
     document_retrieval_metrics: DocumentRetrievalMetricsDataLoader
+    annotation_summaries: AnnotationSummaryDataLoader
     evaluation_summaries: EvaluationSummaryDataLoader
     experiment_annotation_summaries: ExperimentAnnotationSummaryDataLoader
     experiment_error_rates: ExperimentErrorRatesDataLoader

{arize_phoenix-4.16.1 → arize_phoenix-4.18.0}/src/phoenix/server/api/dataloaders/__init__.py

@@ -8,6 +8,7 @@ from phoenix.db.insertion.evaluation import (
 )
 from phoenix.db.insertion.span import ClearProjectSpansEvent, SpanInsertionEvent
 
+from .annotation_summaries import AnnotationSummaryCache, AnnotationSummaryDataLoader
 from .average_experiment_run_latency import AverageExperimentRunLatencyDataLoader
 from .dataset_example_revisions import DatasetExampleRevisionsDataLoader
 from .dataset_example_spans import DatasetExampleSpansDataLoader
@@ -43,6 +44,7 @@ __all__ = [
     "DocumentEvaluationSummaryDataLoader",
     "DocumentEvaluationsDataLoader",
     "DocumentRetrievalMetricsDataLoader",
+    "AnnotationSummaryDataLoader",
     "EvaluationSummaryDataLoader",
     "ExperimentAnnotationSummaryDataLoader",
     "ExperimentErrorRatesDataLoader",
@@ -68,6 +70,9 @@ class CacheForDataLoaders:
     document_evaluation_summary: DocumentEvaluationSummaryCache = field(
         default_factory=DocumentEvaluationSummaryCache,
     )
+    annotation_summary: AnnotationSummaryCache = field(
+        default_factory=AnnotationSummaryCache,
+    )
     evaluation_summary: EvaluationSummaryCache = field(
         default_factory=EvaluationSummaryCache,
     )
@@ -92,6 +97,7 @@ class CacheForDataLoaders:
 
     def _clear_spans(self, project_rowid: int) -> None:
         self._update_spans(project_rowid)
+        self.annotation_summary.invalidate_project(project_rowid)
         self.evaluation_summary.invalidate_project(project_rowid)
         self.document_evaluation_summary.invalidate_project(project_rowid)
 
@@ -113,9 +119,11 @@ class CacheForDataLoaders:
     @invalidate.register
     def _(self, event: SpanEvaluationInsertionEvent) -> None:
         project_rowid, evaluation_name = event
+        self.annotation_summary.invalidate((project_rowid, evaluation_name, "span"))
         self.evaluation_summary.invalidate((project_rowid, evaluation_name, "span"))
 
     @invalidate.register
     def _(self, event: TraceEvaluationInsertionEvent) -> None:
         project_rowid, evaluation_name = event
+        self.annotation_summary.invalidate((project_rowid, evaluation_name, "trace"))
         self.evaluation_summary.invalidate((project_rowid, evaluation_name, "trace"))

arize_phoenix-4.18.0/src/phoenix/server/api/dataloaders/annotation_summaries.py

@@ -0,0 +1,146 @@
+from collections import defaultdict
+from datetime import datetime
+from typing import (
+    Any,
+    DefaultDict,
+    List,
+    Literal,
+    Optional,
+    Tuple,
+)
+
+import pandas as pd
+from aioitertools.itertools import groupby
+from cachetools import LFUCache, TTLCache
+from sqlalchemy import Select, func, or_, select
+from strawberry.dataloader import AbstractCache, DataLoader
+from typing_extensions import TypeAlias, assert_never
+
+from phoenix.db import models
+from phoenix.server.api.dataloaders.cache import TwoTierCache
+from phoenix.server.api.input_types.TimeRange import TimeRange
+from phoenix.server.api.types.AnnotationSummary import AnnotationSummary
+from phoenix.server.types import DbSessionFactory
+from phoenix.trace.dsl import SpanFilter
+
+Kind: TypeAlias = Literal["span", "trace"]
+ProjectRowId: TypeAlias = int
+TimeInterval: TypeAlias = Tuple[Optional[datetime], Optional[datetime]]
+FilterCondition: TypeAlias = Optional[str]
+EvalName: TypeAlias = str
+
+Segment: TypeAlias = Tuple[Kind, ProjectRowId, TimeInterval, FilterCondition]
+Param: TypeAlias = EvalName
+
+Key: TypeAlias = Tuple[Kind, ProjectRowId, Optional[TimeRange], FilterCondition, EvalName]
+Result: TypeAlias = Optional[AnnotationSummary]
+ResultPosition: TypeAlias = int
+DEFAULT_VALUE: Result = None
+
+
+def _cache_key_fn(key: Key) -> Tuple[Segment, Param]:
+    kind, project_rowid, time_range, filter_condition, eval_name = key
+    interval = (
+        (time_range.start, time_range.end) if isinstance(time_range, TimeRange) else (None, None)
+    )
+    return (kind, project_rowid, interval, filter_condition), eval_name
+
+
+_Section: TypeAlias = Tuple[ProjectRowId, EvalName, Kind]
+_SubKey: TypeAlias = Tuple[TimeInterval, FilterCondition]
+
+
+class AnnotationSummaryCache(
+    TwoTierCache[Key, Result, _Section, _SubKey],
+):
+    def __init__(self) -> None:
+        super().__init__(
+            # TTL=3600 (1-hour) because time intervals are always moving forward, but
+            # interval endpoints are rounded down to the hour by the UI, so anything
+            # older than an hour most likely won't be a cache-hit anyway.
+            main_cache=TTLCache(maxsize=64 * 32 * 2, ttl=3600),
+            sub_cache_factory=lambda: LFUCache(maxsize=2 * 2),
+        )
+
+    def invalidate_project(self, project_rowid: ProjectRowId) -> None:
+        for section in self._cache.keys():
+            if section[0] == project_rowid:
+                del self._cache[section]
+
+    def _cache_key(self, key: Key) -> Tuple[_Section, _SubKey]:
+        (kind, project_rowid, interval, filter_condition), eval_name = _cache_key_fn(key)
+        return (project_rowid, eval_name, kind), (interval, filter_condition)
+
+
+class AnnotationSummaryDataLoader(DataLoader[Key, Result]):
+    def __init__(
+        self,
+        db: DbSessionFactory,
+        cache_map: Optional[AbstractCache[Key, Result]] = None,
+    ) -> None:
+        super().__init__(
+            load_fn=self._load_fn,
+            cache_key_fn=_cache_key_fn,
+            cache_map=cache_map,
+        )
+        self._db = db
+
+    async def _load_fn(self, keys: List[Key]) -> List[Result]:
+        results: List[Result] = [DEFAULT_VALUE] * len(keys)
+        arguments: DefaultDict[
+            Segment,
+            DefaultDict[Param, List[ResultPosition]],
+        ] = defaultdict(lambda: defaultdict(list))
+        for position, key in enumerate(keys):
+            segment, param = _cache_key_fn(key)
+            arguments[segment][param].append(position)
+        for segment, params in arguments.items():
+            stmt = _get_stmt(segment, *params.keys())
+            async with self._db() as session:
+                data = await session.stream(stmt)
+                async for eval_name, group in groupby(data, lambda row: row.name):
+                    summary = AnnotationSummary(pd.DataFrame(group))
+                    for position in params[eval_name]:
+                        results[position] = summary
+        return results
+
+
+def _get_stmt(
+    segment: Segment,
+    *eval_names: Param,
+) -> Select[Any]:
+    kind, project_rowid, (start_time, end_time), filter_condition = segment
+    stmt = select()
+    if kind == "span":
+        msa = models.SpanAnnotation
+        name_column, label_column, score_column = msa.name, msa.label, msa.score
+        time_column = models.Span.start_time
+        stmt = stmt.join(models.Span).join_from(models.Span, models.Trace)
+        if filter_condition:
+            sf = SpanFilter(filter_condition)
+            stmt = sf(stmt)
+    elif kind == "trace":
+        mta = models.TraceAnnotation
+        name_column, label_column, score_column = mta.name, mta.label, mta.score
+        time_column = models.Trace.start_time
+        stmt = stmt.join(models.Trace)
+    else:
+        assert_never(kind)
+    stmt = stmt.add_columns(
+        name_column,
+        label_column,
+        func.count().label("record_count"),
+        func.count(label_column).label("label_count"),
+        func.count(score_column).label("score_count"),
+        func.sum(score_column).label("score_sum"),
+    )
+    stmt = stmt.group_by(name_column, label_column)
+    stmt = stmt.order_by(name_column, label_column)
+    stmt = stmt.where(models.Trace.project_rowid == project_rowid)
+    stmt = stmt.where(or_(score_column.is_not(None), label_column.is_not(None)))
+    stmt = stmt.where(name_column.in_(eval_names))
+    if start_time:
+        stmt = stmt.where(start_time <= time_column)
+    if end_time:
+        stmt = stmt.where(time_column < end_time)
+    return stmt

{arize_phoenix-4.16.1 → arize_phoenix-4.18.0}/src/phoenix/server/api/mutations/span_annotations_mutations.py

@@ -73,6 +73,7 @@ class SpanAnnotationMutationMixin:
                             models.SpanAnnotation.annotator_kind,
                             annotation.annotator_kind.value
                             if annotation.annotator_kind is not None
+                            and annotation.annotator_kind is not UNSET
                             else None,
                             False,
                         ),

arize_phoenix-4.18.0/src/phoenix/server/api/types/AnnotationSummary.py

@@ -0,0 +1,55 @@
+from typing import List, Optional, Union, cast
+
+import pandas as pd
+import strawberry
+from strawberry import Private
+
+from phoenix.db import models
+from phoenix.server.api.types.LabelFraction import LabelFraction
+
+AnnotationType = Union[models.SpanAnnotation, models.TraceAnnotation]
+
+
+@strawberry.type
+class AnnotationSummary:
+    df: Private[pd.DataFrame]
+
+    def __init__(self, dataframe: pd.DataFrame) -> None:
+        self.df = dataframe
+
+    @strawberry.field
+    def count(self) -> int:
+        return cast(int, self.df.record_count.sum())
+
+    @strawberry.field
+    def labels(self) -> List[str]:
+        return self.df.label.dropna().tolist()
+
+    @strawberry.field
+    def label_fractions(self) -> List[LabelFraction]:
+        if not (n := self.df.label_count.sum()):
+            return []
+        return [
+            LabelFraction(
+                label=cast(str, row.label),
+                fraction=row.label_count / n,
+            )
+            for row in self.df.loc[
+                self.df.label.notna(),
+                ["label", "label_count"],
+            ].itertuples()
+        ]
+
+    @strawberry.field
+    def mean_score(self) -> Optional[float]:
+        if not (n := self.df.score_count.sum()):
+            return None
+        return cast(float, self.df.score_sum.sum() / n)
+
+    @strawberry.field
+    def score_count(self) -> int:
+        return cast(int, self.df.score_count.sum())
+
+    @strawberry.field
+    def label_count(self) -> int:
+        return cast(int, self.df.label_count.sum())

{arize_phoenix-4.16.1 → arize_phoenix-4.18.0}/src/phoenix/server/api/types/EvaluationSummary.py

@@ -5,16 +5,11 @@ import strawberry
 from strawberry import Private
 
 from phoenix.db import models
+from phoenix.server.api.types.LabelFraction import LabelFraction
 
 AnnotationType = Union[models.SpanAnnotation, models.TraceAnnotation]
 
 
-@strawberry.type
-class LabelFraction:
-    label: str
-    fraction: float
-
-
 @strawberry.type
 class EvaluationSummary:
     df: Private[pd.DataFrame]

arize_phoenix-4.18.0/src/phoenix/server/api/types/LabelFraction.py

@@ -0,0 +1,7 @@
+import strawberry
+
+
+@strawberry.type
+class LabelFraction:
+    label: str
+    fraction: float

{arize_phoenix-4.16.1 → arize_phoenix-4.18.0}/src/phoenix/server/api/types/Project.py

@@ -20,6 +20,7 @@ from phoenix.db import models
 from phoenix.server.api.context import Context
 from phoenix.server.api.input_types.SpanSort import SpanSort, SpanSortConfig
 from phoenix.server.api.input_types.TimeRange import TimeRange
+from phoenix.server.api.types.AnnotationSummary import AnnotationSummary
 from phoenix.server.api.types.DocumentEvaluationSummary import DocumentEvaluationSummary
 from phoenix.server.api.types.EvaluationSummary import EvaluationSummary
 from phoenix.server.api.types.pagination import (
@@ -356,6 +357,29 @@ class Project(Node):
             ("span", self.id_attr, time_range, filter_condition, evaluation_name),
         )
 
+    @strawberry.field
+    async def trace_annotation_summary(
+        self,
+        info: Info[Context, None],
+        evaluation_name: str,
+        time_range: Optional[TimeRange] = UNSET,
+    ) -> Optional[AnnotationSummary]:
+        return await info.context.data_loaders.annotation_summaries.load(
+            ("trace", self.id_attr, time_range, None, evaluation_name),
+        )
+
+    @strawberry.field
+    async def span_annotation_summary(
+        self,
+        info: Info[Context, None],
+        evaluation_name: str,
+        time_range: Optional[TimeRange] = UNSET,
+        filter_condition: Optional[str] = UNSET,
+    ) -> Optional[AnnotationSummary]:
+        return await info.context.data_loaders.annotation_summaries.load(
+            ("span", self.id_attr, time_range, filter_condition, evaluation_name),
+        )
+
     @strawberry.field
     async def document_evaluation_summary(
         self,

{arize_phoenix-4.16.1 → arize_phoenix-4.18.0}/src/phoenix/server/app.py

@@ -58,6 +58,7 @@ from phoenix.exceptions import PhoenixMigrationError
 from phoenix.pointcloud.umap_parameters import UMAPParameters
 from phoenix.server.api.context import Context, DataLoaders
 from phoenix.server.api.dataloaders import (
+    AnnotationSummaryDataLoader,
     AverageExperimentRunLatencyDataLoader,
     CacheForDataLoaders,
     DatasetExampleRevisionsDataLoader,
@@ -286,6 +287,12 @@ def create_graphql_router(
                 ),
                 document_evaluations=DocumentEvaluationsDataLoader(db),
                 document_retrieval_metrics=DocumentRetrievalMetricsDataLoader(db),
+                annotation_summaries=AnnotationSummaryDataLoader(
+                    db,
+                    cache_map=cache_for_dataloaders.annotation_summary
+                    if cache_for_dataloaders
+                    else None,
+                ),
                 evaluation_summaries=EvaluationSummaryDataLoader(
                     db,
                     cache_map=cache_for_dataloaders.evaluation_summary

arize_phoenix-4.18.0/src/phoenix/server/static/.vite/manifest.json

@@ -0,0 +1,78 @@
+{
+  "_components-B2sHrnz0.js": {
+    "file": "assets/components-B2sHrnz0.js",
+    "name": "components",
+    "imports": [
+      "_vendor-BMWfu6zp.js",
+      "_vendor-arizeai-CkyzG9Wl.js",
+      "_pages-BTHjBGHq.js",
+      "_vendor-three-DwGkEfCM.js",
+      "_vendor-codemirror-DO3VqEcD.js"
+    ]
+  },
+  "_pages-BTHjBGHq.js": {
+    "file": "assets/pages-BTHjBGHq.js",
+    "name": "pages",
+    "imports": [
+      "_vendor-BMWfu6zp.js",
+      "_components-B2sHrnz0.js",
+      "_vendor-arizeai-CkyzG9Wl.js",
+      "_vendor-recharts-BGN0SxgJ.js",
+      "_vendor-codemirror-DO3VqEcD.js"
+    ]
+  },
+  "_vendor-!~{003}~.js": {
+    "file": "assets/vendor-DxkFTwjz.css",
+    "src": "_vendor-!~{003}~.js"
+  },
+  "_vendor-BMWfu6zp.js": {
+    "file": "assets/vendor-BMWfu6zp.js",
+    "name": "vendor",
+    "imports": [
+      "_vendor-three-DwGkEfCM.js"
+    ],
+    "css": [
+      "assets/vendor-DxkFTwjz.css"
+    ]
+  },
+  "_vendor-arizeai-CkyzG9Wl.js": {
+    "file": "assets/vendor-arizeai-CkyzG9Wl.js",
+    "name": "vendor-arizeai",
+    "imports": [
+      "_vendor-BMWfu6zp.js"
+    ]
+  },
+  "_vendor-codemirror-DO3VqEcD.js": {
+    "file": "assets/vendor-codemirror-DO3VqEcD.js",
+    "name": "vendor-codemirror",
+    "imports": [
+      "_vendor-BMWfu6zp.js"
+    ]
+  },
+  "_vendor-recharts-BGN0SxgJ.js": {
+    "file": "assets/vendor-recharts-BGN0SxgJ.js",
+    "name": "vendor-recharts",
+    "imports": [
+      "_vendor-BMWfu6zp.js"
+    ]
+  },
+  "_vendor-three-DwGkEfCM.js": {
+    "file": "assets/vendor-three-DwGkEfCM.js",
+    "name": "vendor-three"
+  },
+  "index.tsx": {
+    "file": "assets/index-BIFmgnip.js",
+    "name": "index",
+    "src": "index.tsx",
+    "isEntry": true,
+    "imports": [
+      "_vendor-BMWfu6zp.js",
+      "_vendor-arizeai-CkyzG9Wl.js",
+      "_components-B2sHrnz0.js",
+      "_pages-BTHjBGHq.js",
+      "_vendor-three-DwGkEfCM.js",
+      "_vendor-codemirror-DO3VqEcD.js",
+      "_vendor-recharts-BGN0SxgJ.js"
+    ]
+  }
+}