arize-phoenix 0.0.32__py3-none-any.whl → 0.0.33__py3-none-any.whl

phoenix/experimental/evals/templates/template.py

@@ -0,0 +1,107 @@
+from dataclasses import dataclass
+from string import Formatter
+from typing import Dict, List, Tuple, Union
+
+from ..utils.types import is_list_of
+
+DEFAULT_START_DELIM = "{"
+DEFAULT_END_DELIM = "}"
+
+
+@dataclass
+class PromptTemplate:
+    text: str
+    variables: List[str]
+
+    def __init__(self, text: str, delimiters: List[str] = [DEFAULT_START_DELIM, DEFAULT_END_DELIM]):
+        self.text = text
+        self._start_delim, self._end_delim = self._get_delimiters(delimiters)
+        self._validate()
+        self.variables = self._parse_variables()
+
+    def format(self, variable_values: Dict[str, Union[bool, int, float, str]]) -> str:
+        prompt = self.text
+        for variable_name in self.variables:
+            prompt = prompt.replace(
+                self._start_delim + variable_name + self._end_delim,
+                str(variable_values[variable_name]),
+            )
+        return prompt
+
+    def _get_delimiters(self, delimiters: List[str]) -> Tuple[str, str]:
+        if not is_list_of(delimiters, str):
+            raise TypeError("delimiters must be a list of strings")
+        if len(delimiters) == 1:
+            return delimiters[0], delimiters[0]
+        elif len(delimiters) == 2:
+            return delimiters[0], delimiters[1]
+        else:
+            raise ValueError("delimiters must only contain 2 items in the list")
+
+    def _validate(self) -> None:
+        # Validate that for every open delimiter, we have the corresponding closing one
+        start_count = self.text.count(self._start_delim)
+        end_count = self.text.count(self._end_delim)
+        if start_count != end_count:
+            raise ValueError(
+                f"text poorly formatted. Found {start_count} instances of delimiter "
+                f"{self._start_delim} and {end_count} instances of {self._end_delim}. "
+                "They must be equal to be correctly paired."
+            )
+
+    def _parse_variables(self) -> List[str]:
+        variables = []
+        formatter = Formatter()
+
+        text = self.text
+
+        # Example of this could be a template like: My name is ::name::
+        if self._start_delim == self._end_delim:
+            delim_length = len(self._start_delim)
+            delim_count = text.count(self._start_delim)
+            while delim_count > 0:
+                left_index = text.find(self._start_delim)
+                right_index = text[left_index + delim_length :].find(self._start_delim)
+                text = (
+                    text[0:left_index]
+                    + DEFAULT_START_DELIM
+                    + text[left_index + delim_length : left_index + delim_length + right_index]
+                    + DEFAULT_END_DELIM
+                    + text[left_index + 2 * delim_length + right_index :]
+                )
+                delim_count = text.count(self._start_delim)
+        else:
+            if self._start_delim != "{":
+                text = text.replace(self._start_delim, DEFAULT_START_DELIM)
+            if self._end_delim != "{":
+                text = text.replace(self._end_delim, DEFAULT_END_DELIM)
+
+        for _, variable_name, _, _ in formatter.parse(text):
+            if variable_name:
+                variables.append(variable_name)
+
+        return variables
+
+
+def normalize_template(template: Union[PromptTemplate, str]) -> "PromptTemplate":
+    """
+    Normalizes a template to a PromptTemplate object.
+    Args:
+        template (Union[PromptTemplate, str]): The template to be normalized.
+    Returns:
+        PromptTemplate: The normalized template.
+    """
+    normalized_template = template
+    if not (isinstance(template, PromptTemplate) or isinstance(template, str)):
+        raise TypeError(
+            "Invalid type for argument `template`. Expected a string or PromptTemplate "
+            f"but found {type(template)}."
+        )
+    if isinstance(template, str):
+        try:
+            normalized_template = PromptTemplate(text=template)
+        except Exception as e:
+            raise RuntimeError(f"Error while initializing the PromptTemplate: {e}")
+    else:
+        normalized_template = template
+    return normalized_template

phoenix/experimental/evals/utils/downloads.py

@@ -0,0 +1,33 @@
+"""
+Utility functions for evaluations.
+"""
+
+import json
+from io import BytesIO
+from urllib.error import HTTPError
+from urllib.request import urlopen
+from zipfile import ZipFile
+
+import pandas as pd
+
+
+def download_benchmark_dataset(task: str, dataset_name: str) -> pd.DataFrame:
+    """Downloads an Arize evals benchmark dataset as a pandas dataframe.
+
+    Args:
+        task (str): Task to be performed.
+        dataset_name (str): Name of the dataset.
+
+    Returns:
+        pandas.DataFrame: A pandas dataframe containing the data.
+    """
+    jsonl_file_name = f"{dataset_name}.jsonl"
+    url = f"http://storage.googleapis.com/arize-assets/phoenix/evals/{task}/{jsonl_file_name}.zip"
+    try:
+        with urlopen(url) as response:
+            zip_byte_stream = BytesIO(response.read())
+            with ZipFile(zip_byte_stream) as zip_file:
+                with zip_file.open(jsonl_file_name) as jsonl_file:
+                    return pd.DataFrame(map(json.loads, jsonl_file.readlines()))
+    except HTTPError:
+        raise ValueError(f'Dataset "{dataset_name}" for "{task}" task does not exist.')

phoenix/experimental/evals/utils/threads.py

@@ -0,0 +1,27 @@
+"""High-level support for working with threads in asyncio
+Directly copied from: https://github.com/python/cpython/blob/main/Lib/asyncio/threads.py#L12
+since this helper function 'to_thread' is not available in python<3.9
+"""
+
+import contextvars
+import functools
+from asyncio import events
+from typing import Any
+
+__all__ = ("to_thread",)
+
+
+async def to_thread(func, /, *args, **kwargs) -> Any:  # type:ignore
+    """Asynchronously run function *func* in a separate thread.
+
+    Any *args and **kwargs supplied for this function are directly passed
+    to *func*. Also, the current :class:`contextvars.Context` is propagated,
+    allowing context variables from the main thread to be accessed in the
+    separate thread.
+
+    Return a coroutine that can be awaited to get the eventual result of *func*.
+    """
+    loop = events.get_running_loop()
+    ctx = contextvars.copy_context()
+    func_call = functools.partial(ctx.run, func, *args, **kwargs)
+    return await loop.run_in_executor(None, func_call)

phoenix/experimental/evals/utils/types.py

@@ -0,0 +1,9 @@
+from typing import Sequence
+
+from typing_extensions import TypeVar
+
+T = TypeVar("T", bound=type)
+
+
+def is_list_of(lst: Sequence[object], tp: T) -> bool:
+    return isinstance(lst, list) and all(isinstance(x, tp) for x in lst)

phoenix/experimental/evals/utils.py

@@ -0,0 +1,33 @@
+"""
+Utility functions for evaluations.
+"""
+
+import json
+from io import BytesIO
+from urllib.error import HTTPError
+from urllib.request import urlopen
+from zipfile import ZipFile
+
+import pandas as pd
+
+
+def download_benchmark_dataset(task: str, dataset_name: str) -> pd.DataFrame:
+    """Downloads an Arize evals benchmark dataset as a pandas dataframe.
+
+    Args:
+        task (str): Task to be performed.
+        dataset_name (str): Name of the dataset.
+
+    Returns:
+        pandas.DataFrame: A pandas dataframe containing the data.
+    """
+    jsonl_file_name = f"{dataset_name}.jsonl"
+    url = f"http://storage.googleapis.com/arize-assets/phoenix/evals/{task}/{jsonl_file_name}.zip"
+    try:
+        with urlopen(url) as response:
+            zip_byte_stream = BytesIO(response.read())
+            with ZipFile(zip_byte_stream) as zip_file:
+                with zip_file.open(jsonl_file_name) as jsonl_file:
+                    return pd.DataFrame(map(json.loads, jsonl_file.readlines()))
+    except HTTPError:
+        raise ValueError(f'Dataset "{dataset_name}" for "{task}" task does not exist.')

phoenix/metrics/binning.py

@@ -154,7 +154,6 @@ class IntervalBinning(BinningMethod):
         return summary.set_axis(
             categories,
             axis=0,
-            copy=False,
         )
 
 

phoenix/metrics/timeseries.py

@@ -38,7 +38,7 @@ StopIndex: TypeAlias = int
 
 
 def row_interval_from_sorted_time_index(
-    time_index: pd.Index,
+    time_index: pd.DatetimeIndex,
     time_start: datetime,
     time_stop: datetime,
 ) -> Tuple[StartIndex, StopIndex]:
@@ -170,7 +170,7 @@ def _results(
         sampling_interval=sampling_interval,
     ):
         row_start, row_stop = row_interval_from_sorted_time_index(
-            time_index=dataframe.index,
+            time_index=cast(pd.DatetimeIndex, dataframe.index),
             time_start=time_start,  # inclusive
             time_stop=time_stop,  # exclusive
         )
@@ -191,7 +191,6 @@ def _results(
                     timezone.utc,
                 ),
                 axis=0,
-                copy=False,
             )
 
         yield res.loc[result_slice, :]

phoenix/server/api/context.py

@@ -7,6 +7,7 @@ from starlette.responses import Response
 from starlette.websockets import WebSocket
 
 from phoenix.core.model_schema import Model
+from phoenix.core.traces import Traces
 
 
 @dataclass
@@ -16,3 +17,4 @@ class Context:
     model: Model
     export_path: Path
     corpus: Optional[Model] = None
+    traces: Optional[Traces] = None

phoenix/server/api/input_types/SpanSort.py

@@ -0,0 +1,60 @@
+from enum import Enum
+from functools import partial
+from typing import Any, Iterable, Iterator
+
+import pandas as pd
+import strawberry
+
+from phoenix.core.traces import (
+    CUMULATIVE_LLM_TOKEN_COUNT_COMPLETION,
+    CUMULATIVE_LLM_TOKEN_COUNT_PROMPT,
+    CUMULATIVE_LLM_TOKEN_COUNT_TOTAL,
+    END_TIME,
+    LATENCY_MS,
+    LLM_TOKEN_COUNT_COMPLETION,
+    LLM_TOKEN_COUNT_PROMPT,
+    LLM_TOKEN_COUNT_TOTAL,
+    START_TIME,
+)
+from phoenix.server.api.types.SortDir import SortDir
+from phoenix.trace.schemas import Span
+
+
+@strawberry.enum
+class SpanColumn(Enum):
+    startTime = START_TIME
+    endTime = END_TIME
+    latencyMs = LATENCY_MS
+    tokenCountTotal = LLM_TOKEN_COUNT_TOTAL
+    tokenCountPrompt = LLM_TOKEN_COUNT_PROMPT
+    tokenCountCompletion = LLM_TOKEN_COUNT_COMPLETION
+    cumulativeTokenCountTotal = CUMULATIVE_LLM_TOKEN_COUNT_TOTAL
+    cumulativeTokenCountPrompt = CUMULATIVE_LLM_TOKEN_COUNT_PROMPT
+    cumulativeTokenCountCompletion = CUMULATIVE_LLM_TOKEN_COUNT_COMPLETION
+
+
+@strawberry.input
+class SpanSort:
+    """
+    The sort column and direction for span connections
+    """
+
+    col: SpanColumn
+    dir: SortDir
+
+    def __call__(self, spans: Iterable[Span]) -> Iterator[Span]:
+        """
+        Sorts the spans by the given column and direction
+        """
+        yield from pd.Series(spans, dtype=object).sort_values(
+            key=lambda series: series.apply(partial(_get_column, span_column=self.col)),
+            ascending=self.dir.value == SortDir.asc.value,
+        )
+
+
+def _get_column(span: Span, span_column: SpanColumn) -> Any:
+    if span_column is SpanColumn.startTime:
+        return span.start_time
+    if span_column is SpanColumn.endTime:
+        return span.end_time
+    return span.attributes.get(span_column.value)

phoenix/server/api/schema.py

@@ -1,6 +1,8 @@
 from collections import defaultdict
+from datetime import datetime
 from itertools import chain
-from typing import Dict, List, Optional, Set, Union
+from typing import Dict, List, Optional, Set, Tuple, Union, cast
+from uuid import UUID
 
 import numpy as np
 import numpy.typing as npt
@@ -12,10 +14,17 @@ from typing_extensions import Annotated
 from phoenix.pointcloud.clustering import Hdbscan
 from phoenix.server.api.helpers import ensure_list
 from phoenix.server.api.input_types.ClusterInput import ClusterInput
+from phoenix.server.api.input_types.Coordinates import (
+    InputCoordinate2D,
+    InputCoordinate3D,
+)
+from phoenix.server.api.input_types.SpanSort import SpanSort
 from phoenix.server.api.types.Cluster import Cluster, to_gql_clusters
 
+from ...trace.filter import SpanFilter
 from .context import Context
-from .input_types import Coordinates
+from .input_types.TimeRange import TimeRange
+from .types.DatasetInfo import DatasetInfo
 from .types.DatasetRole import AncillaryDatasetRole, DatasetRole
 from .types.Dimension import to_gql_dimension
 from .types.EmbeddingDimension import (
@@ -26,12 +35,24 @@ from .types.EmbeddingDimension import (
 )
 from .types.Event import create_event_id, unpack_event_id
 from .types.ExportEventsMutation import ExportEventsMutation
+from .types.Functionality import Functionality
 from .types.Model import Model
 from .types.node import GlobalID, Node, from_global_id
+from .types.pagination import Connection, ConnectionArgs, Cursor, connection_from_list
+from .types.Span import Span, to_gql_span
 
 
 @strawberry.type
 class Query:
+    @strawberry.field
+    def functionality(self, info: Info[Context, None]) -> "Functionality":
+        has_model_inferences = not info.context.model.is_empty
+        has_traces = info.context.traces is not None
+        return Functionality(
+            model_inferences=has_model_inferences,
+            tracing=has_traces,
+        )
+
     @strawberry.field
     def model(self) -> Model:
         return Model()
@@ -71,13 +92,13 @@ class Query:
             ),
         ],
         coordinates_2d: Annotated[
-            Optional[List[Coordinates.InputCoordinate2D]],
+            Optional[List[InputCoordinate2D]],
             strawberry.argument(
                 description="Point coordinates. Must be either 2D or 3D.",
             ),
         ] = UNSET,
         coordinates_3d: Annotated[
-            Optional[List[Coordinates.InputCoordinate3D]],
+            Optional[List[InputCoordinate3D]],
             strawberry.argument(
                 description="Point coordinates. Must be either 2D or 3D.",
             ),
@@ -178,6 +199,66 @@ class Query:
             clustered_events=clustered_events,
         )
 
+    @strawberry.field
+    def spans(
+        self,
+        info: Info[Context, None],
+        time_range: Optional[TimeRange] = UNSET,
+        trace_ids: Optional[List[ID]] = UNSET,
+        first: Optional[int] = 50,
+        last: Optional[int] = UNSET,
+        after: Optional[Cursor] = UNSET,
+        before: Optional[Cursor] = UNSET,
+        sort: Optional[SpanSort] = UNSET,
+        root_spans_only: Optional[bool] = False,
+        filter_condition: Optional[str] = None,
+    ) -> Connection[Span]:
+        args = ConnectionArgs(
+            first=first,
+            after=after if isinstance(after, Cursor) else None,
+            last=last,
+            before=before if isinstance(before, Cursor) else None,
+        )
+        if (traces := info.context.traces) is None:
+            return connection_from_list(data=[], args=args)
+        try:
+            predicate = SpanFilter(filter_condition) if filter_condition else None
+        except SyntaxError as e:
+            raise Exception(f"invalid filter condition: {e.msg}") from e  # TODO: add details
+        if not trace_ids:
+            spans = traces.get_spans(
+                start_time=time_range.start if time_range else None,
+                stop_time=time_range.end if time_range else None,
+                root_spans_only=root_spans_only,
+            )
+        else:
+            spans = chain.from_iterable(map(traces.get_trace, map(UUID, trace_ids)))
+        if predicate:
+            spans = filter(predicate, spans)
+        if sort:
+            spans = sort(spans)
+        data = list(map(to_gql_span, spans))
+        return connection_from_list(data=data, args=args)
+
+    @strawberry.field
+    def trace_dataset_info(
+        self,
+        info: Info[Context, None],
+    ) -> Optional[DatasetInfo]:
+        if (traces := info.context.traces) is None:
+            return None
+        if not (span_count := traces.span_count):
+            return None
+        start_time, stop_time = cast(
+            Tuple[datetime, datetime],
+            traces.right_open_time_range,
+        )
+        return DatasetInfo(
+            start_time=start_time,
+            end_time=stop_time,
+            record_count=span_count,
+        )
+
 
 @strawberry.type
 class Mutation(ExportEventsMutation):

phoenix/server/api/types/DataQualityMetric.py

@@ -3,7 +3,16 @@ from functools import partial
 
 import strawberry
 
-from phoenix.metrics.metrics import Cardinality, Count, Max, Mean, Min, PercentEmpty, Quantile, Sum
+from phoenix.metrics.metrics import (
+    Cardinality,
+    Count,
+    Max,
+    Mean,
+    Min,
+    PercentEmpty,
+    Quantile,
+    Sum,
+)
 
 
 @strawberry.enum

phoenix/server/api/types/Dataset.py

@@ -1,4 +1,3 @@
-from datetime import datetime
 from typing import Iterable, List, Optional, Set, Union
 
 import strawberry
@@ -9,15 +8,14 @@ import phoenix.core.model_schema as ms
 from phoenix.core.model_schema import FEATURE, TAG, ScalarDimension
 
 from ..input_types.DimensionInput import DimensionInput
+from .DatasetInfo import DatasetInfo
 from .DatasetRole import AncillaryDatasetRole, DatasetRole
 from .Dimension import Dimension, to_gql_dimension
 from .Event import Event, create_event, create_event_id, parse_event_ids_by_dataset_role
 
 
 @strawberry.type
-class Dataset:
-    start_time: datetime = strawberry.field(description="The start bookend of the data")
-    end_time: datetime = strawberry.field(description="The end bookend of the data")
+class Dataset(DatasetInfo):
     dataset: strawberry.Private[ms.Dataset]
     dataset_role: strawberry.Private[Union[DatasetRole, AncillaryDatasetRole]]
     model: strawberry.Private[ms.Model]

phoenix/server/api/types/DatasetInfo.py

@@ -0,0 +1,10 @@
+from datetime import datetime
+
+import strawberry
+
+
+@strawberry.type
+class DatasetInfo:
+    start_time: datetime = strawberry.field(description="The start bookend of the data")
+    end_time: datetime = strawberry.field(description="The end bookend of the data")
+    record_count: int = strawberry.field(description="The record count of the data")

phoenix/server/api/types/ExportEventsMutation.py

@@ -11,7 +11,10 @@ import phoenix.core.model_schema as ms
 from phoenix.server.api.context import Context
 from phoenix.server.api.input_types.ClusterInput import ClusterInput
 from phoenix.server.api.types.DatasetRole import AncillaryDatasetRole, DatasetRole
-from phoenix.server.api.types.Event import parse_event_ids_by_dataset_role, unpack_event_id
+from phoenix.server.api.types.Event import (
+    parse_event_ids_by_dataset_role,
+    unpack_event_id,
+)
 from phoenix.server.api.types.ExportedFile import ExportedFile
 
 

phoenix/server/api/types/Functionality.py

@@ -0,0 +1,15 @@
+import strawberry
+
+
+@strawberry.type
+class Functionality:
+    """
+    Describes the the functionality of the platform that is enabled
+    """
+
+    model_inferences: bool = strawberry.field(
+        description="Model inferences are available for analysis"
+    )
+    tracing: bool = strawberry.field(
+        description="Generative tracing records are available for analysis"
+    )

phoenix/server/api/types/MimeType.py

@@ -0,0 +1,16 @@
+from enum import Enum
+from typing import Any, Optional
+
+import strawberry
+
+from phoenix.trace import semantic_conventions
+
+
+@strawberry.enum
+class MimeType(Enum):
+    text = semantic_conventions.MimeType.TEXT
+    json = semantic_conventions.MimeType.JSON
+
+    @classmethod
+    def _missing_(cls, v: Any) -> Optional["MimeType"]:
+        return None if v else cls.text

phoenix/server/api/types/Model.py

@@ -40,11 +40,6 @@ class Model:
         include: Optional[DimensionFilter] = UNSET,
         exclude: Optional[DimensionFilter] = UNSET,
     ) -> Connection[Dimension]:
-        """
-        A non-trivial implementation should efficiently fetch only
-        the necessary books after the offset.
-        For simplicity, here we build the list and then slice it accordingly
-        """
         model = info.context.model
         return connection_from_list(
             [
@@ -68,6 +63,7 @@ class Model:
         return Dataset(
             start_time=start,
             end_time=stop,
+            record_count=len(dataset),
             dataset=dataset,
             dataset_role=DatasetRole.primary,
             model=info.context.model,
@@ -81,6 +77,7 @@ class Model:
         return Dataset(
             start_time=start,
             end_time=stop,
+            record_count=len(dataset),
             dataset=dataset,
             dataset_role=DatasetRole.reference,
             model=info.context.model,
@@ -96,6 +93,7 @@ class Model:
         return Dataset(
             start_time=start,
             end_time=stop,
+            record_count=len(dataset),
             dataset=dataset,
             dataset_role=AncillaryDatasetRole.corpus,
             model=info.context.corpus,

phoenix/server/api/types/SortDir.py

@@ -0,0 +1,13 @@
+from enum import Enum
+
+import strawberry
+
+
+@strawberry.enum
+class SortDir(Enum):
+    """
+    Sort directions
+    """
+
+    asc = "asc"
+    desc = "desc"