arize-phoenix 2.3.0__py3-none-any.whl → 2.4.1__py3-none-any.whl

phoenix/session/evaluation.py

@@ -103,13 +103,15 @@ def _extract_subject_id_from_index(
 
 def _extract_result(row: "pd.Series[Any]") -> Optional[pb.Evaluation.Result]:
     score = cast(Optional[float], row.get("score"))
+    if isinstance(score, float) and math.isnan(score):
+        score = None
     label = cast(Optional[str], row.get("label"))
+    if isinstance(label, float) and math.isnan(label):
+        label = None
     explanation = cast(Optional[str], row.get("explanation"))
-    if (
-        (score is None or isinstance(score, float) and math.isnan(score))
-        and not label
-        and not explanation
-    ):
+    if isinstance(explanation, float) and math.isnan(explanation):
+        explanation = None
+    if score is None and not label and not explanation:
         return None
     return pb.Evaluation.Result(
         score=DoubleValue(value=score) if score is not None else None,

phoenix/trace/__init__.py

@@ -1,4 +1,10 @@
-from .span_evaluations import DocumentEvaluations, SpanEvaluations, TraceEvaluations
+from .span_evaluations import DocumentEvaluations, Evaluations, SpanEvaluations, TraceEvaluations
 from .trace_dataset import TraceDataset
 
-__all__ = ["TraceDataset", "SpanEvaluations", "DocumentEvaluations", "TraceEvaluations"]
+__all__ = [
+    "Evaluations",
+    "TraceDataset",
+    "SpanEvaluations",
+    "DocumentEvaluations",
+    "TraceEvaluations",
+]

phoenix/trace/llama_index/callback.py

@@ -214,17 +214,26 @@ def payload_to_semantic_attributes(
         if event_type is CBEventType.LLM:
             if model_name := serialized.get("model"):
                 attributes[LLM_MODEL_NAME] = model_name
-                attributes[LLM_INVOCATION_PARAMETERS] = json.dumps(
-                    {
-                        "model": model_name,
-                        "temperature": serialized["temperature"],
-                        "max_tokens": serialized["max_tokens"],
-                        **serialized["additional_kwargs"],
-                    }
-                )
+                invocation_parameters = _extract_invocation_parameters(serialized)
+                invocation_parameters["model"] = model_name
+                attributes[LLM_INVOCATION_PARAMETERS] = json.dumps(invocation_parameters)
     return attributes
 
 
+def _extract_invocation_parameters(serialized: Mapping[str, Any]) -> Dict[str, Any]:
+    # FIXME: this is only based on openai. Other models have different parameters.
+    if not hasattr(serialized, "get"):
+        return {}
+    invocation_parameters: Dict[str, Any] = {}
+    additional_kwargs = serialized.get("additional_kwargs")
+    if additional_kwargs and isinstance(additional_kwargs, Mapping):
+        invocation_parameters.update(additional_kwargs)
+    for key in ("temperature", "max_tokens"):
+        if (value := serialized.get(key)) is not None:
+            invocation_parameters[key] = value
+    return invocation_parameters
+
+
 class OpenInferenceTraceCallbackHandler(BaseCallbackHandler):
     """Callback handler for storing LLM application trace data in OpenInference format.
     OpenInference is an open standard for capturing and storing AI model

phoenix/trace/span_evaluations.py

@@ -1,15 +1,26 @@
+import json
 from abc import ABC
 from dataclasses import dataclass, field
 from itertools import product
+from pathlib import Path
 from types import MappingProxyType
-from typing import Any, Callable, List, Mapping, Optional, Sequence, Set, Tuple
+from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Set, Tuple, Type, Union
+from uuid import UUID, uuid4
 
 import pandas as pd
 from pandas.api.types import is_integer_dtype, is_numeric_dtype, is_string_dtype
+from pyarrow import Table, parquet
+
+from phoenix.config import TRACE_DATASET_DIR
+from phoenix.exceptions import PhoenixException
 
 EVAL_NAME_COLUMN_PREFIX = "eval."
 
 
+class InvalidParquetMetadataError(PhoenixException):
+    pass
+
+
 class NeedsNamedIndex(ABC):
     index_names: Mapping[Tuple[str, ...], Callable[[Any], bool]]
     all_valid_index_name_sorted_combos: Set[Tuple[str, ...]]
@@ -72,6 +83,7 @@ class NeedsResultColumns(ABC):
 class Evaluations(NeedsNamedIndex, NeedsResultColumns, ABC):
     eval_name: str  # The name for the evaluation, e.g. 'toxicity'
     dataframe: pd.DataFrame = field(repr=False)
+    id: UUID = field(init=False, default_factory=uuid4)
 
     def __len__(self) -> int:
         return len(self.dataframe)
@@ -152,6 +164,58 @@ class Evaluations(NeedsNamedIndex, NeedsResultColumns, ABC):
             tuple(sorted(prod)) for prod in product(*cls.index_names.keys())
         )
 
+    def to_parquet(self, directory: Optional[Union[str, Path]] = None) -> Path:
+        """Persists the evaluations to a parquet file.
+
+        Args:
+            directory (Optional[Union[str, Path]], optional): An optional path
+            to a directory where the parquet file will be saved. If not
+            provided, the parquet file will be saved to a default location.
+
+        Returns:
+            Path: The path to the parquet file, including a randomly generated
+            filename.
+        """
+        directory = Path(directory) if directory else TRACE_DATASET_DIR
+        path = directory / f"evaluations-{self.id}.parquet"
+        table = Table.from_pandas(self.dataframe)
+        table = table.replace_schema_metadata(
+            {
+                **(table.schema.metadata or {}),
+                # explicitly encode keys and values, which are automatically encoded regardless
+                b"arize": json.dumps(
+                    {
+                        "eval_id": str(self.id),
+                        "eval_name": self.eval_name,
+                        "eval_type": self.__class__.__name__,
+                    }
+                ).encode("utf-8"),
+            }
+        )
+        parquet.write_table(table, path)
+        return path
+
+    @classmethod
+    def from_parquet(cls, path: Union[str, Path]) -> "Evaluations":
+        """Loads the evaluations from a parquet file.
+
+        Args:
+            path (Union[str, Path]): Path to a persisted evaluations parquet
+            file.
+
+        Returns:
+            Evaluations: The loaded evaluations. The type of the returned
+            evaluations will be the same as the type of the evaluations that
+            were originally persisted.
+        """
+        schema = parquet.read_schema(path)
+        eval_id, eval_name, evaluations_cls = _parse_schema_metadata(schema.metadata)
+        table = parquet.read_table(path)
+        dataframe = table.to_pandas()
+        evaluations = evaluations_cls(eval_name=eval_name, dataframe=dataframe)
+        object.__setattr__(evaluations, "id", eval_id)
+        return evaluations
+
 
 @dataclass(frozen=True)
 class SpanEvaluations(
@@ -235,3 +299,50 @@ class TraceEvaluations(
     index_names=MappingProxyType({("context.trace_id", "trace_id"): is_string_dtype}),
 ):
     ...
+
+
+def _parse_schema_metadata(metadata: Dict[bytes, Any]) -> Tuple[UUID, str, Type[Evaluations]]:
+    """Validates and parses the schema metadata. Raises an exception if the
+    metadata is invalid.
+
+    Args:
+        metadata (Dict[bytes, Any]): A dictionary of schema metadata from a
+        parquet file.
+
+    Returns:
+        Tuple[str, ModuleType]: The evaluation name and the evaluations class.
+    """
+    if not (arize_metadata_json := metadata.get(b"arize")):
+        raise InvalidParquetMetadataError('Schema metadata is missing "arize" key')
+    try:
+        arize_metadata = json.loads(arize_metadata_json)
+    except json.JSONDecodeError as err:
+        raise InvalidParquetMetadataError(
+            'Encountered invalid JSON string under "arize" key'
+        ) from err
+    evaluations_classes = {subclass.__name__: subclass for subclass in Evaluations.__subclasses__()}
+    if not (
+        isinstance(arize_metadata, dict)
+        and (eval_id := _to_uuid(arize_metadata.get("eval_id")))
+        and isinstance(eval_name := arize_metadata.get("eval_name"), str)
+        and (eval_type := arize_metadata.get("eval_type"))
+        and (evaluations_cls := evaluations_classes.get(eval_type))
+    ):
+        raise InvalidParquetMetadataError(f"Invalid Arize metadata: {arize_metadata}")
+    return eval_id, eval_name, evaluations_cls
+
+
+def _to_uuid(value: Any) -> Optional[UUID]:
+    """
+    Converts an input to a UUID if possible, otherwise returns None.
+
+    Args:
+        value (Any): The value to convert to a UUID.
+
+    Returns:
+        Optional[UUID]: A UUID if the value could be converted, otherwise None.
+    """
+    try:
+        return UUID(value)
+    except Exception:
+        return None

phoenix/version.py

@@ -1 +1 @@
-__version__ = "2.3.0"
+__version__ = "2.4.1"