arize-phoenix 3.16.0__py3-none-any.whl → 7.7.0__py3-none-any.whl

phoenix/{datasets → inferences}/schema.py

@@ -1,13 +1,14 @@
 import json
+from collections.abc import Mapping
 from dataclasses import asdict, dataclass, replace
-from typing import Any, Dict, List, Mapping, Optional, Tuple, Union
+from typing import Any, Optional, Union
 
-EmbeddingFeatures = Dict[str, "EmbeddingColumnNames"]
+EmbeddingFeatures = dict[str, "EmbeddingColumnNames"]
 SchemaFieldName = str
-SchemaFieldValue = Union[Optional[str], Optional[List[str]], Optional[EmbeddingFeatures]]
+SchemaFieldValue = Union[Optional[str], Optional[list[str]], Optional[EmbeddingFeatures]]
 
-MULTI_COLUMN_SCHEMA_FIELD_NAMES: Tuple[str, ...] = ("feature_column_names", "tag_column_names")
-SINGLE_COLUMN_SCHEMA_FIELD_NAMES: Tuple[str, ...] = (
+MULTI_COLUMN_SCHEMA_FIELD_NAMES: tuple[str, ...] = ("feature_column_names", "tag_column_names")
+SINGLE_COLUMN_SCHEMA_FIELD_NAMES: tuple[str, ...] = (
     "prediction_id_column_name",
     "timestamp_column_name",
     "prediction_label_column_name",
@@ -19,7 +20,7 @@ LLM_SCHEMA_FIELD_NAMES = ["prompt_column_names", "response_column_names"]
 
 
 @dataclass(frozen=True)
-class EmbeddingColumnNames(Dict[str, Any]):
+class EmbeddingColumnNames(dict[str, Any]):
     """
     A dataclass to hold the column names for the embedding features.
     An embedding feature is a feature that is represented by a vector.
@@ -34,7 +35,6 @@ class EmbeddingColumnNames(Dict[str, Any]):
 @dataclass(frozen=True)
 class RetrievalEmbeddingColumnNames(EmbeddingColumnNames):
     """
-    *** Experimental ***
     A relationship is a column that maps a prediction to another record.
 
     Example
@@ -81,8 +81,8 @@ class Schema:
     prediction_id_column_name: Optional[str] = None
     id_column_name: Optional[str] = None  # Syntax sugar for prediction_id_column_name
     timestamp_column_name: Optional[str] = None
-    feature_column_names: Optional[List[str]] = None
-    tag_column_names: Optional[List[str]] = None
+    feature_column_names: Optional[list[str]] = None
+    tag_column_names: Optional[list[str]] = None
     prediction_label_column_name: Optional[str] = None
     prediction_score_column_name: Optional[str] = None
     actual_label_column_name: Optional[str] = None
@@ -92,7 +92,7 @@ class Schema:
     # document_column_names is used explicitly when the schema is used to capture a corpus
     document_column_names: Optional[EmbeddingColumnNames] = None
     embedding_feature_column_names: Optional[EmbeddingFeatures] = None
-    excluded_column_names: Optional[List[str]] = None
+    excluded_column_names: Optional[list[str]] = None
 
     def __post_init__(self) -> None:
         # re-map document_column_names to be in the prompt_column_names position
@@ -108,7 +108,7 @@ class Schema:
     def replace(self, **changes: Any) -> "Schema":
         return replace(self, **changes)
 
-    def asdict(self) -> Dict[str, str]:
+    def asdict(self) -> dict[str, str]:
         return asdict(self)
 
     def to_json(self) -> str:

phoenix/{datasets → inferences}/validation.py

@@ -1,5 +1,4 @@
 import math
-from typing import List
 
 import numpy as np
 from pandas import DataFrame, Series
@@ -11,8 +10,8 @@ from .schema import EmbeddingColumnNames, Schema
 RESERVED_EMBEDDING_NAMES = ("prompt", "response")
 
 
-def _check_valid_schema(schema: Schema) -> List[err.ValidationError]:
-    errs: List[str] = []
+def _check_valid_schema(schema: Schema) -> list[err.ValidationError]:
+    errs: list[str] = []
     if schema.excluded_column_names is None:
         return []
 
@@ -34,7 +33,7 @@ def _check_valid_schema(schema: Schema) -> List[err.ValidationError]:
     return []
 
 
-def validate_dataset_inputs(dataframe: DataFrame, schema: Schema) -> List[err.ValidationError]:
+def validate_inferences_inputs(dataframe: DataFrame, schema: Schema) -> list[err.ValidationError]:
     errors = _check_missing_columns(dataframe, schema)
     if errors:
         return errors
@@ -53,12 +52,12 @@ def validate_dataset_inputs(dataframe: DataFrame, schema: Schema) -> List[err.Va
     return []
 
 
-def _check_valid_embedding_data(dataframe: DataFrame, schema: Schema) -> List[err.ValidationError]:
+def _check_valid_embedding_data(dataframe: DataFrame, schema: Schema) -> list[err.ValidationError]:
     embedding_col_names = schema.embedding_feature_column_names
     if embedding_col_names is None:
         return []
 
-    embedding_errors: List[err.ValidationError] = []
+    embedding_errors: list[err.ValidationError] = []
     for embedding_name, column_names in embedding_col_names.items():
         if embedding_name in RESERVED_EMBEDDING_NAMES:
             embedding_errors += _validate_reserved_embedding_name(embedding_name, schema)
@@ -71,8 +70,8 @@ def _check_valid_embedding_data(dataframe: DataFrame, schema: Schema) -> List[er
 
 def _check_valid_prompt_response_data(
     dataframe: DataFrame, schema: Schema
-) -> List[err.ValidationError]:
-    prompt_response_errors: List[err.ValidationError] = []
+) -> list[err.ValidationError]:
+    prompt_response_errors: list[err.ValidationError] = []
 
     prompt_response_column_names = {
         "prompt": schema.prompt_column_names,
@@ -89,7 +88,7 @@ def _check_valid_prompt_response_data(
 
 def _validate_reserved_embedding_name(
     embedding_name: str, schema: Schema
-) -> List[err.ValidationError]:
+) -> list[err.ValidationError]:
     if embedding_name == "prompt" and schema.prompt_column_names is not None:
         return [err.InvalidEmbeddingReservedName(embedding_name, "schema.prompt_column_names")]
     elif embedding_name == "response" and schema.response_column_names is not None:
@@ -99,9 +98,9 @@ def _validate_reserved_embedding_name(
 
 def _validate_embedding_vector(
     dataframe: DataFrame, name: str, vector_column_name: str
-) -> List[err.ValidationError]:
+) -> list[err.ValidationError]:
     vector_column = dataframe[vector_column_name]
-    errors: List[err.ValidationError] = []
+    errors: list[err.ValidationError] = []
     vector_length = None
 
     for vector in vector_column:
@@ -156,8 +155,8 @@ def _validate_embedding_vector(
     return errors
 
 
-def _check_column_types(dataframe: DataFrame, schema: Schema) -> List[err.ValidationError]:
-    wrong_type_cols: List[str] = []
+def _check_column_types(dataframe: DataFrame, schema: Schema) -> list[err.ValidationError]:
+    wrong_type_cols: list[str] = []
     if schema.prediction_id_column_name is not None:
         if not (
             is_numeric_dtype(dataframe.dtypes[schema.prediction_id_column_name])
@@ -172,7 +171,7 @@ def _check_column_types(dataframe: DataFrame, schema: Schema) -> List[err.Valida
     return []
 
 
-def _check_missing_columns(dataframe: DataFrame, schema: Schema) -> List[err.ValidationError]:
+def _check_missing_columns(dataframe: DataFrame, schema: Schema) -> list[err.ValidationError]:
     # converting to a set first makes the checks run a lot faster
     existing_columns = set(dataframe.columns)
     missing_columns = []

phoenix/logging/__init__.py

@@ -0,0 +1,3 @@
+from ._config import setup_logging
+
+__all__ = ["setup_logging"]

phoenix/logging/_config.py

@@ -0,0 +1,90 @@
+import atexit
+import logging
+import logging.config
+import logging.handlers
+import queue
+from sys import stderr, stdout
+
+from typing_extensions import assert_never
+
+from phoenix.config import LoggingMode
+from phoenix.logging._filter import NonErrorFilter
+from phoenix.settings import Settings
+
+from ._formatter import PhoenixJSONFormatter
+
+
+def setup_logging() -> None:
+    """
+    Configures logging for the specified logging mode.
+    """
+    logging_mode = Settings.logging_mode
+    if logging_mode is LoggingMode.DEFAULT:
+        _setup_library_logging()
+    elif logging_mode is LoggingMode.STRUCTURED:
+        _setup_application_logging()
+    else:
+        assert_never(logging_mode)
+
+
+def _setup_library_logging() -> None:
+    """
+    Configures logging if Phoenix is used as a library
+    """
+    logger = logging.getLogger("phoenix")
+    logger.setLevel(Settings.logging_level)
+    db_logger = logging.getLogger("sqlalchemy")
+    db_logger.setLevel(Settings.db_logging_level)
+    logger.info("Default logging ready")
+
+
+def _setup_application_logging() -> None:
+    """
+    Configures logging if Phoenix is used as an application
+    """
+    sql_engine_logger = logging.getLogger("sqlalchemy.engine.Engine")
+    # Remove all existing handlers
+    for handler in sql_engine_logger.handlers[:]:
+        sql_engine_logger.removeHandler(handler)
+        handler.close()
+
+    phoenix_logger = logging.getLogger("phoenix")
+    phoenix_logger.setLevel(Settings.logging_level)
+    phoenix_logger.propagate = False  # Do not pass records to the root logger
+    sql_logger = logging.getLogger("sqlalchemy")
+    sql_logger.setLevel(Settings.db_logging_level)
+    sql_logger.propagate = False  # Do not pass records to the root logger
+
+    log_queue = queue.Queue()  # type:ignore
+    queue_handler = logging.handlers.QueueHandler(log_queue)
+    phoenix_logger.addHandler(queue_handler)
+    sql_logger.addHandler(queue_handler)
+
+    fmt_keys = {
+        "level": "levelname",
+        "message": "message",
+        "timestamp": "timestamp",
+        "logger": "name",
+        "module": "module",
+        "function": "funcName",
+        "line": "lineno",
+        "thread_name": "threadName",
+    }
+    formatter = PhoenixJSONFormatter(fmt_keys=fmt_keys)
+
+    # stdout handler
+    stdout_handler = logging.StreamHandler(stdout)
+    stdout_handler.setFormatter(formatter)
+    stdout_handler.setLevel(Settings.logging_level)
+    stdout_handler.addFilter(NonErrorFilter())
+
+    # stderr handler
+    stderr_handler = logging.StreamHandler(stderr)
+    stderr_handler.setFormatter(formatter)
+    stderr_handler.setLevel(logging.WARNING)
+
+    queue_listener = logging.handlers.QueueListener(log_queue, stdout_handler, stderr_handler)
+    if queue_listener is not None:
+        queue_listener.start()
+        atexit.register(queue_listener.stop)
+    phoenix_logger.info("Structured logging ready")

phoenix/logging/_filter.py

@@ -0,0 +1,6 @@
+import logging
+
+
+class NonErrorFilter(logging.Filter):
+    def filter(self, record: logging.LogRecord) -> bool:
+        return record.levelno <= logging.INFO

phoenix/logging/_formatter.py

@@ -0,0 +1,69 @@
+import datetime as dt
+import json
+import logging
+from typing import Optional
+
+LOG_RECORD_BUILTIN_ATTRS = {
+    "args",
+    "asctime",
+    "created",
+    "exc_info",
+    "exc_text",
+    "filename",
+    "funcName",
+    "levelname",
+    "levelno",
+    "lineno",
+    "module",
+    "msecs",
+    "message",
+    "msg",
+    "name",
+    "pathname",
+    "process",
+    "processName",
+    "relativeCreated",
+    "stack_info",
+    "thread",
+    "threadName",
+    "taskName",
+}
+
+
+class PhoenixJSONFormatter(logging.Formatter):
+    def __init__(
+        self,
+        *,
+        fmt_keys: Optional[dict[str, str]] = None,
+    ):
+        super().__init__()
+        self.fmt_keys = fmt_keys if fmt_keys is not None else {}
+
+    def format(self, record: logging.LogRecord) -> str:
+        message = self._prepare_log_dict(record)
+        return json.dumps(message, default=str)
+
+    def _prepare_log_dict(self, record: logging.LogRecord) -> dict[str, str]:
+        always_fields = {
+            "message": record.getMessage(),
+            "timestamp": dt.datetime.fromtimestamp(record.created, tz=dt.timezone.utc).isoformat(),
+        }
+        if record.exc_info is not None:
+            always_fields["exc_info"] = self.formatException(record.exc_info)
+
+        if record.stack_info is not None:
+            always_fields["stack_info"] = self.formatStack(record.stack_info)
+
+        message = {
+            key: msg_val
+            if (msg_val := always_fields.pop(val, None)) is not None
+            else getattr(record, val)
+            for key, val in self.fmt_keys.items()
+        }
+        message.update(always_fields)
+
+        for key, val in record.__dict__.items():
+            if key not in LOG_RECORD_BUILTIN_ATTRS:
+                message[key] = val
+
+        return message

phoenix/metrics/__init__.py

@@ -1,8 +1,9 @@
 import logging
 import warnings
 from abc import ABC, abstractmethod
+from collections.abc import Iterable, Mapping
 from dataclasses import dataclass
-from typing import Any, Iterable, List, Mapping, Optional, Union
+from typing import Any, Optional, Union
 
 import numpy as np
 import pandas as pd
@@ -36,13 +37,13 @@ class Metric(ABC):
     def calc(self, dataframe: pd.DataFrame) -> Any: ...
 
     @abstractmethod
-    def operands(self) -> List[Column]: ...
+    def operands(self) -> list[Column]: ...
 
     def __call__(
         self,
         df: pd.DataFrame,
         /,
-        subset_rows: Optional[Union[slice, List[int]]] = None,
+        subset_rows: Optional[Union[slice, list[int]]] = None,
     ) -> Any:
         """
         Computes the metric on a dataframe.
@@ -51,7 +52,7 @@ class Metric(ABC):
         ----------
         df: pandas DataFrame
             The dataframe input to the metric.
-        subset_rows: Optional[Union[slice, List[int]]] = None
+        subset_rows: Optional[Union[slice, list[int]]] = None
             Optionally specifying a subset of rows for the computation.
             Can be a list or slice (e.g. `slice(100, 200)`) of integers.
         """

phoenix/metrics/binning.py

@@ -1,8 +1,9 @@
 import warnings
 from abc import ABC, abstractmethod
+from collections.abc import Iterable, Sequence
 from dataclasses import dataclass
 from functools import partial
-from typing import Any, Iterable, Optional, Sequence, cast
+from typing import Any, Optional, cast
 
 import numpy as np
 import pandas as pd
@@ -78,7 +79,7 @@ class IntervalBinning(BinningMethod):
             else pd.IntervalIndex(
                 (
                     pd.Interval(
-                        np.NINF,
+                        -np.inf,
                         np.inf,
                         closed="neither",
                     ),
@@ -208,7 +209,7 @@ class QuantileBinning(IntervalBinning):
         # Extend min and max to infinities, unless len(breaks) < 3,
         # in which case the min is kept and two bins are created.
         breaks = breaks[1:-1] if len(breaks) > 2 else breaks[:1]
-        breaks = [np.NINF] + breaks + [np.inf]
+        breaks = [-np.inf] + breaks + [np.inf]
         return pd.IntervalIndex.from_breaks(
             breaks,
             closed="left",

phoenix/metrics/metrics.py

@@ -1,8 +1,9 @@
 import math
 import warnings
+from collections.abc import Callable
 from dataclasses import dataclass, field
 from functools import cached_property
-from typing import Callable, Union, cast
+from typing import Union, cast
 
 import numpy as np
 import numpy.typing as npt

phoenix/metrics/mixins.py

@@ -7,10 +7,11 @@ on cooperative multiple inheritance and method resolution order in Python.
 import collections
 import inspect
 from abc import ABC, abstractmethod
+from collections.abc import Callable
 from dataclasses import dataclass, field, fields, replace
 from functools import cached_property
 from itertools import repeat
-from typing import TYPE_CHECKING, Any, Callable, Dict, Iterator, List, Mapping, Optional
+from typing import TYPE_CHECKING, Any, Iterator, Mapping, Optional
 
 import numpy as np
 import pandas as pd
@@ -42,7 +43,7 @@ class VectorOperator(ABC):
 
 @dataclass(frozen=True)
 class NullaryOperator(Metric, ABC):
-    def operands(self) -> List[Column]:
+    def operands(self) -> list[Column]:
         return []
 
 
@@ -55,7 +56,7 @@ class UnaryOperator(Metric, ABC):
 
     operand: Column = Column()
 
-    def operands(self) -> List[Column]:
+    def operands(self) -> list[Column]:
         return [self.operand]
 
 
@@ -98,10 +99,10 @@ class EvaluationMetricKeywordParameters(_BaseMapping):
         return sum(1 for _ in self)
 
     @property
-    def columns(self) -> List[Column]:
+    def columns(self) -> list[Column]:
         return [v for v in self.values() if isinstance(v, Column)]
 
-    def __call__(self, df: pd.DataFrame) -> Dict[str, Any]:
+    def __call__(self, df: pd.DataFrame) -> dict[str, Any]:
         return {k: v(df) if isinstance(v, Column) else v for k, v in self.items()}
 
 
@@ -142,7 +143,7 @@ class EvaluationMetric(Metric, ABC):
                 ),
             )
 
-    def operands(self) -> List[Column]:
+    def operands(self) -> list[Column]:
         return [self.actual, self.predicted] + self.parameters.columns
 
     def calc(self, df: pd.DataFrame) -> float:

phoenix/metrics/retrieval_metrics.py

@@ -1,5 +1,6 @@
+from collections.abc import Iterable
 from dataclasses import dataclass, field
-from typing import Iterable, Optional, cast
+from typing import Optional, cast
 
 import numpy as np
 import pandas as pd

phoenix/metrics/timeseries.py

@@ -1,7 +1,8 @@
+from collections.abc import Callable, Iterable, Iterator
 from datetime import datetime, timedelta, timezone
 from functools import partial
 from itertools import accumulate, repeat
-from typing import Callable, Iterable, Iterator, Tuple, cast
+from typing import cast
 
 import pandas as pd
 from typing_extensions import TypeAlias
@@ -41,12 +42,12 @@ def row_interval_from_sorted_time_index(
     time_index: pd.DatetimeIndex,
     time_start: datetime,
     time_stop: datetime,
-) -> Tuple[StartIndex, StopIndex]:
+) -> tuple[StartIndex, StopIndex]:
     """
     Returns end exclusive time slice from sorted index.
     """
     return cast(
-        Tuple[StartIndex, StopIndex],
+        tuple[StartIndex, StopIndex],
         time_index.searchsorted((time_start, time_stop)),
     )
 
@@ -86,7 +87,7 @@ def _groupers(
     end_time: datetime,
     evaluation_window: timedelta,
     sampling_interval: timedelta,
-) -> Iterator[Tuple[StartTime, EndTime, pd.Grouper]]:
+) -> Iterator[tuple[StartTime, EndTime, pd.Grouper]]:
     """
     Yields pandas.Groupers from time series parameters.
     """

phoenix/metrics/wrappers.py

@@ -18,7 +18,7 @@ from abc import ABC
 from enum import Enum
 from inspect import Signature
 from itertools import chain, islice
-from typing import Any, Dict, List, Tuple, cast
+from typing import Any, cast
 
 import numpy as np
 import pandas as pd
@@ -27,6 +27,8 @@ from sklearn import metrics as sk
 from sklearn.utils.multiclass import check_classification_targets
 from wrapt import PartialCallableObjectProxy
 
+from phoenix.config import SKLEARN_VERSION
+
 
 class Eval(PartialCallableObjectProxy, ABC):  # type: ignore
     def __call__(
@@ -157,7 +159,7 @@ def _coerce_dtype_if_necessary(
 def _eliminate_missing_values_from_all_series(
     *args: Any,
     **kwargs: Any,
-) -> Tuple[List[Any], Dict[str, Any]]:
+) -> tuple[list[Any], dict[str, Any]]:
     positional_arguments = list(args)
     keyword_arguments = dict(kwargs)
     all_series = [
@@ -232,5 +234,9 @@ class SkEval(Enum):
     r2_score = RegressionEval(sk.r2_score)
     recall_score = ClassificationEval(sk.recall_score)
     roc_auc_score = ScoredClassificationEval(sk.roc_auc_score)
-    root_mean_squared_error = RegressionEval(sk.mean_squared_error, squared=False)
+    root_mean_squared_error = (
+        RegressionEval(sk.mean_squared_error, squared=False)
+        if SKLEARN_VERSION < (1, 6)
+        else RegressionEval(sk.root_mean_squared_error)
+    )
     zero_one_loss = ClassificationEval(sk.zero_one_loss)

phoenix/pointcloud/clustering.py

@@ -1,13 +1,11 @@
 from dataclasses import asdict, dataclass
-from typing import List, Set
 
 import numpy as np
 import numpy.typing as npt
-from hdbscan import HDBSCAN
 from typing_extensions import TypeAlias
 
 RowIndex: TypeAlias = int
-RawCluster: TypeAlias = Set[RowIndex]
+RawCluster: TypeAlias = set[RowIndex]
 Matrix: TypeAlias = npt.NDArray[np.float64]
 
 
@@ -17,9 +15,11 @@ class Hdbscan:
     min_samples: float = 1
     cluster_selection_epsilon: float = 0.0
 
-    def find_clusters(self, mat: Matrix) -> List[RawCluster]:
+    def find_clusters(self, mat: Matrix) -> list[RawCluster]:
+        from fast_hdbscan import HDBSCAN
+
         cluster_ids: npt.NDArray[np.int_] = HDBSCAN(**asdict(self)).fit_predict(mat)
-        ans: List[RawCluster] = [set() for _ in range(np.max(cluster_ids) + 1)]
+        ans: list[RawCluster] = [set() for _ in range(np.max(cluster_ids) + 1)]
         for row_idx, cluster_id in enumerate(cluster_ids):
             if cluster_id > -1:
                 ans[cluster_id].add(row_idx)

phoenix/pointcloud/pointcloud.py

@@ -1,9 +1,9 @@
+from collections.abc import Hashable, Mapping
 from dataclasses import dataclass
-from typing import Dict, List, Mapping, Protocol, Set, Tuple
+from typing import Protocol, TypeVar
 
 import numpy as np
 import numpy.typing as npt
-from strawberry import ID
 from typing_extensions import TypeAlias
 
 from phoenix.pointcloud.clustering import RawCluster
@@ -12,13 +12,15 @@ Vector: TypeAlias = npt.NDArray[np.float64]
 Matrix: TypeAlias = npt.NDArray[np.float64]
 RowIndex: TypeAlias = int
 
+_IdType = TypeVar("_IdType", bound=Hashable)
+
 
 class DimensionalityReducer(Protocol):
     def project(self, mat: Matrix, n_components: int) -> Matrix: ...
 
 
 class ClustersFinder(Protocol):
-    def find_clusters(self, mat: Matrix) -> List[RawCluster]: ...
+    def find_clusters(self, mat: Matrix) -> list[RawCluster]: ...
 
 
 @dataclass(frozen=True)
@@ -28,9 +30,9 @@ class PointCloud:
 
     def generate(
         self,
-        data: Mapping[ID, Vector],
+        data: Mapping[_IdType, Vector],
         n_components: int = 3,
-    ) -> Tuple[Dict[ID, Vector], Dict[str, Set[ID]]]:
+    ) -> tuple[dict[_IdType, Vector], dict[str, set[_IdType]]]:
         """
         Given a set of vectors, projects them onto lower dimensions, and
         finds clusters among the projections.

phoenix/pointcloud/projectors.py

@@ -6,12 +6,6 @@ import numpy as np
 import numpy.typing as npt
 from typing_extensions import TypeAlias
 
-with warnings.catch_warnings():
-    from numba.core.errors import NumbaWarning
-
-    warnings.simplefilter("ignore", category=NumbaWarning)
-    from umap import UMAP
-
 Matrix: TypeAlias = npt.NDArray[np.float64]
 
 
@@ -25,6 +19,11 @@ class Umap:
     min_dist: float = 0.1
 
     def project(self, mat: Matrix, n_components: int) -> Matrix:
+        with warnings.catch_warnings():
+            from numba.core.errors import NumbaWarning
+
+            warnings.simplefilter("ignore", category=NumbaWarning)
+            from umap import UMAP
         config = asdict(self)
         config["n_components"] = n_components
         if len(mat) <= n_components: