arize-phoenix 4.4.4rc5__py3-none-any.whl → 4.4.4rc6__py3-none-any.whl

phoenix/{datasets → experiments}/evaluators/utils.py

@@ -1,20 +1,19 @@
 import functools
 import inspect
-from abc import ABC
-from types import MappingProxyType
-from typing import Any, Awaitable, Callable, Mapping, Optional, Union
+from itertools import chain, islice, repeat
+from typing import TYPE_CHECKING, Any, Callable, Optional, Union
 
-from typing_extensions import TypeAlias
-
-from phoenix.datasets.types import (
+from phoenix.experiments.types import (
     AnnotatorKind,
     EvaluationResult,
     JSONSerializable,
-    TaskOutput,
 )
 
+if TYPE_CHECKING:
+    from phoenix.experiments.evaluators.base import Evaluator
+
 
-def _unwrap_json(obj: JSONSerializable) -> JSONSerializable:
+def unwrap_json(obj: JSONSerializable) -> JSONSerializable:
     if isinstance(obj, dict):
         if len(obj) == 1:
             key = next(iter(obj.keys()))
@@ -80,7 +79,7 @@ def create_evaluator(
     if isinstance(kind, str):
         kind = AnnotatorKind(kind.upper())
 
-    def wrapper(func: Callable[..., Any]) -> Evaluator:
+    def wrapper(func: Callable[..., Any]) -> "Evaluator":
         nonlocal name
         if not name:
             if hasattr(func, "__self__"):
@@ -108,6 +107,8 @@ def _wrap_coroutine_evaluation_function(
     sig: inspect.Signature,
     convert_to_score: Callable[[Any], EvaluationResult],
 ) -> Callable[[Callable[..., Any]], "Evaluator"]:
+    from phoenix.experiments.evaluators.base import Evaluator
+
     def wrapper(func: Callable[..., Any]) -> "Evaluator":
         class AsyncEvaluator(Evaluator):
             def __init__(self) -> None:
@@ -134,6 +135,8 @@ def _wrap_sync_evaluation_function(
     sig: inspect.Signature,
     convert_to_score: Callable[[Any], EvaluationResult],
 ) -> Callable[[Callable[..., Any]], "Evaluator"]:
+    from phoenix.experiments.evaluators.base import Evaluator
+
     def wrapper(func: Callable[..., Any]) -> "Evaluator":
         class SyncEvaluator(Evaluator):
             def __init__(self) -> None:
@@ -155,138 +158,32 @@ def _wrap_sync_evaluation_function(
 
 
 def _default_eval_scorer(result: Any) -> EvaluationResult:
+    if isinstance(result, EvaluationResult):
+        return result
     if isinstance(result, bool):
         return EvaluationResult(score=float(result), label=str(result))
-    elif isinstance(result, (int, float)):
+    if hasattr(result, "__float__"):
         return EvaluationResult(score=float(result))
-    elif isinstance(result, EvaluationResult):
-        return result
-    else:
-        raise ValueError(f"Unsupported evaluation result type: {type(result)}")
-
-
-ExampleOutput: TypeAlias = Mapping[str, JSONSerializable]
-ExampleMetadata: TypeAlias = Mapping[str, JSONSerializable]
-ExampleInput: TypeAlias = Mapping[str, JSONSerializable]
-
-EvaluatorName: TypeAlias = str
-EvaluatorKind: TypeAlias = str
-EvaluatorOutput: TypeAlias = Union[EvaluationResult, bool, int, float, str]
-
-
-class Evaluator(ABC):
-    """
-    A helper super class to guide the implementation of an `Evaluator` object.
-    Subclasses must implement either the `evaluate` or `async_evaluate` method.
-    Implementing both methods is recommended, but not required.
-
-    This Class is intended to be subclassed, and should not be instantiated directly.
-    """
-
-    _kind: AnnotatorKind
-    _name: EvaluatorName
-
-    @functools.cached_property
-    def name(self) -> EvaluatorName:
-        if hasattr(self, "_name"):
-            return self._name
-        return self.__class__.__name__
-
-    @functools.cached_property
-    def kind(self) -> EvaluatorKind:
-        if hasattr(self, "_kind"):
-            return self._kind.value
-        return AnnotatorKind.CODE.value
-
-    def __new__(cls, *args: Any, **kwargs: Any) -> "Evaluator":
-        if cls is Evaluator:
-            raise TypeError(f"{cls.__name__} is an abstract class and should not be instantiated.")
-        return object.__new__(cls)
-
-    def evaluate(
-        self,
-        *,
-        output: Optional[TaskOutput] = None,
-        expected: Optional[ExampleOutput] = None,
-        metadata: ExampleMetadata = MappingProxyType({}),
-        input: ExampleInput = MappingProxyType({}),
-        **kwargs: Any,
-    ) -> EvaluationResult:
-        # For subclassing, one should implement either this sync method or the
-        # async version. Implementing both is recommended but not required.
-        raise NotImplementedError
-
-    async def async_evaluate(
-        self,
-        *,
-        output: Optional[TaskOutput] = None,
-        expected: Optional[ExampleOutput] = None,
-        metadata: ExampleMetadata = MappingProxyType({}),
-        input: ExampleInput = MappingProxyType({}),
-        **kwargs: Any,
-    ) -> EvaluationResult:
-        # For subclassing, one should implement either this async method or the
-        # sync version. Implementing both is recommended but not required.
-        return self.evaluate(
-            output=output,
-            expected=expected,
-            metadata=metadata,
-            input=input,
-            **kwargs,
-        )
-
-    def __init_subclass__(cls, is_abstract: bool = False, **kwargs: Any) -> None:
-        super().__init_subclass__(**kwargs)
-        if is_abstract:
-            return
-        evaluate_fn_signature = inspect.signature(Evaluator.evaluate)
-        for super_cls in inspect.getmro(cls):
-            if super_cls in (LLMEvaluator, Evaluator):
-                break
-            if evaluate := super_cls.__dict__.get(Evaluator.evaluate.__name__):
-                assert callable(evaluate), "`evaluate()` method should be callable"
-                # need to remove the first param, i.e. `self`
-                _validate_sig(functools.partial(evaluate, None), "evaluate")
-                return
-            if async_evaluate := super_cls.__dict__.get(Evaluator.async_evaluate.__name__):
-                assert callable(async_evaluate), "`async_evaluate()` method should be callable"
-                # need to remove the first param, i.e. `self`
-                _validate_sig(functools.partial(async_evaluate, None), "async_evaluate")
-                return
-        raise ValueError(
-            f"Evaluator must implement either "
-            f"`def evaluate{evaluate_fn_signature}` or "
-            f"`async def async_evaluate{evaluate_fn_signature}`"
-        )
-
-
-def _validate_sig(fn: Callable[..., Any], fn_name: str) -> None:
-    sig = inspect.signature(fn)
-    validate_signature(sig)
-    for param in sig.parameters.values():
-        if param.kind is inspect.Parameter.VAR_KEYWORD:
-            return
-    else:
-        raise ValueError(f"`{fn_name}` should allow variadic keyword arguments `**kwargs`")
-
-
-class LLMEvaluator(Evaluator, ABC, is_abstract=True):
-    """
-    A convenience super class for setting `kind` as LLM.
-
-    This Class is intended to be subclassed, and should not be instantiated directly.
-    """
-
-    _kind = AnnotatorKind.LLM
-
-    def __new__(cls, *args: Any, **kwargs: Any) -> "LLMEvaluator":
-        if cls is LLMEvaluator:
-            raise TypeError(f"{cls.__name__} is an abstract class and should not be instantiated.")
-        return object.__new__(cls)
-
-
-ExperimentEvaluator: TypeAlias = Union[
-    Evaluator,
-    Callable[..., EvaluatorOutput],
-    Callable[..., Awaitable[EvaluatorOutput]],
-]
+    if isinstance(result, str):
+        return EvaluationResult(label=result)
+    if isinstance(result, (tuple, list)) and 0 < len(result) <= 3:
+        # Possible interpretations are:
+        # - 3-tuple: (Score, Label, Explanation)
+        # - 2-tuple: (Score, Explanation) or (Label, Explanation)
+        # - 1-tuple: (Score, ) or (Label, )
+        # Note that (Score, Label) conflicts with (Score, Explanation) and we
+        # pick the latter because it's probably more prevalent. To get
+        # (Score, Label), use a 3-tuple instead, i.e. (Score, Label, None).
+        a, b, c = islice(chain(result, repeat(None)), 3)
+        score, label, explanation = None, a, b
+        if hasattr(a, "__float__"):
+            try:
+                score = float(a)
+            except ValueError:
+                pass
+            else:
+                label, explanation = (None, b) if len(result) < 3 else (b, c)
+        return EvaluationResult(score=score, label=label, explanation=explanation)
+    if result is None:
+        return EvaluationResult(score=0)
+    raise ValueError(f"Unsupported evaluation result type: {type(result)}")