arize 8.0.0a22__py3-none-any.whl → 8.0.0b0__py3-none-any.whl

arize/experiments/evaluators/__init__.py

@@ -0,0 +1 @@
+"""Evaluator implementations for experiment evaluation in the Arize SDK."""

arize/experiments/evaluators/base.py

@@ -1,10 +1,13 @@
+"""Base evaluator classes for experiment evaluation."""
+
 from __future__ import annotations
 
 import functools
 import inspect
 from abc import ABC
+from collections.abc import Awaitable, Callable, Mapping, Sequence
 from types import MappingProxyType
-from typing import Any, Awaitable, Callable, Mapping, Sequence
+from typing import TYPE_CHECKING
 
 from arize.experiments.evaluators.types import (
     AnnotatorKind,
@@ -14,17 +17,19 @@ from arize.experiments.evaluators.types import (
     EvaluatorOutput,
     JSONSerializable,
 )
-from arize.experiments.types import (
-    ExampleInput,
-    ExampleMetadata,
-    ExampleOutput,
-    TaskOutput,
-)
+
+if TYPE_CHECKING:
+    from arize.experiments.types import (
+        ExampleInput,
+        ExampleMetadata,
+        ExampleOutput,
+        TaskOutput,
+    )
 
 
 class Evaluator(ABC):
-    """
-    A helper super class to guide the implementation of an `Evaluator` object.
+    """A helper super class to guide the implementation of an `Evaluator` object.
+
     Subclasses must implement either the `evaluate` or `async_evaluate` method.
     Implementing both methods is recommended, but not required.
 
@@ -36,17 +41,20 @@ class Evaluator(ABC):
 
     @functools.cached_property
     def name(self) -> EvaluatorName:
+        """Return the name of this evaluator."""
         if hasattr(self, "_name"):
             return self._name
         return self.__class__.__name__
 
     @functools.cached_property
     def kind(self) -> EvaluatorKind:
+        """Return the kind of this evaluator (CODE or LLM)."""
         if hasattr(self, "_kind"):
             return self._kind
         return AnnotatorKind.CODE.value
 
-    def __new__(cls, *args: Any, **kwargs: Any) -> Evaluator:
+    def __new__(cls, *args: object, **kwargs: object) -> Evaluator:
+        """Create a new evaluator instance, preventing direct instantiation of abstract class."""
         if cls is Evaluator:
             raise TypeError(
                 f"{cls.__name__} is an abstract class and should not be instantiated."
@@ -62,20 +70,23 @@ class Evaluator(ABC):
         experiment_output: TaskOutput | None = None,
         dataset_output: ExampleOutput = MappingProxyType({}),
         metadata: ExampleMetadata = MappingProxyType({}),
-        **kwargs: Any,
+        **kwargs: object,
     ) -> EvaluationResult:
-        """
-        Evaluate the given inputs and produce an evaluation result.
+        """Evaluate the given inputs and produce an evaluation result.
+
         This method should be implemented by subclasses to perform the actual
         evaluation logic. It is recommended to implement both this synchronous
         method and the asynchronous `async_evaluate` method, but it is not required.
+
         Args:
-            output (Optional[TaskOutput]): The output produced by the task.
-            expected (Optional[ExampleOutput]): The expected output for comparison.
             dataset_row (Optional[Mapping[str, JSONSerializable]]): A row from the dataset.
-            metadata (ExampleMetadata): Metadata associated with the example.
             input (ExampleInput): The input provided for evaluation.
+            output (Optional[TaskOutput]): The output produced by the task.
+            experiment_output (Optional[TaskOutput]): The experiment output for comparison.
+            dataset_output (ExampleOutput): The expected output from the dataset.
+            metadata (ExampleMetadata): Metadata associated with the example.
             **kwargs (Any): Additional keyword arguments.
+
         Raises:
             NotImplementedError: If the method is not implemented by the subclass.
         """
@@ -92,22 +103,26 @@ class Evaluator(ABC):
         experiment_output: TaskOutput | None = None,
         dataset_output: ExampleOutput = MappingProxyType({}),
         metadata: ExampleMetadata = MappingProxyType({}),
-        **kwargs: Any,
+        **kwargs: object,
     ) -> EvaluationResult:
-        """
-        Asynchronously evaluate the given inputs and produce an evaluation result.
+        """Asynchronously evaluate the given inputs and produce an evaluation result.
+
         This method should be implemented by subclasses to perform the actual
         evaluation logic. It is recommended to implement both this asynchronous
         method and the synchronous `evaluate` method, but it is not required.
+
         Args:
-            output (Optional[TaskOutput]): The output produced by the task.
-            expected (Optional[ExampleOutput]): The expected output for comparison.
             dataset_row (Optional[Mapping[str, JSONSerializable]]): A row from the dataset.
-            metadata (ExampleMetadata): Metadata associated with the example.
             input (ExampleInput): The input provided for evaluation.
+            output (Optional[TaskOutput]): The output produced by the task.
+            experiment_output (Optional[TaskOutput]): The experiment output for comparison.
+            dataset_output (ExampleOutput): The expected output from the dataset.
+            metadata (ExampleMetadata): Metadata associated with the example.
             **kwargs (Any): Additional keyword arguments.
+
         Returns:
             EvaluationResult: The result of the evaluation.
+
         Raises:
             NotImplementedError: If the method is not implemented by the subclass.
         """
@@ -124,8 +139,14 @@ class Evaluator(ABC):
         )
 
     def __init_subclass__(
-        cls, is_abstract: bool = False, **kwargs: Any
+        cls, is_abstract: bool = False, **kwargs: object
     ) -> None:
+        """Validate subclass implementation when inherited.
+
+        Args:
+            is_abstract: Whether the subclass is abstract and should skip validation.
+            **kwargs: Additional keyword arguments for parent class.
+        """
         super().__init_subclass__(**kwargs)
         if is_abstract:
             return
@@ -136,9 +157,10 @@ class Evaluator(ABC):
             if evaluate := super_cls.__dict__.get(Evaluator.evaluate.__name__):
                 if isinstance(evaluate, classmethod):
                     evaluate = evaluate.__func__
-                assert callable(evaluate), (
-                    "`evaluate()` method should be callable"
-                )
+                if not callable(evaluate):
+                    raise TypeError(
+                        f"`evaluate()` method should be callable, got {type(evaluate)}"
+                    )
                 # need to remove the first param, i.e. `self`
                 _validate_sig(functools.partial(evaluate, None), "evaluate")
                 return
@@ -147,9 +169,10 @@ class Evaluator(ABC):
             ):
                 if isinstance(async_evaluate, classmethod):
                     async_evaluate = async_evaluate.__func__
-                assert callable(async_evaluate), (
-                    "`async_evaluate()` method should be callable"
-                )
+                if not callable(async_evaluate):
+                    raise TypeError(
+                        f"`async_evaluate()` method should be callable, got {type(async_evaluate)}"
+                    )
                 # need to remove the first param, i.e. `self`
                 _validate_sig(
                     functools.partial(async_evaluate, None), "async_evaluate"
@@ -162,7 +185,7 @@ class Evaluator(ABC):
         )
 
 
-def _validate_sig(fn: Callable[..., Any], fn_name: str) -> None:
+def _validate_sig(fn: Callable[..., object], fn_name: str) -> None:
     sig = inspect.signature(fn)
     validate_evaluator_signature(sig)
     for param in sig.parameters.values():
@@ -175,6 +198,14 @@ def _validate_sig(fn: Callable[..., Any], fn_name: str) -> None:
 
 
 def validate_evaluator_signature(sig: inspect.Signature) -> None:
+    """Validate that a function signature is compatible for use as an evaluator.
+
+    Args:
+        sig: The function signature to validate.
+
+    Raises:
+        ValueError: If the signature is invalid for use as an evaluator.
+    """
     # Check that the wrapped function has a valid signature for use as an evaluator
     # If it does not, raise an error to exit early before running evaluations
     params = sig.parameters
@@ -199,23 +230,24 @@ def validate_evaluator_signature(sig: inspect.Signature) -> None:
             ):
                 continue
             raise ValueError(
-                f"Invalid parameter names in evaluation function: {', '.join(not_found)}. "
+                f"Invalid parameter names in evaluation function: {not_found}. "
                 "Parameters names for multi-argument functions must be "
                 f"any of: {', '.join(valid_named_params)}."
             )
 
 
 class CodeEvaluator(Evaluator, ABC, is_abstract=True):
-    """
-    A convenience super class for defining code evaluators. There are functionally
-    no differences between this class and the `Evaluator` class, except that this
-    class has a default `_kind` attribute for AnnotatorKind.CODE.
+    """A convenience super class for defining code evaluators.
+
+    There are functionally no differences between this class and the `Evaluator` class,
+    except that this class has a default `_kind` attribute for AnnotatorKind.CODE.
     This class is intended to be subclassed, and should not be instantiated directly.
     """
 
     _kind = str(AnnotatorKind.CODE)
 
-    def __new__(cls, *args: Any, **kwargs: Any) -> CodeEvaluator:
+    def __new__(cls, *args: object, **kwargs: object) -> CodeEvaluator:
+        """Create a new code evaluator instance, preventing direct instantiation of abstract class."""
         if cls is CodeEvaluator:
             raise TypeError(
                 f"{cls.__name__} is an abstract class and should not be instantiated."
@@ -224,16 +256,17 @@ class CodeEvaluator(Evaluator, ABC, is_abstract=True):
 
 
 class LLMEvaluator(Evaluator, ABC, is_abstract=True):
-    """
-    A convenience super class for defining LLM evaluators. There are functionally
-    no differences between this class and the `Evaluator` class, except that this
-    class has a default `_kind` attribute for AnnotatorKind.LLM.
+    """A convenience super class for defining LLM evaluators.
+
+    There are functionally no differences between this class and the `Evaluator` class,
+    except that this class has a default `_kind` attribute for AnnotatorKind.LLM.
     This class is intended to be subclassed, and should not be instantiated directly.
     """
 
     _kind = str(AnnotatorKind.LLM)
 
-    def __new__(cls, *args: Any, **kwargs: Any) -> LLMEvaluator:
+    def __new__(cls, *args: object, **kwargs: object) -> LLMEvaluator:
+        """Create a new LLM evaluator instance, preventing direct instantiation of abstract class."""
         if cls is LLMEvaluator:
             raise TypeError(
                 f"{cls.__name__} is an abstract class and should not be instantiated."

arize/experiments/evaluators/exceptions.py

@@ -1,10 +1,13 @@
+"""Evaluator-specific exception classes."""
+
+
 class ArizeException(Exception):
-    pass
+    """Base exception for Arize experiment evaluator errors."""
 
 
 class ArizeContextLimitExceeded(ArizeException):
-    pass
+    """Raised when context limit is exceeded during evaluation."""
 
 
 class ArizeTemplateMappingError(ArizeException):
-    pass
+    """Raised when template mapping fails during evaluation."""