arize-phoenix 3.16.0__py3-none-any.whl → 7.7.0__py3-none-any.whl

phoenix/exceptions.py

@@ -2,5 +2,9 @@ class PhoenixException(Exception):
     pass
 
 
-class PhoenixContextLimitExceeded(PhoenixException):
+class PhoenixEvaluationNameIsMissing(PhoenixException):
+    pass
+
+
+class PhoenixMigrationError(PhoenixException):
     pass

phoenix/experiments/__init__.py

@@ -0,0 +1,6 @@
+from .functions import evaluate_experiment, run_experiment
+
+__all__ = [
+    "evaluate_experiment",
+    "run_experiment",
+]

phoenix/experiments/evaluators/__init__.py

@@ -0,0 +1,29 @@
+from phoenix.experiments.evaluators.code_evaluators import (
+    ContainsAllKeywords,
+    ContainsAnyKeyword,
+    ContainsKeyword,
+    JSONParsable,
+    MatchesRegex,
+)
+from phoenix.experiments.evaluators.llm_evaluators import (
+    CoherenceEvaluator,
+    ConcisenessEvaluator,
+    HelpfulnessEvaluator,
+    LLMCriteriaEvaluator,
+    RelevanceEvaluator,
+)
+from phoenix.experiments.evaluators.utils import create_evaluator
+
+__all__ = [
+    "create_evaluator",
+    "ContainsAllKeywords",
+    "ContainsAnyKeyword",
+    "ContainsKeyword",
+    "JSONParsable",
+    "MatchesRegex",
+    "CoherenceEvaluator",
+    "ConcisenessEvaluator",
+    "LLMCriteriaEvaluator",
+    "HelpfulnessEvaluator",
+    "RelevanceEvaluator",
+]

phoenix/experiments/evaluators/base.py

@@ -0,0 +1,158 @@
+import functools
+import inspect
+from abc import ABC
+from collections.abc import Awaitable, Callable
+from types import MappingProxyType
+from typing import Any, Optional, Union
+
+from typing_extensions import TypeAlias
+
+from phoenix.experiments.evaluators.utils import validate_evaluator_signature
+from phoenix.experiments.types import (
+    AnnotatorKind,
+    EvaluationResult,
+    EvaluatorKind,
+    EvaluatorName,
+    EvaluatorOutput,
+    ExampleInput,
+    ExampleMetadata,
+    ExampleOutput,
+    TaskOutput,
+)
+
+
+class Evaluator(ABC):
+    """
+    A helper super class to guide the implementation of an `Evaluator` object.
+    Subclasses must implement either the `evaluate` or `async_evaluate` method.
+    Implementing both methods is recommended, but not required.
+
+    This Class is intended to be subclassed, and should not be instantiated directly.
+    """
+
+    _kind: AnnotatorKind
+    _name: EvaluatorName
+
+    @functools.cached_property
+    def name(self) -> EvaluatorName:
+        if hasattr(self, "_name"):
+            return self._name
+        return self.__class__.__name__
+
+    @functools.cached_property
+    def kind(self) -> EvaluatorKind:
+        if hasattr(self, "_kind"):
+            return self._kind.value
+        return AnnotatorKind.CODE.value
+
+    def __new__(cls, *args: Any, **kwargs: Any) -> "Evaluator":
+        if cls is Evaluator:
+            raise TypeError(f"{cls.__name__} is an abstract class and should not be instantiated.")
+        return object.__new__(cls)
+
+    def evaluate(
+        self,
+        *,
+        output: Optional[TaskOutput] = None,
+        expected: Optional[ExampleOutput] = None,
+        metadata: ExampleMetadata = MappingProxyType({}),
+        input: ExampleInput = MappingProxyType({}),
+        **kwargs: Any,
+    ) -> EvaluationResult:
+        # For subclassing, one should implement either this sync method or the
+        # async version. Implementing both is recommended but not required.
+        raise NotImplementedError
+
+    async def async_evaluate(
+        self,
+        *,
+        output: Optional[TaskOutput] = None,
+        expected: Optional[ExampleOutput] = None,
+        metadata: ExampleMetadata = MappingProxyType({}),
+        input: ExampleInput = MappingProxyType({}),
+        **kwargs: Any,
+    ) -> EvaluationResult:
+        # For subclassing, one should implement either this async method or the
+        # sync version. Implementing both is recommended but not required.
+        return self.evaluate(
+            output=output,
+            expected=expected,
+            metadata=metadata,
+            input=input,
+            **kwargs,
+        )
+
+    def __init_subclass__(cls, is_abstract: bool = False, **kwargs: Any) -> None:
+        super().__init_subclass__(**kwargs)
+        if is_abstract:
+            return
+        evaluate_fn_signature = inspect.signature(Evaluator.evaluate)
+        for super_cls in inspect.getmro(cls):
+            if super_cls in (LLMEvaluator, Evaluator):
+                break
+            if evaluate := super_cls.__dict__.get(Evaluator.evaluate.__name__):
+                if isinstance(evaluate, classmethod):
+                    evaluate = evaluate.__func__
+                assert callable(evaluate), "`evaluate()` method should be callable"
+                # need to remove the first param, i.e. `self`
+                _validate_sig(functools.partial(evaluate, None), "evaluate")
+                return
+            if async_evaluate := super_cls.__dict__.get(Evaluator.async_evaluate.__name__):
+                if isinstance(async_evaluate, classmethod):
+                    async_evaluate = async_evaluate.__func__
+                assert callable(async_evaluate), "`async_evaluate()` method should be callable"
+                # need to remove the first param, i.e. `self`
+                _validate_sig(functools.partial(async_evaluate, None), "async_evaluate")
+                return
+        raise ValueError(
+            f"Evaluator must implement either "
+            f"`def evaluate{evaluate_fn_signature}` or "
+            f"`async def async_evaluate{evaluate_fn_signature}`"
+        )
+
+
+def _validate_sig(fn: Callable[..., Any], fn_name: str) -> None:
+    sig = inspect.signature(fn)
+    validate_evaluator_signature(sig)
+    for param in sig.parameters.values():
+        if param.kind is inspect.Parameter.VAR_KEYWORD:
+            return
+    else:
+        raise ValueError(f"`{fn_name}` should allow variadic keyword arguments `**kwargs`")
+
+
+class CodeEvaluator(Evaluator, ABC, is_abstract=True):
+    """
+    A convenience super class for defining code evaluators.
+
+    This class is intended to be subclassed, and should not be instantiated directly.
+    """
+
+    _kind = AnnotatorKind.CODE
+
+    def __new__(cls, *args: Any, **kwargs: Any) -> "CodeEvaluator":
+        if cls is CodeEvaluator:
+            raise TypeError(f"{cls.__name__} is an abstract class and should not be instantiated.")
+        return object.__new__(cls)
+
+
+class LLMEvaluator(Evaluator, ABC, is_abstract=True):
+    """
+    A convenience super class for defining LLM evaluators.
+
+    This class is intended to be subclassed, and should not be instantiated directly.
+    """
+
+    _kind = AnnotatorKind.LLM
+
+    def __new__(cls, *args: Any, **kwargs: Any) -> "LLMEvaluator":
+        if cls is LLMEvaluator:
+            raise TypeError(f"{cls.__name__} is an abstract class and should not be instantiated.")
+        return object.__new__(cls)
+
+
+ExperimentEvaluator: TypeAlias = Union[
+    Evaluator,
+    Callable[..., EvaluatorOutput],
+    Callable[..., Awaitable[EvaluatorOutput]],
+]

phoenix/experiments/evaluators/code_evaluators.py

@@ -0,0 +1,184 @@
+from __future__ import annotations
+
+import json
+import re
+from typing import (
+    Any,
+    Optional,
+    Pattern,  # import from re module when we drop support for 3.8
+    Union,
+)
+
+from phoenix.experiments.evaluators.base import CodeEvaluator
+from phoenix.experiments.types import EvaluationResult, TaskOutput
+
+
+class JSONParsable(CodeEvaluator):
+    """
+    An evaluator that checks if the output of an experiment run is a JSON-parsable string.
+
+    Example:
+
+        .. code-block:: python
+            from phoenix.experiments import run_experiment
+            from phoenix.experiments.evaluators import JSONParsable
+
+            run_experiment(dataset, task, evaluators=[JSONParsable])
+    """
+
+    @classmethod
+    def evaluate(self, *, output: Optional[TaskOutput] = None, **_: Any) -> EvaluationResult:
+        assert isinstance(output, str), "Experiment run output must be a string"
+        try:
+            json.loads(output)
+            json_parsable = True
+        except BaseException:
+            json_parsable = False
+        return EvaluationResult(
+            score=int(json_parsable),
+        )
+
+
+class ContainsKeyword(CodeEvaluator):
+    """
+    An evaluator that checks if a keyword is present in the output of an experiment run.
+
+    Args:
+        keyword (str): The keyword to search for in the output.
+        name (str, optional): An optional name for the evaluator. Defaults to "Contains(<keyword>)".
+
+    Example:
+
+        .. code-block:: python
+            from phoenix.experiments import run_experiment
+            from phoenix.experiments.evaluators import ContainsKeyword
+
+            run_experiment(dataset, task, evaluators=[ContainsKeyword("foo")])
+    """
+
+    def __init__(self, keyword: str, name: Optional[str] = None) -> None:
+        self.keyword = keyword
+        self._name = name or f"Contains({repr(keyword)})"
+
+    def evaluate(self, *, output: Optional[TaskOutput] = None, **_: Any) -> EvaluationResult:
+        assert isinstance(output, str), "Experiment run output must be a string"
+        found = self.keyword in output
+        return EvaluationResult(
+            score=float(found),
+            explanation=(
+                f"the string {repr(self.keyword)} was "
+                f"{'found' if found else 'not found'} in the output"
+            ),
+        )
+
+
+class ContainsAnyKeyword(CodeEvaluator):
+    """
+    An evaluator that checks if any of the keywords are present in the output of an experiment run.
+
+    Args:
+        keywords (list[str]): The keywords to search for in the output.
+        name (str, optional): An optional name for the evaluator. Defaults to
+            "ContainsAny(<keywords>)".
+
+    Example:
+
+        .. code-block:: python
+            from phoenix.experiments import run_experiment
+            from phoenix.experiments.evaluators import ContainsAnyKeyword
+
+            run_experiment(dataset, task, evaluators=[ContainsAnyKeyword(["foo", "bar"])])
+    """
+
+    def __init__(self, keywords: list[str], name: Optional[str] = None) -> None:
+        self.keywords = keywords
+        self._name = name or f"ContainsAny({keywords})"
+
+    def evaluate(self, *, output: Optional[TaskOutput] = None, **_: Any) -> EvaluationResult:
+        assert isinstance(output, str), "Experiment run output must be a string"
+        found = [keyword for keyword in self.keywords if keyword in output]
+        if found:
+            explanation = f"the keywords {found} were found in the output"
+        else:
+            explanation = f"none of the keywords {self.keywords} were found in the output"
+        return EvaluationResult(
+            score=float(bool(found)),
+            explanation=explanation,
+        )
+
+
+class ContainsAllKeywords(CodeEvaluator):
+    """
+    An evaluator that checks if all of the keywords are present in the output of an experiment run.
+
+    Args:
+        keywords (list[str]): The keywords to search for in the output.
+        name (str, optional): An optional name for the evaluator. Defaults to
+            "ContainsAll(<keywords>)".
+
+    Example:
+        .. code-block:: python
+
+            from phoenix.experiments import run_experiment
+            from phoenix.experiments.evaluators import ContainsAllKeywords
+
+            run_experiment(dataset, task, evaluators=[ContainsAllKeywords(["foo", "bar"])])
+    """
+
+    def __init__(self, keywords: list[str], name: Optional[str] = None) -> None:
+        self.keywords = keywords
+        self._name = name or f"ContainsAll({keywords})"
+
+    def evaluate(self, *, output: Optional[TaskOutput] = None, **_: Any) -> EvaluationResult:
+        assert isinstance(output, str), "Experiment run output must be a string"
+        not_found = [keyword for keyword in self.keywords if keyword not in output]
+        if not_found:
+            contains_all = False
+            explanation = f"the keywords {not_found} were not found in the output"
+        else:
+            contains_all = True
+            explanation = f"all of the keywords {self.keywords} were found in the output"
+        return EvaluationResult(
+            score=float(contains_all),
+            explanation=explanation,
+        )
+
+
+class MatchesRegex(CodeEvaluator):
+    r"""
+    An experiment evaluator that checks if the output of an experiment run matches a regex pattern.
+
+    Args:
+        pattern (Union[str, Pattern[str]]): The regex pattern to match the output against.
+        name (str, optional): An optional name for the evaluator. Defaults to "matches_({pattern})".
+
+    Example:
+        .. code-block:: python
+
+            from phoenix.experiments import run_experiment
+            from phoenix.experiments.evaluators import MatchesRegex
+
+            phone_number_evaluator = MatchesRegex(r"\d{3}-\d{3}-\d{4}", name="valid-phone-number")
+            run_experiment(dataset, task, evaluators=[phone_number_evaluator])
+    """
+
+    def __init__(self, pattern: Union[str, Pattern[str]], name: Optional[str] = None) -> None:
+        if isinstance(pattern, str):
+            pattern = re.compile(pattern)
+        self.pattern = pattern
+        assert isinstance(pattern, re.Pattern)
+        self._name = name or f"matches_({pattern})"
+
+    def evaluate(self, *, output: Optional[TaskOutput] = None, **_: Any) -> EvaluationResult:
+        assert isinstance(output, str), "Experiment run output must be a string"
+        matches = self.pattern.findall(output)
+        if matches:
+            explanation = (
+                f"the substrings {matches} matched the regex pattern {self.pattern.pattern}"
+            )
+        else:
+            explanation = f"no substrings matched the regex pattern {self.pattern.pattern}"
+        return EvaluationResult(
+            score=float(bool(matches)),
+            explanation=explanation,
+        )