arize-phoenix 4.4.4rc5__py3-none-any.whl → 4.5.0__py3-none-any.whl

phoenix/datasets/evaluators/llm_evaluators.py

@@ -1,244 +0,0 @@
-import re
-from types import MappingProxyType
-from typing import Any, Callable, Optional, Type
-
-from phoenix.datasets.evaluators.utils import (
-    ExampleInput,
-    ExampleMetadata,
-    ExperimentEvaluator,
-    LLMEvaluator,
-    _unwrap_json,
-)
-from phoenix.datasets.types import (
-    EvaluationResult,
-    TaskOutput,
-)
-from phoenix.evals.models.base import BaseModel as LLMBaseModel
-from phoenix.evals.utils import snap_to_rail
-
-
-class LLMCriteriaEvaluator(LLMEvaluator):
-    _base_template = (
-        "Determine if the following text is {criteria}. {description}"
-        "First, explain step-by-step why you think the text is or is not {criteria}. Then provide "
-        "a single word label; 'true' if the text is {criteria} or 'false' if the text is not "
-        "{criteria}. Here is an example template for whether the text meets a criteria:\n\n"
-        "CRITERIA: the text is '{criteria}'\n"
-        "TEXT: *the provided text to evaluate*\n"
-        "EXPLANATION: *a step by step explanation of your reasoning for whether the text meets "
-        "the criteria*\n"
-        "LABEL: *true or false*\n\n"
-        "Follow this template for the following example:\n\n"
-        "CRITERIA: the text is '{criteria}'\n"
-        "TEXT: {text}\n"
-        "EXPLANATION: "
-    )
-    _description = "In this context, '{criteria}' means the text '{description}'. "
-
-    def __init__(
-        self,
-        model: LLMBaseModel,
-        criteria: str,
-        description: str,
-        name: str,
-    ):
-        self.model = model
-        self.criteria = criteria
-        self.description = description
-        self.template = self._format_base_template(self.criteria, self.description)
-        self._name = name
-
-    def evaluate(self, *, output: Optional[TaskOutput] = None, **_: Any) -> EvaluationResult:
-        formatted_template = self._format_eval_template(output)
-        unparsed_response = self.model._generate(formatted_template)
-        return self._parse_eval_output(unparsed_response)
-
-    async def async_evaluate(
-        self, *, output: Optional[TaskOutput] = None, **_: Any
-    ) -> EvaluationResult:
-        formatted_template = self._format_eval_template(output)
-        unparsed_response = await self.model._async_generate(formatted_template)
-        return self._parse_eval_output(unparsed_response)
-
-    def _format_eval_template(self, output: TaskOutput) -> str:
-        assert output is not None
-        result = _unwrap_json(output)
-        return self.template.format(text=str(result))
-
-    def _parse_eval_output(self, unparsed_response: str) -> EvaluationResult:
-        raw_label, explanation = (
-            _parse_label_from_explanation(unparsed_response),
-            unparsed_response,
-        )
-        label = snap_to_rail(raw_label, ["true", "false"])
-        if label == "true":
-            score = 1.0
-        elif label == "false":
-            score = 0.0
-        else:
-            raise RuntimeError(f"Could not parse LLM evaluation: {unparsed_response}")
-        return EvaluationResult(
-            score=score,
-            explanation=explanation,
-            metadata={},
-        )
-
-    @classmethod
-    def _format_base_template(cls, criteria: str, description: Optional[str] = None) -> str:
-        formatted_description = cls._description.format(criteria=criteria, description=description)
-        formatted_template = cls._base_template.format(
-            criteria=criteria,
-            description=formatted_description,
-            text="{text}",  # leave the text field as a placeholder
-        )
-        return formatted_template
-
-
-def criteria_evaluator_factory(
-    class_name: str, criteria: str, description: str, default_name: str
-) -> Type[ExperimentEvaluator]:
-    def _init(self, model: LLMBaseModel, name: str = default_name) -> None:  # type: ignore
-        LLMCriteriaEvaluator.__init__(self, model, criteria, description, name=name)
-
-    return type(
-        class_name,
-        (LLMCriteriaEvaluator,),
-        {
-            "__init__": _init,
-            "__module__": __name__,
-            "template": LLMCriteriaEvaluator._format_base_template(criteria, description),
-        },
-    )
-
-
-ConcisenessEvaluator = criteria_evaluator_factory(
-    class_name="ConcisenessEvaluator",
-    criteria="concise",
-    description="is just a few sentences and easy to follow",
-    default_name="Conciseness",
-)
-
-
-HelpfulnessEvaluator = criteria_evaluator_factory(
-    class_name="HelpfulnessEvaluator",
-    criteria="helpful",
-    description="provides useful information",
-    default_name="Helpfulness",
-)
-
-
-CoherenceEvaluator = criteria_evaluator_factory(
-    class_name="CoherenceEvaluator",
-    criteria="coherent",
-    description="is coherent, well-structured, and logically sound",
-    default_name="Coherence",
-)
-
-
-def _parse_label_from_explanation(raw_string: str) -> str:
-    label_delimiter = r"(\W*label\W*)"
-    parts = re.split(label_delimiter, raw_string, flags=re.IGNORECASE)
-    if len(parts) > 1:
-        # Find the last occurrence of the delimiter and take the part after it
-        last_index = len(parts) - 1
-        while last_index > 0:
-            if re.match(label_delimiter, parts[last_index - 1], flags=re.IGNORECASE):
-                return parts[last_index].strip()
-            last_index -= 1
-    return raw_string
-
-
-class RelevanceEvaluator(LLMEvaluator):
-    template = (
-        "Determine if the following response is relevant to the query. In this context, "
-        "'relevance' means that the response directly addresses the core question or topic of the "
-        "query. First, explain step-by-step why you think the text is or is not relevant. "
-        "Then provide a single word label; 'true' if the text is relevant or 'false' if the text "
-        "is not relevant. "
-        "Here is an example template for your reponse:\n\n"
-        "CRITERIA: the response is 'relevant' to the query\n"
-        "QUERY: *text that contains a query*\n"
-        "RESPONSE: *a response that may or may not be relevant to the query*\n"
-        "EXPLANATION: *a step by step explanation of your reasoning for whether or not the "
-        "response is relevant to the query*\n"
-        "LABEL: *true or false*\n\n"
-        "Follow this template for the following example:\n\n"
-        "CRITERIA: the response is 'relevant' to the query\n"
-        "QUERY: {reference}\n"
-        "RESPONSE: {submission}\n"
-        "EXPLANATION: "
-    )
-
-    def __init__(
-        self,
-        model: LLMBaseModel,
-        get_query: Optional[Callable[[ExampleInput, ExampleMetadata], str]] = None,
-        get_response: Optional[Callable[[Optional[TaskOutput], ExampleMetadata], str]] = None,
-        name: str = "RelevanceEvaluator",
-    ):
-        self.model = model
-        self._name = name
-        self.get_query = get_query or self._default_get_query
-        self.get_response = get_response or self._default_get_response
-
-    def _format_eval_template(
-        self,
-        output: Optional[TaskOutput] = None,
-        input: ExampleInput = MappingProxyType({}),
-        metadata: ExampleMetadata = MappingProxyType({}),
-    ) -> str:
-        assert output is not None
-        query = self.get_query(input, metadata)
-        response = self.get_response(output, metadata)
-        return self.template.format(query=query, response=response)
-
-    def _parse_eval_output(self, unparsed_response: str) -> EvaluationResult:
-        raw_label, explanation = (
-            _parse_label_from_explanation(unparsed_response),
-            unparsed_response,
-        )
-        label = snap_to_rail(raw_label, ["true", "false"])
-        if label == "true":
-            score = 1.0
-        elif label == "false":
-            score = 0.0
-        else:
-            raise RuntimeError(f"Could not parse LLM evaluation: {unparsed_response}")
-        return EvaluationResult(
-            score=score,
-            explanation=explanation,
-            metadata={},
-        )
-
-    def _default_get_query(self, input: ExampleInput, *args: Any, **kwargs: Any) -> str:
-        return str(input)
-
-    def _default_get_response(
-        self, output: Optional[TaskOutput] = None, *args: Any, **kwargs: Any
-    ) -> str:
-        assert output is not None
-        return str(_unwrap_json(output))
-
-    def evaluate(
-        self,
-        *,
-        output: Optional[TaskOutput] = None,
-        metadata: ExampleMetadata = MappingProxyType({}),
-        input: ExampleInput = MappingProxyType({}),
-        **_: Any,
-    ) -> EvaluationResult:
-        formatted_template = self._format_eval_template(output, input, metadata)
-        unparsed_response = self.model._generate(formatted_template)
-        return self._parse_eval_output(unparsed_response)
-
-    async def async_evaluate(
-        self,
-        *,
-        output: Optional[TaskOutput] = None,
-        metadata: ExampleMetadata = MappingProxyType({}),
-        input: ExampleInput = MappingProxyType({}),
-        **_: Any,
-    ) -> EvaluationResult:
-        formatted_template = self._format_eval_template(output, input, metadata)
-        unparsed_response = await self.model._async_generate(formatted_template)
-        return self._parse_eval_output(unparsed_response)

phoenix/datasets/evaluators/utils.py

@@ -1,292 +0,0 @@
-import functools
-import inspect
-from abc import ABC
-from types import MappingProxyType
-from typing import Any, Awaitable, Callable, Mapping, Optional, Union
-
-from typing_extensions import TypeAlias
-
-from phoenix.datasets.types import (
-    AnnotatorKind,
-    EvaluationResult,
-    JSONSerializable,
-    TaskOutput,
-)
-
-
-def _unwrap_json(obj: JSONSerializable) -> JSONSerializable:
-    if isinstance(obj, dict):
-        if len(obj) == 1:
-            key = next(iter(obj.keys()))
-            output = obj[key]
-            assert isinstance(
-                output, (dict, list, str, int, float, bool, type(None))
-            ), "Output must be JSON serializable"
-            return output
-    return obj
-
-
-def validate_signature(sig: inspect.Signature) -> None:
-    # Check that the wrapped function has a valid signature for use as an evaluator
-    # If it does not, raise an error to exit early before running evaluations
-    params = sig.parameters
-    valid_named_params = {"input", "output", "expected", "metadata"}
-    if len(params) == 0:
-        raise ValueError("Evaluation function must have at least one parameter.")
-    if len(params) > 1:
-        for not_found in set(params) - valid_named_params:
-            param = params[not_found]
-            if (
-                param.kind is inspect.Parameter.VAR_KEYWORD
-                or param.default is not inspect.Parameter.empty
-            ):
-                continue
-            raise ValueError(
-                (
-                    f"Invalid parameter names in evaluation function: {', '.join(not_found)}. "
-                    "Parameters names for multi-argument functions must be "
-                    f"any of: {', '.join(valid_named_params)}."
-                )
-            )
-
-
-def _bind_signature(sig: inspect.Signature, **kwargs: Any) -> inspect.BoundArguments:
-    parameter_mapping = {
-        "input": kwargs.get("input"),
-        "output": kwargs.get("output"),
-        "expected": kwargs.get("expected"),
-        "metadata": kwargs.get("metadata"),
-    }
-    params = sig.parameters
-    if len(params) == 1:
-        parameter_name = next(iter(params))
-        if parameter_name in parameter_mapping:
-            return sig.bind(parameter_mapping[parameter_name])
-        else:
-            return sig.bind(parameter_mapping["output"])
-    return sig.bind_partial(
-        **{name: parameter_mapping[name] for name in set(parameter_mapping).intersection(params)}
-    )
-
-
-def create_evaluator(
-    kind: Union[str, AnnotatorKind] = AnnotatorKind.CODE,
-    name: Optional[str] = None,
-    scorer: Optional[Callable[[Any], EvaluationResult]] = None,
-) -> Callable[[Callable[..., Any]], "Evaluator"]:
-    if scorer is None:
-        scorer = _default_eval_scorer
-
-    if isinstance(kind, str):
-        kind = AnnotatorKind(kind.upper())
-
-    def wrapper(func: Callable[..., Any]) -> Evaluator:
-        nonlocal name
-        if not name:
-            if hasattr(func, "__self__"):
-                name = func.__self__.__class__.__name__
-            elif hasattr(func, "__name__"):
-                name = func.__name__
-            else:
-                name = str(func)
-        assert name is not None
-
-        wrapped_signature = inspect.signature(func)
-        validate_signature(wrapped_signature)
-
-        if inspect.iscoroutinefunction(func):
-            return _wrap_coroutine_evaluation_function(name, kind, wrapped_signature, scorer)(func)
-        else:
-            return _wrap_sync_evaluation_function(name, kind, wrapped_signature, scorer)(func)
-
-    return wrapper
-
-
-def _wrap_coroutine_evaluation_function(
-    name: str,
-    annotator_kind: AnnotatorKind,
-    sig: inspect.Signature,
-    convert_to_score: Callable[[Any], EvaluationResult],
-) -> Callable[[Callable[..., Any]], "Evaluator"]:
-    def wrapper(func: Callable[..., Any]) -> "Evaluator":
-        class AsyncEvaluator(Evaluator):
-            def __init__(self) -> None:
-                self._name = name
-                self._kind = annotator_kind
-
-            @functools.wraps(func)
-            async def __call__(self, *args: Any, **kwargs: Any) -> Any:
-                return await func(*args, **kwargs)
-
-            async def async_evaluate(self, **kwargs: Any) -> EvaluationResult:
-                bound_signature = _bind_signature(sig, **kwargs)
-                result = await func(*bound_signature.args, **bound_signature.kwargs)
-                return convert_to_score(result)
-
-        return AsyncEvaluator()
-
-    return wrapper
-
-
-def _wrap_sync_evaluation_function(
-    name: str,
-    annotator_kind: AnnotatorKind,
-    sig: inspect.Signature,
-    convert_to_score: Callable[[Any], EvaluationResult],
-) -> Callable[[Callable[..., Any]], "Evaluator"]:
-    def wrapper(func: Callable[..., Any]) -> "Evaluator":
-        class SyncEvaluator(Evaluator):
-            def __init__(self) -> None:
-                self._name = name
-                self._kind = annotator_kind
-
-            @functools.wraps(func)
-            def __call__(self, *args: Any, **kwargs: Any) -> Any:
-                return func(*args, **kwargs)
-
-            def evaluate(self, **kwargs: Any) -> EvaluationResult:
-                bound_signature = _bind_signature(sig, **kwargs)
-                result = func(*bound_signature.args, **bound_signature.kwargs)
-                return convert_to_score(result)
-
-        return SyncEvaluator()
-
-    return wrapper
-
-
-def _default_eval_scorer(result: Any) -> EvaluationResult:
-    if isinstance(result, bool):
-        return EvaluationResult(score=float(result), label=str(result))
-    elif isinstance(result, (int, float)):
-        return EvaluationResult(score=float(result))
-    elif isinstance(result, EvaluationResult):
-        return result
-    else:
-        raise ValueError(f"Unsupported evaluation result type: {type(result)}")
-
-
-ExampleOutput: TypeAlias = Mapping[str, JSONSerializable]
-ExampleMetadata: TypeAlias = Mapping[str, JSONSerializable]
-ExampleInput: TypeAlias = Mapping[str, JSONSerializable]
-
-EvaluatorName: TypeAlias = str
-EvaluatorKind: TypeAlias = str
-EvaluatorOutput: TypeAlias = Union[EvaluationResult, bool, int, float, str]
-
-
-class Evaluator(ABC):
-    """
-    A helper super class to guide the implementation of an `Evaluator` object.
-    Subclasses must implement either the `evaluate` or `async_evaluate` method.
-    Implementing both methods is recommended, but not required.
-
-    This Class is intended to be subclassed, and should not be instantiated directly.
-    """
-
-    _kind: AnnotatorKind
-    _name: EvaluatorName
-
-    @functools.cached_property
-    def name(self) -> EvaluatorName:
-        if hasattr(self, "_name"):
-            return self._name
-        return self.__class__.__name__
-
-    @functools.cached_property
-    def kind(self) -> EvaluatorKind:
-        if hasattr(self, "_kind"):
-            return self._kind.value
-        return AnnotatorKind.CODE.value
-
-    def __new__(cls, *args: Any, **kwargs: Any) -> "Evaluator":
-        if cls is Evaluator:
-            raise TypeError(f"{cls.__name__} is an abstract class and should not be instantiated.")
-        return object.__new__(cls)
-
-    def evaluate(
-        self,
-        *,
-        output: Optional[TaskOutput] = None,
-        expected: Optional[ExampleOutput] = None,
-        metadata: ExampleMetadata = MappingProxyType({}),
-        input: ExampleInput = MappingProxyType({}),
-        **kwargs: Any,
-    ) -> EvaluationResult:
-        # For subclassing, one should implement either this sync method or the
-        # async version. Implementing both is recommended but not required.
-        raise NotImplementedError
-
-    async def async_evaluate(
-        self,
-        *,
-        output: Optional[TaskOutput] = None,
-        expected: Optional[ExampleOutput] = None,
-        metadata: ExampleMetadata = MappingProxyType({}),
-        input: ExampleInput = MappingProxyType({}),
-        **kwargs: Any,
-    ) -> EvaluationResult:
-        # For subclassing, one should implement either this async method or the
-        # sync version. Implementing both is recommended but not required.
-        return self.evaluate(
-            output=output,
-            expected=expected,
-            metadata=metadata,
-            input=input,
-            **kwargs,
-        )
-
-    def __init_subclass__(cls, is_abstract: bool = False, **kwargs: Any) -> None:
-        super().__init_subclass__(**kwargs)
-        if is_abstract:
-            return
-        evaluate_fn_signature = inspect.signature(Evaluator.evaluate)
-        for super_cls in inspect.getmro(cls):
-            if super_cls in (LLMEvaluator, Evaluator):
-                break
-            if evaluate := super_cls.__dict__.get(Evaluator.evaluate.__name__):
-                assert callable(evaluate), "`evaluate()` method should be callable"
-                # need to remove the first param, i.e. `self`
-                _validate_sig(functools.partial(evaluate, None), "evaluate")
-                return
-            if async_evaluate := super_cls.__dict__.get(Evaluator.async_evaluate.__name__):
-                assert callable(async_evaluate), "`async_evaluate()` method should be callable"
-                # need to remove the first param, i.e. `self`
-                _validate_sig(functools.partial(async_evaluate, None), "async_evaluate")
-                return
-        raise ValueError(
-            f"Evaluator must implement either "
-            f"`def evaluate{evaluate_fn_signature}` or "
-            f"`async def async_evaluate{evaluate_fn_signature}`"
-        )
-
-
-def _validate_sig(fn: Callable[..., Any], fn_name: str) -> None:
-    sig = inspect.signature(fn)
-    validate_signature(sig)
-    for param in sig.parameters.values():
-        if param.kind is inspect.Parameter.VAR_KEYWORD:
-            return
-    else:
-        raise ValueError(f"`{fn_name}` should allow variadic keyword arguments `**kwargs`")
-
-
-class LLMEvaluator(Evaluator, ABC, is_abstract=True):
-    """
-    A convenience super class for setting `kind` as LLM.
-
-    This Class is intended to be subclassed, and should not be instantiated directly.
-    """
-
-    _kind = AnnotatorKind.LLM
-
-    def __new__(cls, *args: Any, **kwargs: Any) -> "LLMEvaluator":
-        if cls is LLMEvaluator:
-            raise TypeError(f"{cls.__name__} is an abstract class and should not be instantiated.")
-        return object.__new__(cls)
-
-
-ExperimentEvaluator: TypeAlias = Union[
-    Evaluator,
-    Callable[..., EvaluatorOutput],
-    Callable[..., Awaitable[EvaluatorOutput]],
-]