arize-phoenix 5.5.2__py3-none-any.whl → 5.7.0__py3-none-any.whl

phoenix/experiments/evaluators/llm_evaluators.py

@@ -1,6 +1,7 @@
 import re
+from collections.abc import Callable
 from types import MappingProxyType
-from typing import Any, Callable, Optional, Type
+from typing import Any, Optional
 
 from phoenix.evals.models.base import BaseModel as LLMBaseModel
 from phoenix.evals.utils import snap_to_rail
@@ -121,7 +122,7 @@ class LLMCriteriaEvaluator(LLMEvaluator):
 
 def criteria_evaluator_factory(
     class_name: str, criteria: str, description: str, default_name: str
-) -> Type[ExperimentEvaluator]:
+) -> type[ExperimentEvaluator]:
     def _init(self, model: LLMBaseModel, name: str = default_name) -> None:  # type: ignore
         LLMCriteriaEvaluator.__init__(self, model, criteria, description, name=name)
 
@@ -240,8 +241,8 @@ class RelevanceEvaluator(LLMEvaluator):
         "LABEL: *true or false*\n\n"
         "Follow this template for the following example:\n\n"
         "CRITERIA: the response is 'relevant' to the query\n"
-        "QUERY: {reference}\n"
-        "RESPONSE: {submission}\n"
+        "QUERY: {query}\n"
+        "RESPONSE: {response}\n"
         "EXPLANATION: "
     )
 
@@ -318,3 +319,155 @@ class RelevanceEvaluator(LLMEvaluator):
         formatted_template = self._format_eval_template(output, input, metadata)
         unparsed_response = await self.model._async_generate(formatted_template)
         return self._parse_eval_output(unparsed_response)
+
+
+class LLMRelationalEvaluator(LLMEvaluator):
+    """
+    An LLM experiment evaluator that checks how a response is related to reference text.
+
+    `LLMRelationalEvaluator` uses the chain-of-thought technique to perform a binary evaluation of
+    how a response is related to reference text in a specified manner. When used as an experiment
+    evaluator, `LLMRelationalEvaluator` will return a score of 1.0 if the response is related to
+    the reference text in the specified manner and a score of 0.0 if not. The explanation
+    produced by the chain-of-thought technique will be included in the experiment evaluation as
+    well.
+
+    In order to evaluate how a response is related to reference text, a specific relation and
+    description of that relation must be specified. The relation should be a phrase that can be
+    used in the following manner: "The response '{relation}' the reference". The description
+    should complete the sentence "In this context, '{relation}' means the response {description}".
+
+    Example relations and descriptions:
+        - "is a good summary of" - "the response clearly concisely summarizes the reference"
+        - "directly quotes" - "the response contains specific information from the reference"
+        - "professionally addresses" - "the response is respectful and relevant to the reference"
+
+    Args:
+        model: The LLM model wrapper to use for evaluation. Compatible models can be imported from
+            the `phoenix.evals` module.
+        relation: The relation to evaluate the text against, the relation should be a phrase that
+            can be used in the following manner: "The response '{relation}' the reference".
+        description (str): A description of the relation, used to clarify instructions to the LLM.
+            The description should complete the sentence "In this context, '{relation}'
+            means {description}". It is helpful to specifically use the words "response" and
+            "reference" to describe the relation.
+        name (str): The name of the evaluator
+        get_reference (callable, optional): A function that extracts the reference from the input of
+            the experiment task. The function should take the input and metadata of the dataset
+            example and return a string. By default, the function will return the string
+            representation of the input.
+        get_response (callable, optional): A function that extracts the response from the output of
+            the experiment task. The function should take the output and metadata of the experiment
+            task and return a string. By default, the function will return the string representation
+            of the output.
+    """
+
+    _base_template = (
+        "Determine if the following response '{relation}' the reference. {description}"
+        "First, explain step-by-step why you think the response '{relation}' the reference. "
+        "Then provide a single word label; 'true' if the response '{relation}' the reference or "
+        "'false' if the text is not '{relation}' to the reference. "
+        "Here is an example template for your reponse:\n\n"
+        "CRITERIA: the response '{relation}' the reference\n"
+        "REFERENCE: *text that contains a reference*\n"
+        "RESPONSE: *a response that may or may not be '{relation}' to the reference*\n"
+        "EXPLANATION: *a step by step explanation of your reasoning for whether or not the "
+        "response '{relation}' the reference*\n"
+        "LABEL: *true or false*\n\n"
+        "Follow this template for the following example:\n\n"
+        "CRITERIA: the response '{relation}' the reference\n"
+        "REFERENCE: {reference}\n"
+        "RESPONSE: {response}\n"
+        "EXPLANATION: "
+    )
+    _description = "In this context, '{relation}' means '{description}'. "
+
+    def __init__(
+        self,
+        model: LLMBaseModel,
+        relation: str,
+        description: str,
+        name: str,
+        get_reference: Optional[Callable[[ExampleInput, ExampleMetadata], str]] = None,
+        get_response: Optional[Callable[[Optional[TaskOutput], ExampleMetadata], str]] = None,
+    ):
+        self.model = model
+        self._name = name
+        self.relation = relation
+        self.description = description
+        self.template = self._format_base_template(self.relation, self.description)
+        self.get_reference = get_reference or self._default_get_reference
+        self.get_response = get_response or self._default_get_response
+
+    @classmethod
+    def _format_base_template(cls, relation: str, description: Optional[str] = None) -> str:
+        formatted_description = cls._description.format(relation=relation, description=description)
+        formatted_template = cls._base_template.format(
+            relation=relation,
+            description=formatted_description,
+            response="{response}",  # leave the response field as a placeholder
+            reference="{reference}",  # leave the reference field as a placeholder
+        )
+        return formatted_template
+
+    def _format_eval_template(
+        self,
+        output: Optional[TaskOutput] = None,
+        input: ExampleInput = MappingProxyType({}),
+        metadata: ExampleMetadata = MappingProxyType({}),
+    ) -> str:
+        assert output is not None
+        reference = self.get_reference(input, metadata)
+        response = self.get_response(output, metadata)
+        return self.template.format(reference=reference, response=response)
+
+    def _parse_eval_output(self, unparsed_response: str) -> EvaluationResult:
+        raw_label, explanation = (
+            _parse_label_from_explanation(unparsed_response),
+            unparsed_response,
+        )
+        label = snap_to_rail(raw_label, ["true", "false"])
+        if label == "true":
+            score = 1.0
+        elif label == "false":
+            score = 0.0
+        else:
+            raise RuntimeError(f"Could not parse LLM evaluation: {unparsed_response}")
+        return EvaluationResult(
+            score=score,
+            explanation=explanation,
+            metadata={},
+        )
+
+    def _default_get_reference(self, input: ExampleInput, *args: Any, **kwargs: Any) -> str:
+        return str(input)
+
+    def _default_get_response(
+        self, output: Optional[TaskOutput] = None, *args: Any, **kwargs: Any
+    ) -> str:
+        assert output is not None
+        return str(unwrap_json(output))
+
+    def evaluate(
+        self,
+        *,
+        output: Optional[TaskOutput] = None,
+        metadata: ExampleMetadata = MappingProxyType({}),
+        input: ExampleInput = MappingProxyType({}),
+        **_: Any,
+    ) -> EvaluationResult:
+        formatted_template = self._format_eval_template(output, input, metadata)
+        unparsed_response = self.model._generate(formatted_template)
+        return self._parse_eval_output(unparsed_response)
+
+    async def async_evaluate(
+        self,
+        *,
+        output: Optional[TaskOutput] = None,
+        metadata: ExampleMetadata = MappingProxyType({}),
+        input: ExampleInput = MappingProxyType({}),
+        **_: Any,
+    ) -> EvaluationResult:
+        formatted_template = self._format_eval_template(output, input, metadata)
+        unparsed_response = await self.model._async_generate(formatted_template)
+        return self._parse_eval_output(unparsed_response)

phoenix/experiments/evaluators/utils.py

@@ -1,6 +1,7 @@
 import functools
 import inspect
-from typing import TYPE_CHECKING, Any, Callable, Optional, Union
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any, Optional, Union
 
 from phoenix.experiments.types import (
     AnnotatorKind,
@@ -134,7 +135,7 @@ def create_evaluator(
             from textdistance import levenshtein
 
             @create_evaluator(kind="CODE", name="levenshtein-distance")
-            def ld(output: str, expected: str) -> Tuple[float, str]:
+            def ld(output: str, expected: str) -> tuple[float, str]:
                 return (
                     levenshtein(output, expected),
                     f"Levenshtein distance between {output} and {expected}"

phoenix/experiments/functions.py

@@ -4,24 +4,13 @@ import inspect
 import json
 import traceback
 from binascii import hexlify
+from collections.abc import Awaitable, Mapping, Sequence
 from contextlib import ExitStack
 from copy import deepcopy
 from dataclasses import replace
 from datetime import datetime, timezone
 from itertools import product
-from typing import (
-    Any,
-    Awaitable,
-    Dict,
-    Literal,
-    Mapping,
-    Optional,
-    Sequence,
-    Tuple,
-    Type,
-    Union,
-    cast,
-)
+from typing import Any, Literal, Optional, Union, cast
 from urllib.parse import urljoin
 
 import httpx
@@ -76,7 +65,7 @@ from phoenix.utilities.client import VersionedAsyncClient, VersionedClient
 from phoenix.utilities.json import jsonify
 
 
-def _phoenix_clients() -> Tuple[httpx.Client, httpx.AsyncClient]:
+def _phoenix_clients() -> tuple[httpx.Client, httpx.AsyncClient]:
     return VersionedClient(
         base_url=get_base_url(),
     ), VersionedAsyncClient(
@@ -91,7 +80,7 @@ Evaluators: TypeAlias = Union[
 ]
 
 
-RateLimitErrors: TypeAlias = Union[Type[BaseException], Sequence[Type[BaseException]]]
+RateLimitErrors: TypeAlias = Union[type[BaseException], Sequence[type[BaseException]]]
 
 
 def run_experiment(
@@ -369,7 +358,7 @@ def run_experiment(
             exp_run = replace(exp_run, id=resp.json()["data"]["id"])
         return exp_run
 
-    _errors: Tuple[Type[BaseException], ...]
+    _errors: tuple[type[BaseException], ...]
     if not isinstance(rate_limit_errors, Sequence):
         _errors = (rate_limit_errors,) if rate_limit_errors is not None else ()
     else:
@@ -498,7 +487,7 @@ def evaluate_experiment(
     root_span_kind = EVALUATOR
 
     def sync_evaluate_run(
-        obj: Tuple[Example, ExperimentRun, Evaluator],
+        obj: tuple[Example, ExperimentRun, Evaluator],
     ) -> ExperimentEvaluationRun:
         example, experiment_run, evaluator = obj
         result: Optional[EvaluationResult] = None
@@ -550,7 +539,7 @@ def evaluate_experiment(
         return eval_run
 
     async def async_evaluate_run(
-        obj: Tuple[Example, ExperimentRun, Evaluator],
+        obj: tuple[Example, ExperimentRun, Evaluator],
     ) -> ExperimentEvaluationRun:
         example, experiment_run, evaluator = obj
         result: Optional[EvaluationResult] = None
@@ -611,7 +600,7 @@ def evaluate_experiment(
             eval_run = replace(eval_run, id=resp.json()["data"]["id"])
         return eval_run
 
-    _errors: Tuple[Type[BaseException], ...]
+    _errors: tuple[type[BaseException], ...]
     if not isinstance(rate_limit_errors, Sequence):
         _errors = (rate_limit_errors,) if rate_limit_errors is not None else ()
     else:
@@ -649,7 +638,7 @@ def evaluate_experiment(
 
 
 def _evaluators_by_name(obj: Optional[Evaluators]) -> Mapping[EvaluatorName, Evaluator]:
-    evaluators_by_name: Dict[EvaluatorName, Evaluator] = {}
+    evaluators_by_name: dict[EvaluatorName, Evaluator] = {}
     if obj is None:
         return evaluators_by_name
     if isinstance(mapping := obj, Mapping):
@@ -678,7 +667,7 @@ def _evaluators_by_name(obj: Optional[Evaluators]) -> Mapping[EvaluatorName, Eva
     return evaluators_by_name
 
 
-def _get_tracer(project_name: Optional[str] = None) -> Tuple[Tracer, Resource]:
+def _get_tracer(project_name: Optional[str] = None) -> tuple[Tracer, Resource]:
     resource = Resource({ResourceAttributes.PROJECT_NAME: project_name} if project_name else {})
     tracer_provider = trace_sdk.TracerProvider(resource=resource)
     span_processor = (

phoenix/experiments/tracing.py

@@ -1,9 +1,10 @@
 from __future__ import annotations
 
+from collections.abc import Callable, Iterator
 from contextlib import contextmanager
 from contextvars import ContextVar
 from threading import Lock
-from typing import Any, Callable, Iterator, Optional
+from typing import Any, Optional
 
 from opentelemetry.sdk.resources import Resource
 from opentelemetry.sdk.trace import ReadableSpan

phoenix/experiments/types.py

@@ -3,6 +3,13 @@ from __future__ import annotations
 import json
 import textwrap
 from collections import Counter
+from collections.abc import (
+    Awaitable,
+    Callable,
+    Iterable,
+    Iterator,
+    Mapping,
+)
 from copy import copy, deepcopy
 from dataclasses import dataclass, field, fields
 from datetime import datetime
@@ -10,23 +17,7 @@ from enum import Enum
 from functools import cached_property
 from importlib.metadata import version
 from random import getrandbits
-from typing import (
-    Any,
-    Awaitable,
-    Callable,
-    Dict,
-    FrozenSet,
-    Iterable,
-    Iterator,
-    List,
-    Mapping,
-    Optional,
-    Tuple,
-    TypeVar,
-    Union,
-    cast,
-    overload,
-)
+from typing import Any, Optional, TypeVar, Union, cast, overload
 
 import pandas as pd
 from typing_extensions import TypeAlias
@@ -41,7 +32,7 @@ class AnnotatorKind(Enum):
     LLM = "LLM"
 
 
-JSONSerializable: TypeAlias = Optional[Union[Dict[str, Any], List[Any], str, int, float, bool]]
+JSONSerializable: TypeAlias = Optional[Union[dict[str, Any], list[Any], str, int, float, bool]]
 ExperimentId: TypeAlias = str
 DatasetId: TypeAlias = str
 DatasetVersionId: TypeAlias = str
@@ -63,7 +54,7 @@ Explanation: TypeAlias = Optional[str]
 EvaluatorName: TypeAlias = str
 EvaluatorKind: TypeAlias = str
 EvaluatorOutput: TypeAlias = Union[
-    "EvaluationResult", bool, int, float, str, Tuple[Score, Label, Explanation]
+    "EvaluationResult", bool, int, float, str, tuple[Score, Label, Explanation]
 ]
 
 DRY_RUN: ExperimentId = "DRY_RUN"
@@ -135,14 +126,14 @@ class Dataset:
         return iter(self.examples.values())
 
     @cached_property
-    def _keys(self) -> Tuple[str, ...]:
+    def _keys(self) -> tuple[str, ...]:
         return tuple(self.examples.keys())
 
     @overload
     def __getitem__(self, key: int) -> Example: ...
     @overload
-    def __getitem__(self, key: slice) -> List[Example]: ...
-    def __getitem__(self, key: Union[int, slice]) -> Union[Example, List[Example]]:
+    def __getitem__(self, key: slice) -> list[Example]: ...
+    def __getitem__(self, key: Union[int, slice]) -> Union[Example, list[Example]]:
         if isinstance(key, int):
             return self.examples[self._keys[key]]
         return [self.examples[k] for k in self._keys[key]]
@@ -306,7 +297,7 @@ class ExperimentParameters:
 
 @dataclass(frozen=True)
 class EvaluationParameters:
-    eval_names: FrozenSet[str]
+    eval_names: frozenset[str]
     exp_params: ExperimentParameters
 
 
@@ -485,8 +476,8 @@ class RanExperiment(Experiment):
     dataset: Dataset = field(repr=False)
     runs: Mapping[ExperimentRunId, ExperimentRun] = field(repr=False)
     task_summary: TaskSummary = field(repr=False)
-    eval_runs: Tuple[ExperimentEvaluationRun, ...] = field(repr=False, default=())
-    eval_summaries: Tuple[EvaluationSummary, ...] = field(repr=False, default=())
+    eval_runs: tuple[ExperimentEvaluationRun, ...] = field(repr=False, default=())
+    eval_summaries: tuple[EvaluationSummary, ...] = field(repr=False, default=())
 
     @property
     def url(self) -> str:
@@ -514,14 +505,14 @@ class RanExperiment(Experiment):
         return iter(self.runs.values())
 
     @cached_property
-    def _keys(self) -> Tuple[str, ...]:
+    def _keys(self) -> tuple[str, ...]:
         return tuple(self.runs.keys())
 
     @overload
     def __getitem__(self, key: int) -> ExperimentRun: ...
     @overload
-    def __getitem__(self, key: slice) -> List[ExperimentRun]: ...
-    def __getitem__(self, key: Union[int, slice]) -> Union[ExperimentRun, List[ExperimentRun]]:
+    def __getitem__(self, key: slice) -> list[ExperimentRun]: ...
+    def __getitem__(self, key: Union[int, slice]) -> Union[ExperimentRun, list[ExperimentRun]]:
         if isinstance(key, int):
             return self.runs[self._keys[key]]
         return [self.runs[k] for k in self._keys[key]]
@@ -596,7 +587,7 @@ class RanExperiment(Experiment):
         raise NotImplementedError
 
 
-def _asdict(dc: Any) -> Dict[str, Any]:
+def _asdict(dc: Any) -> dict[str, Any]:
     # non-recursive version of `dataclasses.asdict()`
     return {field.name: getattr(dc, field.name) for field in fields(dc)}
 

phoenix/experiments/utils.py

@@ -1,5 +1,6 @@
 import functools
-from typing import Any, Callable
+from collections.abc import Callable
+from typing import Any
 
 from phoenix.config import get_web_base_url
 

phoenix/inferences/errors.py

@@ -1,5 +1,6 @@
 from abc import abstractmethod
-from typing import Any, Iterable, List, Union
+from collections.abc import Iterable
+from typing import Any, Union
 
 
 class ValidationError(Exception):
@@ -57,8 +58,8 @@ class InvalidSchemaError(ValidationError):
 class DatasetError(Exception):
     """An error raised when the dataset is invalid or incomplete"""
 
-    def __init__(self, errors: Union[ValidationError, List[ValidationError]]):
-        self.errors: List[ValidationError] = errors if isinstance(errors, list) else [errors]
+    def __init__(self, errors: Union[ValidationError, list[ValidationError]]):
+        self.errors: list[ValidationError] = errors if isinstance(errors, list) else [errors]
 
     def __str__(self) -> str:
         return "\n".join(map(str, self.errors))
@@ -142,7 +143,7 @@ class EmbeddingVectorSizeMismatch(ValidationError):
     vector lengths"""
 
     def __init__(
-        self, embedding_feature_name: str, vector_column_name: str, vector_lengths: List[int]
+        self, embedding_feature_name: str, vector_column_name: str, vector_lengths: list[int]
     ) -> None:
         self.embedding_feature_name = embedding_feature_name
         self.vector_column_name = vector_column_name
@@ -238,5 +239,5 @@ class MissingTimestampColumnName(ValidationError):
 class SchemaError(Exception):
     """An error raised when the Schema is invalid or incomplete"""
 
-    def __init__(self, errors: Union[ValidationError, List[ValidationError]]):
+    def __init__(self, errors: Union[ValidationError, list[ValidationError]]):
         self.errors = errors

phoenix/inferences/fixtures.py

@@ -1,9 +1,10 @@
 import json
 import logging
+from collections.abc import Iterator
 from dataclasses import dataclass, replace
 from enum import Enum, auto
 from pathlib import Path
-from typing import Iterator, NamedTuple, Optional, Tuple
+from typing import NamedTuple, Optional
 from urllib import request
 from urllib.parse import quote, urljoin
 
@@ -39,7 +40,7 @@ class Fixture:
     corpus_file_name: Optional[str] = None
     corpus_schema: Optional[Schema] = None
 
-    def paths(self) -> Iterator[Tuple[InferencesRole, Path]]:
+    def paths(self) -> Iterator[tuple[InferencesRole, Path]]:
         return (
             (role, Path(self.prefix) / name)
             for role, name in zip(
@@ -397,7 +398,7 @@ wikipedia_fixture = Fixture(
     corpus_file_name="corpus.parquet",
 )
 
-FIXTURES: Tuple[Fixture, ...] = (
+FIXTURES: tuple[Fixture, ...] = (
     sentiment_classification_language_drift_fixture,
     image_classification_fixture,
     fashion_mnist_fixture,
@@ -416,7 +417,7 @@ NAME_TO_FIXTURE = {fixture.name: fixture for fixture in FIXTURES}
 def get_inferences(
     fixture_name: str,
     no_internet: bool = False,
-) -> Tuple[Inferences, Optional[Inferences], Optional[Inferences]]:
+) -> tuple[Inferences, Optional[Inferences], Optional[Inferences]]:
     """
     Downloads primary and reference inferences for a fixture if they are not found
     locally.
@@ -550,7 +551,7 @@ class GCSAssets(NamedTuple):
         )
 
 
-def _download(fixture: Fixture, location: Path) -> Iterator[Tuple[InferencesRole, Path]]:
+def _download(fixture: Fixture, location: Path) -> Iterator[tuple[InferencesRole, Path]]:
     for role, path in fixture.paths():
         yield role, GCSAssets().metadata(path).save_artifact(location)